diff --git a/.github/workflows/deploy-jth-sandbox.yml b/.github/workflows/deploy-jth-sandbox.yml new file mode 100644 index 000000000..ed3056476 --- /dev/null +++ b/.github/workflows/deploy-jth-sandbox.yml @@ -0,0 +1,89 @@ +name: Deploy jth sandbox to AWS + +on: + push: + branches: + - hyp3-jth-sandbox + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-jth-sandbox + template_bucket: cf-templates-bywc0durdnqy-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: DEFAULT + opera_rtc_s1_end_date: Default + job_files: >- + job_spec/AUTORIFT.yml + job_spec/INSAR_GAMMA.yml + job_spec/RTC_GAMMA.yml + job_spec/INSAR_ISCE_BURST.yml + job_spec/INSAR_ISCE_MULTI_BURST.yml + job_spec/ARIA_S1_GUNW.yml + job_spec/OPERA_RTC_S1.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: EDC + ami_id: /ngap/amis/image_id_ecs_al2023_x86 + distribution_url: '' + + environment: + name: ${{ matrix.environment }} + + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - run: | + jq -n '{"Parameters": $ARGS.named}' \ + --arg VpcId '${{ secrets.VPC_ID }}' \ + --arg SubnetIds '${{ secrets.SUBNET_IDS }}' \ + --arg SecretArn '${{ secrets.SECRET_ARN }}' \ + --arg ImageTag '${{ matrix.IMAGE_TAG }}' \ + --arg ProductLifetimeInDays '${{ matrix.product_lifetime_in_days }} ' \ + --arg AuthPublicKey '${{ secrets.AUTH_PUBLIC_KEY }}' \ + --arg DefaultCreditsPerUser '${{ matrix.default_credits_per_user }}' \ + --arg DefaultApplicationStatus '${{ matrix.default_application_status }}' \ + --arg OperaRtcS1EndDate '${{ matrix.opera_rtc_s1_end_date }}' \ + --arg AmiId '${{ matrix.ami_id }}' \ + --arg DefaultMaxvCpus '${{ matrix.default_max_vcpus }}' \ + --arg ExpandedMaxvCpus '${{ matrix.expanded_max_vcpus }}' \ + --arg MonthlyBudget '${{ secrets.MONTHLY_BUDGET }}' \ + --arg RequiredSurplus '${{ matrix.required_surplus }}' \ + --arg InstanceTypes '${{ matrix.instance_types }}' \ + --arg BucketReadPrincipals '${{ secrets.BUCKET_READ_PRINCIPALS }}' \ + --arg DistributionUrl '${{ matrix.distribution_url }}' \ + > parameters.json + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + API_NAME: ${{ matrix.environment }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + COST_PROFILE: ${{ matrix.cost_profile }} + JOB_FILES: ${{ matrix.job_files }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + PARAMETER_FILE: parameters.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 87560b194..de635d3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.12.0] + +### Added +- Added a new `PATCH /jobs` endpoint that accepts up to 100 job IDs and updates those jobs with the given `name` value. The update is not transactional, so it's possible that only some of the jobs will be updated if an error occurs. See https://github.com/ASFHyP3/hyp3/issues/2972 + ## [10.11.18] ### Fixed diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index f427c86a9..36898e79d 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -46,6 +46,24 @@ paths: schema: $ref: "#/components/schemas/jobs_response" + patch: + description: |- + Update a list of up to 100 jobs. The update is not transactional, + so it's possible that only some of the jobs will be updated if an error occurs. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/patch_jobs_body" + required: true + responses: + "200": + description: 200 response + content: + application/json: + schema: + type: object + get: description: Get list of previously run jobs. parameters: @@ -199,6 +217,19 @@ components: jobs: $ref: "#/components/schemas/list_of_new_jobs" + patch_jobs_body: + description: List of jobs to update with a new name. + type: object + required: + - job_ids + - name + additionalProperties: false + properties: + job_ids: + $ref: "#/components/schemas/job_ids_list" + name: + $ref: "#/components/schemas/name_patch" + jobs_response: description: List of submitted jobs. type: object @@ -263,6 +294,11 @@ components: type: number minimum: 0 + job_ids_list: + type: array + items: + $ref: "#/components/schemas/job_id" + job_names_list: type: array items: diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index ad8f1491f..4aabb1088 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -72,8 +72,28 @@ def get_job_by_id(job_id: str) -> dict: def patch_job_by_id(body: dict, job_id: str, user: str) -> dict: + return _patch_job(job_id, body['name'], user) + + +def patch_jobs(body: dict, user: str) -> None: + job_ids = body['job_ids'] + name = body['name'] + + if len(job_ids) == 0: + abort(problem_format(400, 'Must provide at least one job ID')) + + # Max job IDs value is also documented in OpenAPI spec + max_job_ids = 100 + if len(job_ids) > max_job_ids: + abort(problem_format(400, f'Cannot update more than {max_job_ids} jobs')) + + for job_id in set(job_ids): + _patch_job(job_id, name, user) + + +def _patch_job(job_id: str, name: str, user: str) -> dict: try: - job = dynamo.jobs.update_job_for_user(job_id, body['name'], user) + job = dynamo.jobs.update_job_for_user(job_id, name, user) except UpdateJobForDifferentUserError as e: abort(problem_format(403, str(e))) except UpdateJobNotFoundError as e: diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 548c163d9..4ede62204 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -145,6 +145,13 @@ def jobs_post() -> Response: return jsonify(handlers.post_jobs(request.get_json(), g.user)) +@app.route('/jobs', methods=['PATCH']) +@openapi +def jobs_patch() -> Response: + handlers.patch_jobs(request.get_json(), g.user) + return jsonify({}) + + @app.route('/jobs', methods=['GET']) @openapi def jobs_get() -> Response: diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 4f86d8f04..1b29799fa 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -262,7 +262,9 @@ def update_job_for_user(job_id: str, name: str | None, user_id: str) -> dict: raise UpdateJobNotFoundError(f'Job {job_id} does not exist') else: assert e.response['Item']['user_id']['S'] != user_id - raise UpdateJobForDifferentUserError("You cannot modify a different user's job") + raise UpdateJobForDifferentUserError( + f'You cannot modify job {job_id} because it belongs to a different user' + ) raise return job diff --git a/tests/test_api/test_api_spec.py b/tests/test_api/test_api_spec.py index b28c98259..86136fcd0 100644 --- a/tests/test_api/test_api_spec.py +++ b/tests/test_api/test_api_spec.py @@ -4,7 +4,7 @@ ENDPOINTS = { - JOBS_URI: {'GET', 'HEAD', 'OPTIONS', 'POST'}, + JOBS_URI: {'GET', 'HEAD', 'OPTIONS', 'POST', 'PATCH'}, JOBS_URI + '/foo': {'GET', 'HEAD', 'OPTIONS', 'PATCH'}, USER_URI: {'GET', 'HEAD', 'OPTIONS', 'PATCH'}, } diff --git a/tests/test_api/test_patch_job_by_id.py b/tests/test_api/test_patch_job_by_id.py index e160d63a4..dd071d60a 100644 --- a/tests/test_api/test_patch_job_by_id.py +++ b/tests/test_api/test_patch_job_by_id.py @@ -144,11 +144,17 @@ def test_patch_job_different_user(client, tables): response = client.patch(f'{JOBS_URI}/40183948-48a1-42d2-a96b-ce44fbba301b', json={'name': 'newname'}) assert response.status_code == HTTPStatus.FORBIDDEN - assert response.json['detail'] == "You cannot modify a different user's job" + assert ( + response.json['detail'] + == 'You cannot modify job 40183948-48a1-42d2-a96b-ce44fbba301b because it belongs to a different user' + ) response = client.patch(f'{JOBS_URI}/40183948-48a1-42d2-a96b-ce44fbba301b', json={'name': None}) assert response.status_code == HTTPStatus.FORBIDDEN - assert response.json['detail'] == "You cannot modify a different user's job" + assert ( + response.json['detail'] + == 'You cannot modify job 40183948-48a1-42d2-a96b-ce44fbba301b because it belongs to a different user' + ) assert tables.jobs_table.scan()['Items'] == [ { diff --git a/tests/test_api/test_patch_jobs.py b/tests/test_api/test_patch_jobs.py new file mode 100644 index 000000000..779f184b9 --- /dev/null +++ b/tests/test_api/test_patch_jobs.py @@ -0,0 +1,298 @@ +from http import HTTPStatus + +from test_api.conftest import JOBS_URI, login + + +def test_patch_jobs(client, tables): + table_items = [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname1', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'name': 'oldname2', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': 'b0dd41bc-5597-4a58-afe4-81f30e53bbb0', + 'name': 'oldname3', + 'somefield': 'somevalue', + 'user_id': 'user2', + }, + ] + + for item in table_items: + tables.jobs_table.put_item(Item=item) + + login(client, 'user1') + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': [ + '33d85ea0-9342-4c21-ae59-5bec3f71612c', + '40183948-48a1-42d2-a96b-ce44fbba301b', + ], + 'name': 'newname', + }, + ) + + assert response.status_code == HTTPStatus.OK + assert response.json == {} + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'newname', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'name': 'newname', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': 'b0dd41bc-5597-4a58-afe4-81f30e53bbb0', + 'name': 'oldname3', + 'somefield': 'somevalue', + 'user_id': 'user2', + }, + ] + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': [ + '33d85ea0-9342-4c21-ae59-5bec3f71612c', + '40183948-48a1-42d2-a96b-ce44fbba301b', + ], + 'name': None, + }, + ) + + assert response.status_code == HTTPStatus.OK + assert response.json == {} + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': 'b0dd41bc-5597-4a58-afe4-81f30e53bbb0', + 'name': 'oldname3', + 'somefield': 'somevalue', + 'user_id': 'user2', + }, + ] + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': [ + '33d85ea0-9342-4c21-ae59-5bec3f71612c', + '40183948-48a1-42d2-a96b-ce44fbba301b', + ], + 'name': 'anothernewname', + }, + ) + + assert response.status_code == HTTPStatus.OK + assert response.json == {} + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'anothernewname', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'name': 'anothernewname', + 'somefield': 'somevalue', + 'user_id': 'user1', + }, + { + 'job_id': 'b0dd41bc-5597-4a58-afe4-81f30e53bbb0', + 'name': 'oldname3', + 'somefield': 'somevalue', + 'user_id': 'user2', + }, + ] + + +def test_job_ids_length(client, tables): + table_items = [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname', + 'user_id': 'user1', + }, + ] + + for item in table_items: + tables.jobs_table.put_item(Item=item) + + login(client, 'user1') + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'] * 100, + 'name': 'newname', + }, + ) + assert response.status_code == HTTPStatus.OK + assert response.json == {} + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'newname', + 'user_id': 'user1', + }, + ] + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'] * 101, + 'name': 'newname2', + }, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.json['detail'] == 'Cannot update more than 100 jobs' + + response = client.patch( + JOBS_URI, + json={ + 'job_ids': [], + 'name': 'newname2', + }, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.json['detail'] == 'Must provide at least one job ID' + + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'newname', + 'user_id': 'user1', + }, + ] + + +def test_job_not_found(client, tables): + table_items = [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname', + 'user_id': 'user1', + }, + ] + for item in table_items: + tables.jobs_table.put_item(Item=item) + + login(client, 'user1') + + response = client.patch(JOBS_URI, json={'job_ids': ['40183948-48a1-42d2-a96b-ce44fbba301b'], 'name': 'newname'}) + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.json['detail'] == 'Job 40183948-48a1-42d2-a96b-ce44fbba301b does not exist' + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname', + 'user_id': 'user1', + }, + ] + + +def test_different_user(client, tables): + table_items = [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'name': 'oldname', + 'user_id': 'user2', + }, + ] + for item in table_items: + tables.jobs_table.put_item(Item=item) + + login(client, 'user1') + + response = client.patch(JOBS_URI, json={'job_ids': ['40183948-48a1-42d2-a96b-ce44fbba301b'], 'name': 'newname'}) + assert response.status_code == HTTPStatus.FORBIDDEN + assert ( + response.json['detail'] + == 'You cannot modify job 40183948-48a1-42d2-a96b-ce44fbba301b because it belongs to a different user' + ) + + response = client.patch(JOBS_URI, json={'job_ids': ['40183948-48a1-42d2-a96b-ce44fbba301b'], 'name': None}) + assert response.status_code == HTTPStatus.FORBIDDEN + assert ( + response.json['detail'] + == 'You cannot modify job 40183948-48a1-42d2-a96b-ce44fbba301b because it belongs to a different user' + ) + + assert tables.jobs_table.scan()['Items'] == [ + { + 'job_id': '33d85ea0-9342-4c21-ae59-5bec3f71612c', + 'name': 'oldname', + 'user_id': 'user1', + }, + { + 'job_id': '40183948-48a1-42d2-a96b-ce44fbba301b', + 'name': 'oldname', + 'user_id': 'user2', + }, + ] + + +def test_patch_jobs_schema(client, tables): + login(client, 'user1') + + response = client.patch(JOBS_URI, json={'name': 'newname'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "'job_ids' is a required property" in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c']}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "'name' is a required property" in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'], 'foo': 'bar'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "'foo' was unexpected" in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['foo'], 'name': 'newname'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "'foo' is not a 'uuid'" in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'], 'name': ''}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "'' should be non-empty" in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'], 'name': '-' * 100}) + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.json['detail'] == 'Job 33d85ea0-9342-4c21-ae59-5bec3f71612c does not exist' + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'], 'name': '-' * 101}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert 'is too long' in response.json['detail'] + + response = client.patch(JOBS_URI, json={'job_ids': ['33d85ea0-9342-4c21-ae59-5bec3f71612c'], 'name': True}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "True is not of type 'string'" in response.json['detail'] diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 9575f7d82..5a6d9ca71 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -832,10 +832,14 @@ def test_update_job_for_different_user(tables): for item in table_items: tables.jobs_table.put_item(Item=item) - with pytest.raises(UpdateJobForDifferentUserError, match=r'^You cannot modify a different user\'s job$'): + with pytest.raises( + UpdateJobForDifferentUserError, match=r'^You cannot modify job job2 because it belongs to a different user$' + ): dynamo.jobs.update_job_for_user('job2', 'newname', 'user1') - with pytest.raises(UpdateJobForDifferentUserError, match=r'^You cannot modify a different user\'s job$'): + with pytest.raises( + UpdateJobForDifferentUserError, match=r'^You cannot modify job job2 because it belongs to a different user$' + ): dynamo.jobs.update_job_for_user('job2', None, 'user1') assert tables.jobs_table.scan()['Items'] == [