From ffa1d88537a904dbfe072478c5be1dcf43769caf Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 2 Dec 2025 09:55:18 -0900 Subject: [PATCH 01/94] add deploy-sandbox.yml --- .github/workflows/deploy-sandbox.yml | 89 ++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/deploy-sandbox.yml diff --git a/.github/workflows/deploy-sandbox.yml b/.github/workflows/deploy-sandbox.yml new file mode 100644 index 000000000..851cb0607 --- /dev/null +++ b/.github/workflows/deploy-sandbox.yml @@ -0,0 +1,89 @@ +name: Deploy HyP3 Sandbox to AWS + +on: + push: + branches: + - hyp3-sandbox + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-sandbox + template_bucket: cf-templates-bywc0durdnqy-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: DEFAULT + opera_rtc_s1_end_date: Default + job_files: >- + job_spec/AUTORIFT.yml + job_spec/INSAR_GAMMA.yml + job_spec/RTC_GAMMA.yml + job_spec/INSAR_ISCE_BURST.yml + job_spec/INSAR_ISCE_MULTI_BURST.yml + job_spec/ARIA_S1_GUNW.yml + job_spec/OPERA_RTC_S1.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: EDC + ami_id: /ngap/amis/image_id_ecs_al2023_x86 + distribution_url: '' # TODO + + environment: + name: ${{ matrix.environment }} + + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - run: | + jq -n '{"Parameters": $ARGS.named}' \ + --arg VpcId '${{ secrets.VPC_ID }}' \ + --arg SubnetIds '${{ secrets.SUBNET_IDS }}' \ + --arg SecretArn '${{ secrets.SECRET_ARN }}' \ + --arg ImageTag '${{ matrix.IMAGE_TAG }}' \ + --arg ProductLifetimeInDays '${{ matrix.product_lifetime_in_days }} ' \ + --arg AuthPublicKey '${{ secrets.AUTH_PUBLIC_KEY }}' \ + --arg DefaultCreditsPerUser '${{ matrix.default_credits_per_user }}' \ + --arg DefaultApplicationStatus '${{ matrix.default_application_status }}' \ + --arg OperaRtcS1EndDate '${{ matrix.opera_rtc_s1_end_date }}' \ + --arg AmiId '${{ matrix.ami_id }}' \ + --arg DefaultMaxvCpus '${{ matrix.default_max_vcpus }}' \ + --arg ExpandedMaxvCpus '${{ matrix.expanded_max_vcpus }}' \ + --arg MonthlyBudget '${{ secrets.MONTHLY_BUDGET }}' \ + --arg RequiredSurplus '${{ matrix.required_surplus }}' \ + --arg InstanceTypes '${{ matrix.instance_types }}' \ + --arg BucketReadPrincipals '${{ secrets.BUCKET_READ_PRINCIPALS }}' \ + --arg DistributionUrl '${{ matrix.distribution_url }}' \ + > parameters.json + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + API_NAME: ${{ matrix.environment }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + COST_PROFILE: ${{ matrix.cost_profile }} + JOB_FILES: ${{ matrix.job_files }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + PARAMETER_FILE: parameters.json From 49a3eabd8c67deb44d055b7eab5ec47c66e54954 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 2 Dec 2025 10:52:20 -0900 Subject: [PATCH 02/94] rename hyp3-sandbox to hyp3-edc-sandbox --- .../{deploy-sandbox.yml => deploy-edc-sandbox.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{deploy-sandbox.yml => deploy-edc-sandbox.yml} (97%) diff --git a/.github/workflows/deploy-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml similarity index 97% rename from .github/workflows/deploy-sandbox.yml rename to .github/workflows/deploy-edc-sandbox.yml index 851cb0607..79874fbb3 100644 --- a/.github/workflows/deploy-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -1,9 +1,9 @@ -name: Deploy HyP3 Sandbox to AWS +name: Deploy HyP3 EDC Sandbox to AWS on: push: branches: - - hyp3-sandbox + - hyp3-edc-sandbox concurrency: ${{ github.workflow }}-${{ github.ref }} @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: include: - - environment: hyp3-sandbox + - environment: hyp3-edc-sandbox template_bucket: cf-templates-bywc0durdnqy-us-west-2 image_tag: test product_lifetime_in_days: 14 From dff2aa6680b14bf3b18617470e46e64d7ee07635 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 2 Dec 2025 11:41:18 -0900 Subject: [PATCH 03/94] deploy-edc-sandbox perms --- .github/workflows/deploy-edc-sandbox.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index 79874fbb3..2ef2e493b 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -1,5 +1,8 @@ name: Deploy HyP3 EDC Sandbox to AWS +permissions: + contents: read + on: push: branches: From 3d29a01903abaefa11580b6240d3af348f3b667b Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 10 Dec 2025 17:04:46 -0800 Subject: [PATCH 04/94] Update OPERA_DIST_S1.yml --- job_spec/OPERA_DIST_S1.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/job_spec/OPERA_DIST_S1.yml b/job_spec/OPERA_DIST_S1.yml index 62afe01f3..1daba4572 100644 --- a/job_spec/OPERA_DIST_S1.yml +++ b/job_spec/OPERA_DIST_S1.yml @@ -140,6 +140,12 @@ OPERA_DIST_S1: type: boolean default: false example: false + model_context_length: + api_schema: + description: Maximum number of baseline images to use. If -1, use largest allowable baseline. + type: int + default: -1 + example: 10 cost_profiles: DEFAULT: cost: 1.0 @@ -186,6 +192,8 @@ OPERA_DIST_S1: - Ref::model_compilation - --use_date_encoding - Ref::use_date_encoding + - --model_context_length + - Ref::model_context_length timeout: 3600 # 1 hr compute_environment: DistS1 vcpu: 1 From 4bd505908b586577c2a89da6437fc5dc1f30227e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 10 Dec 2025 19:39:17 -0800 Subject: [PATCH 05/94] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c430477..8de492d5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [10.12.1] +### Added +- `model_context_length` parameter for `OPERA_DIST_S1` job specification. + +## [10.12.1] + ### Fixed - Doubled the memory requirement for `INSAR_ISCE_MULTI_BURST` jobs at 20x4 looks and >= 13 burst pairs. As a result, the credit cost for such jobs as also doubled from 5 to 10. Fixes https://github.com/ASFHyP3/hyp3/issues/2933 - ScaleCluster now takes no action when desired vCPUs only marginally exceeds target vCPUs. Fixes https://github.com/ASFHyP3/hyp3/issues/2965 From 3e310349e1149036461496a41f197db9e83b1d5d Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 10 Dec 2025 19:39:29 -0800 Subject: [PATCH 06/94] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de492d5b..227bd9ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [10.12.1] +## [10.12.2] ### Added - `model_context_length` parameter for `OPERA_DIST_S1` job specification. From 674d9c1db641c3a67a93204264cf3ebcd269b9fb Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 11 Dec 2025 09:19:39 -0800 Subject: [PATCH 07/94] Update OPERA_DIST_S1.yml --- job_spec/OPERA_DIST_S1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/OPERA_DIST_S1.yml b/job_spec/OPERA_DIST_S1.yml index 1daba4572..ec9bdcb69 100644 --- a/job_spec/OPERA_DIST_S1.yml +++ b/job_spec/OPERA_DIST_S1.yml @@ -119,7 +119,7 @@ OPERA_DIST_S1: type: integer default: 7 minimum: 1 - maximum: 16 + maximum: 32 example: 7 batch_size_for_norm_param_estimation: api_schema: From 2a01337f6b728a1613b03b4f570c726eea1c020c Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 11 Dec 2025 09:22:09 -0800 Subject: [PATCH 08/94] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 227bd9ebf..f434bcd64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `model_context_length` parameter for `OPERA_DIST_S1` job specification. +- Update max stride for models with larger input size. ## [10.12.1] From 0c0ef2ca0ef68e521095cc96a534f28a13bb2532 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 11 Dec 2025 11:40:41 -0900 Subject: [PATCH 09/94] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f434bcd64..ace60f0c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [10.12.2] ### Added -- `model_context_length` parameter for `OPERA_DIST_S1` job specification. -- Update max stride for models with larger input size. +- Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. + +### Changed +- Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. ## [10.12.1] From 420693ccb1662a6e2b261ffee20984916d81404e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 11 Dec 2025 11:44:35 -0900 Subject: [PATCH 10/94] Fix OPERA_DIST_S1 --- job_spec/OPERA_DIST_S1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/OPERA_DIST_S1.yml b/job_spec/OPERA_DIST_S1.yml index ec9bdcb69..04c8148e1 100644 --- a/job_spec/OPERA_DIST_S1.yml +++ b/job_spec/OPERA_DIST_S1.yml @@ -143,7 +143,7 @@ OPERA_DIST_S1: model_context_length: api_schema: description: Maximum number of baseline images to use. If -1, use largest allowable baseline. - type: int + type: integer default: -1 example: 10 cost_profiles: From 0c2ff1f11e809bd93872617db311e4be70a310ad Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 16 Dec 2025 09:49:44 -0900 Subject: [PATCH 11/94] distribution_url --- .github/workflows/deploy-edc-sandbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index 2ef2e493b..e0e456b65 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -39,7 +39,7 @@ jobs: required_surplus: 0 security_environment: EDC ami_id: /ngap/amis/image_id_ecs_al2023_x86 - distribution_url: '' # TODO + distribution_url: 'https://d3bvvghf83wjqc.cloudfront.net' environment: name: ${{ matrix.environment }} From 0599eef9b527a51cd32d6deefdea4852ecdf5cea Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 16 Dec 2025 10:37:35 -0900 Subject: [PATCH 12/94] delete jth-sandbox --- .github/workflows/deploy-jth-sandbox.yml | 89 ------------------------ 1 file changed, 89 deletions(-) delete mode 100644 .github/workflows/deploy-jth-sandbox.yml diff --git a/.github/workflows/deploy-jth-sandbox.yml b/.github/workflows/deploy-jth-sandbox.yml deleted file mode 100644 index ed3056476..000000000 --- a/.github/workflows/deploy-jth-sandbox.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Deploy jth sandbox to AWS - -on: - push: - branches: - - hyp3-jth-sandbox - -concurrency: ${{ github.workflow }}-${{ github.ref }} - -jobs: - deploy: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - environment: hyp3-jth-sandbox - template_bucket: cf-templates-bywc0durdnqy-us-west-2 - image_tag: test - product_lifetime_in_days: 14 - default_credits_per_user: 0 - default_application_status: APPROVED - cost_profile: DEFAULT - opera_rtc_s1_end_date: Default - job_files: >- - job_spec/AUTORIFT.yml - job_spec/INSAR_GAMMA.yml - job_spec/RTC_GAMMA.yml - job_spec/INSAR_ISCE_BURST.yml - job_spec/INSAR_ISCE_MULTI_BURST.yml - job_spec/ARIA_S1_GUNW.yml - job_spec/OPERA_RTC_S1.yml - instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge - default_max_vcpus: 640 - expanded_max_vcpus: 640 - required_surplus: 0 - security_environment: EDC - ami_id: /ngap/amis/image_id_ecs_al2023_x86 - distribution_url: '' - - environment: - name: ${{ matrix.environment }} - - steps: - - uses: actions/checkout@v4.2.2 - with: - fetch-depth: 0 - - - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} - aws-region: ${{ secrets.AWS_REGION }} - - - uses: actions/setup-python@v5 - with: - python-version: 3.13 - - - run: | - jq -n '{"Parameters": $ARGS.named}' \ - --arg VpcId '${{ secrets.VPC_ID }}' \ - --arg SubnetIds '${{ secrets.SUBNET_IDS }}' \ - --arg SecretArn '${{ secrets.SECRET_ARN }}' \ - --arg ImageTag '${{ matrix.IMAGE_TAG }}' \ - --arg ProductLifetimeInDays '${{ matrix.product_lifetime_in_days }} ' \ - --arg AuthPublicKey '${{ secrets.AUTH_PUBLIC_KEY }}' \ - --arg DefaultCreditsPerUser '${{ matrix.default_credits_per_user }}' \ - --arg DefaultApplicationStatus '${{ matrix.default_application_status }}' \ - --arg OperaRtcS1EndDate '${{ matrix.opera_rtc_s1_end_date }}' \ - --arg AmiId '${{ matrix.ami_id }}' \ - --arg DefaultMaxvCpus '${{ matrix.default_max_vcpus }}' \ - --arg ExpandedMaxvCpus '${{ matrix.expanded_max_vcpus }}' \ - --arg MonthlyBudget '${{ secrets.MONTHLY_BUDGET }}' \ - --arg RequiredSurplus '${{ matrix.required_surplus }}' \ - --arg InstanceTypes '${{ matrix.instance_types }}' \ - --arg BucketReadPrincipals '${{ secrets.BUCKET_READ_PRINCIPALS }}' \ - --arg DistributionUrl '${{ matrix.distribution_url }}' \ - > parameters.json - - uses: ./.github/actions/deploy-hyp3 - with: - TEMPLATE_BUCKET: ${{ matrix.template_bucket }} - STACK_NAME: ${{ matrix.environment }} - API_NAME: ${{ matrix.environment }} - CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} - COST_PROFILE: ${{ matrix.cost_profile }} - JOB_FILES: ${{ matrix.job_files }} - SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} - PARAMETER_FILE: parameters.json From 4f6e5d703dd238088a1fd745099568e15f81a343 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 16 Dec 2025 10:52:54 -0900 Subject: [PATCH 13/94] edc-sandbox cost profile --- .github/workflows/deploy-edc-sandbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index e0e456b65..a2f8fec75 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -23,7 +23,7 @@ jobs: product_lifetime_in_days: 14 default_credits_per_user: 0 default_application_status: APPROVED - cost_profile: DEFAULT + cost_profile: EDC opera_rtc_s1_end_date: Default job_files: >- job_spec/AUTORIFT.yml From 131669dd49b4011a9b9bc9086212b0191e857e9e Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 18 Dec 2025 15:54:04 -0900 Subject: [PATCH 14/94] start updating deployment docs --- README.md | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 81f4bca06..64c62e106 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,10 @@ also referred to as "security environments" throughout our code and docs - JPL - JPL-public +For EDC, you will also need to refer to our +[Deploy HyP3 to Earthdata Cloud](https://github.com/ASFHyP3/.github-private/blob/main/docs/Deploy-HyP3-to-Earthdata-Cloud.md) +internal docs article (only accessible to members of ASF). + > [!IMPORTANT] > JPL deployments _must_ start with the JPL security environment, but can be migrated to `JPL-public` > after they are fully deployed and approved to have a public bucket. @@ -67,19 +71,6 @@ For JPL, these deployment docs assume that: - the JPL account was set up in the "default" manner by the JPL cloud team - the developer deploying the account is able to log in with the `power_user` role -For a new EDC deployment, you need the following items (not necessarily a comprehensive list): -- SSL certificate in AWS Certificate Manager for custom CloudFront domain name -- ID of the CloudFront Origin Access Identity used to access data in S3 - -EDC UAT/prod deployment steps are not fully documented here. -When deploying HyP3 to a new EDC account for the first time, you should also refer to the -[SOP for deploying HyP3 to EDC](https://asfdaac.atlassian.net/wiki/spaces/ST/pages/2290319361/SOP-ASF-DAAC-EDC-011). -You should then be able to deploy additional copies of HyP3 to an EDC Sandbox account -by following this README alone. - -After deploying HyP3 to an EDC Sandbox account, you'll need to follow our documentation on -[Accessing Private API Gateways in Earthdata Cloud](https://github.com/ASFHyP3/.github-private/blob/main/docs/Accessing-Private-API-Gateways-in-Earthdata-Cloud.md). - > [!TIP] > You can expand and collapse details specific to a security environment as you go through this README. > Make sure you're looking at the details for the security environment you're deploying into! @@ -242,6 +233,8 @@ Go to AWS console -> Secrets Manager, then: #### Upload SSL cert +TODO: update this section + > [!WARNING] > This step must be done by an ASF employee. @@ -250,8 +243,6 @@ To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its If HyP3 is being deployed to an ASF-managed AWS account, we can use the master certificate that covers all `*.asf.alaska.edu` domains. Otherwise, we'll need a deployment specific certificate. -*Important: Skip this step for EDC Sandbox deployments.* -
ASF-managed AWS account: Upload the ASF master SSL certificate
@@ -301,8 +292,9 @@ and then upload them to AWS Certificate Manager (ACM): (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) 4. Add the following environment secrets: - `AWS_REGION` - e.g. `us-west-2` - - `BUCKET_READ_PRINCIPALS` (EDC only) - List of AWS IAM principals granted read access to data in S3 for Earthdata Cloud deployments. For EDC Sandbox deployments, if you don't know what to put here, you can simply set it to `arn:aws:iam:::root`, where `` is the AWS account ID for the EDC Sandbox account. + - `BUCKET_READ_PRINCIPALS` (EDC only) - List of AWS IAM principals granted read access to data in S3 for Earthdata Cloud deployments. - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate that you imported manually (aws console -> certificate manager -> list certificates, e.g. `arn:aws:acm:us-west-2:xxxxxxxxxxxx:certificate/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`) + - TODO: update wording - `CLOUDFORMATION_ROLE_ARN` (ASF only) - part of the `hyp3-ci` stack that you deployed, e.g. `arn:aws:iam::xxxxxxxxxxxx:role/hyp3-ci-CloudformationDeploymentRole-XXXXXXXXXXXXX` - `SECRET_ARN` - ARN for the AWS Secrets Manager Secret that you created manually - `V2_AWS_ACCESS_KEY_ID` - AWS access key ID: @@ -317,7 +309,7 @@ and then upload them to AWS Certificate Manager (ACM): You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow or create a new one for the deployment. If you need to create a new one, we recommend copying one of the -`deploy-*-sandbox.yml` workflows, and then updating all of the fields (`environment`, `domain`, `template_bucket`, etc.) +existing workflows, and then updating all of the fields as appropriate for your deployment. Also make sure to update the top-level `name` of the workflow and the name of the branch to deploy from. (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) @@ -336,12 +328,10 @@ Once HyP3 is deployed, there are a few follow on tasks you may need to do for a > [!WARNING] > This step must be done by an ASF employee. -*Important: Skip this step for EDC Sandbox deployments.* - Open a PR adding a line to https://gitlab.asf.alaska.edu/operations/puppet/-/blob/production/modules/legacy_dns/files/asf.alaska.edu.db for the new custom domain name (AWS console -> api gateway -> custom domain names -> "API Gateway domain name"). -Ask the Platform team in the `~development-support` channel in Mattermost to review/merge the PR. +Ask someone from ASF support to review/merge the PR. Changes should take effect within 15-60 minutes after merging. Confirm that a Swagger UI is available at your chosen API URL. From 94f3f8f65d54e4ace72f6b2e3eb53fecdb17ad50 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 18 Dec 2025 16:11:36 -0900 Subject: [PATCH 15/94] update ssl cert docs for edc and jpl --- README.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 64c62e106..deaebeb84 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Go to AWS console -> Secrets Manager, then: #### Upload SSL cert -TODO: update this section +TODO: finish updating this section; do we still reuse a master cert for all ASF-managed deployments? > [!WARNING] > This step must be done by an ASF employee. @@ -260,21 +260,15 @@ Upload the `*.asf.alaska.edu` SSL certificate to AWS Certificate Manager (ACM): Externally-managed AWS account (e.g., JPL, EDC, CloudBank): Request and upload deployment specific SSL certificate
-Submit a Platform request in ASF JIRA for a new certificate, including the domain name -(e.g. `hyp3-foobar.asf.alaska.edu`). +*Note: For EDC accounts, you should be in the `us-east-1` region.* -Once you receive the certificate's private key and links to download the certificate in various formats, -download these files: -1. Certificate Body (the "as Certificate Only, PEM encoded" link from the email) -2. Certificate Private Key (from the Platform team; typically in MatterMost) -3. Certificate Chain (the "as Root/Intermediate(s) only, PEM encoded" link from the email) +AWS console -> AWS Certificate Manager -> Request a public certificate: +- The domain name should be something like `hyp3-foobar.asf.alaska.edu` +- Choose "DNS validation" -and then upload them to AWS Certificate Manager (ACM): - -1. AWS console -> Certificate Manager (ACM) -> import certificate - 1. The contents of (1) above goes in Certificate body - 1. The contents of (2) above goes in Certificate private key - 1. The contents of (3) above goes in Certificate chain +Then add the validation record to +https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy_dns/files/asf.alaska.edu.db +(see previous records for examples).
From d715f9f7388c11c624503c4babfc612634467af8 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 18 Dec 2025 16:15:46 -0900 Subject: [PATCH 16/94] clarify region --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index deaebeb84..22ff372b1 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,9 @@ Upload the `*.asf.alaska.edu` SSL certificate to AWS Certificate Manager (ACM): Externally-managed AWS account (e.g., JPL, EDC, CloudBank): Request and upload deployment specific SSL certificate
-*Note: For EDC accounts, you should be in the `us-east-1` region.* +*Note: For EDC accounts, you should create the cert in the `us-east-1` region +for use with the CloudFront distribution that you will create later, +even if you're deploying HyP3 to `us-west-2`.* AWS console -> AWS Certificate Manager -> Request a public certificate: - The domain name should be something like `hyp3-foobar.asf.alaska.edu` From e5ad0ad2f26161bf7ae7b884b870dbc5be73342d Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 18 Dec 2025 16:30:17 -0900 Subject: [PATCH 17/94] update gh secrets in deploy docs --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 22ff372b1..f43ea3421 100644 --- a/README.md +++ b/README.md @@ -288,9 +288,7 @@ https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) 4. Add the following environment secrets: - `AWS_REGION` - e.g. `us-west-2` - - `BUCKET_READ_PRINCIPALS` (EDC only) - List of AWS IAM principals granted read access to data in S3 for Earthdata Cloud deployments. - - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate that you imported manually (aws console -> certificate manager -> list certificates, e.g. `arn:aws:acm:us-west-2:xxxxxxxxxxxx:certificate/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`) - - TODO: update wording + - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate - `CLOUDFORMATION_ROLE_ARN` (ASF only) - part of the `hyp3-ci` stack that you deployed, e.g. `arn:aws:iam::xxxxxxxxxxxx:role/hyp3-ci-CloudformationDeploymentRole-XXXXXXXXXXXXX` - `SECRET_ARN` - ARN for the AWS Secrets Manager Secret that you created manually - `V2_AWS_ACCESS_KEY_ID` - AWS access key ID: From 6d26374b5dea96516acdb27900fbc734609d8d71 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 18 Dec 2025 16:31:12 -0900 Subject: [PATCH 18/94] wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f43ea3421..8e6e45f71 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) 4. Add the following environment secrets: - `AWS_REGION` - e.g. `us-west-2` - - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate + - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate that you created manually - `CLOUDFORMATION_ROLE_ARN` (ASF only) - part of the `hyp3-ci` stack that you deployed, e.g. `arn:aws:iam::xxxxxxxxxxxx:role/hyp3-ci-CloudformationDeploymentRole-XXXXXXXXXXXXX` - `SECRET_ARN` - ARN for the AWS Secrets Manager Secret that you created manually - `V2_AWS_ACCESS_KEY_ID` - AWS access key ID: From a801e853fef343a1570b2a0e9d54dc9dbe538567 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 19 Dec 2025 09:23:42 -0900 Subject: [PATCH 19/94] finish updating ssl cert docs --- README.md | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/README.md b/README.md index 8e6e45f71..5cad8dcea 100644 --- a/README.md +++ b/README.md @@ -233,33 +233,8 @@ Go to AWS console -> Secrets Manager, then: #### Upload SSL cert -TODO: finish updating this section; do we still reuse a master cert for all ASF-managed deployments? - -> [!WARNING] -> This step must be done by an ASF employee. - To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL). -If HyP3 is being deployed to an ASF-managed AWS account, we can use the master certificate that covers all -`*.asf.alaska.edu` domains. Otherwise, we'll need a deployment specific certificate. - -
-ASF-managed AWS account: Upload the ASF master SSL certificate -
- -Upload the `*.asf.alaska.edu` SSL certificate to AWS Certificate Manager (ACM): - -1. AWS console -> Certificate Manager (ACM) -> import certificate -1. Open https://gitlab.asf.alaska.edu/operations/puppet/-/tree/production/modules/certificates/files - 1. The contents of the `asf.alaska.edu.cer` file go in "Certificate body" - 1. The contents of the `asf.alaska.edu.key` file go in "Certificate private key" - 1. The contents of the `intermediates.pem` file go in "Certificate chain" -
- -
-Externally-managed AWS account (e.g., JPL, EDC, CloudBank): Request and upload deployment specific SSL certificate -
- *Note: For EDC accounts, you should create the cert in the `us-east-1` region for use with the CloudFront distribution that you will create later, even if you're deploying HyP3 to `us-west-2`.* @@ -272,8 +247,6 @@ Then add the validation record to https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy_dns/files/asf.alaska.edu.db (see previous records for examples). -
- ### Create the GitHub environment > [!WARNING] From ac5adaf1eb8eb13b22dfb724e765675c63805d04 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 19 Dec 2025 09:25:54 -0900 Subject: [PATCH 20/94] wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cad8dcea..a395b2f38 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ Go to AWS console -> Secrets Manager, then: 1. Click the orange "Next" button (we won't configure rotation) 1. Click the orange "Store" button to save the Secret -#### Upload SSL cert +#### Request SSL cert To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL). From 88f1fd641a86e26633dd673d0d0d46ea6fcfde46 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 19 Dec 2025 09:28:49 -0900 Subject: [PATCH 21/94] revise --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a395b2f38..1ab2619ea 100644 --- a/README.md +++ b/README.md @@ -233,12 +233,12 @@ Go to AWS console -> Secrets Manager, then: #### Request SSL cert -To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL). - *Note: For EDC accounts, you should create the cert in the `us-east-1` region for use with the CloudFront distribution that you will create later, even if you're deploying HyP3 to `us-west-2`.* +To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL): + AWS console -> AWS Certificate Manager -> Request a public certificate: - The domain name should be something like `hyp3-foobar.asf.alaska.edu` - Choose "DNS validation" From 2c6638f3651683e1a88ff4035605d6d0b1ecfb5b Mon Sep 17 00:00:00 2001 From: Kaleb Burris Date: Mon, 12 Jan 2026 10:49:52 -0900 Subject: [PATCH 22/94] fix: Run the checks workflow on a push to any PR to main or develop --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index baaa6a2af..bddcc4919 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,10 @@ name: Run tests -on: push +on: + pull_request: + branches: + - main + - develop env: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_HYP3: v0.0.0 From 6c2dde59f2f2a4d7f00a0c965cc584e9ab2a9cc2 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Mon, 12 Jan 2026 15:50:40 -0700 Subject: [PATCH 23/94] adding hyp3-slimsar-test to deploy yml to deploy test hyp3 for slimsar processing --- .github/workflows/deploy-custom-test.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index 856c11edc..ae2073115 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -127,6 +127,23 @@ jobs: required_surplus: 0 security_environment: ASF ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + + - environment: hyp3-slimsar-test + domain: hyp3-slimsar-test.asf.alaska.edu + template_bucket: cf-templates-1ce4dbsaugtx1-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: DEFAULT + job_files: >- + job_spec/INSAR_ISCE_BURST.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id environment: name: ${{ matrix.environment }} From f0a022c549ff6b803e859b95087da1c87039f122 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Mon, 12 Jan 2026 16:11:43 -0700 Subject: [PATCH 24/94] add changelog entry for new deployment --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ace60f0c7..523b65a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [10.12.2] +### Added +- Added the custom hyp3-slimsar-test deployment + ### Added - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. From d975076d86d6c62b87038015915b3bb9446631ec Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Mon, 12 Jan 2026 16:19:11 -0700 Subject: [PATCH 25/94] combine changelog added entries --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 523b65a23..b984027be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [10.12.2] -### Added -- Added the custom hyp3-slimsar-test deployment - ### Added - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. +- Added the custom hyp3-slimsar-test deployment ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. From 57df621f6577136f1676c770076a1ea31309d776 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 12:13:10 -0700 Subject: [PATCH 26/94] adding in ReadMe clarifications and edits from my first hyp3 deployment --- README.md | 71 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1ab2619ea..89093f099 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,9 @@ Once the `github-actions` IAM user has been created, you can create an AWS acces which we will use to deploy HyP3 via CI/CD tooling: 1. Go to AWS console -> IAM -> Users -> github-actions -> security credentials tab -> "create access key". -2. Store the access key ID and secret access key using your team's password manager. +2. Select "Other" for key usage +3. Add tag value to describe the key such as `hyp3-foobar-access` +4. Store the access key ID and secret access key using your team's password manager. You will use them below in "Create the GitHub environment"
@@ -223,13 +225,13 @@ you will need to create an Earthdata Login user for your deployment if you do no Go to AWS console -> Secrets Manager, then: 1. Click the orange "Store a new secret" button -1. For "Secret Type" select "Other type of secret" -1. Enter all required secret key-value pairs. Notably, the keys should be the secret names as listed (case-sensitive) in the [job specs](./job_spec/) that will be deployed -1. Click the orange "Next" button -1. Give the secret the same name that you plan to give to the HyP3 CloudFormation stack when you deploy it (below) -1. Click the orange "Next" button -1. Click the orange "Next" button (we won't configure rotation) -1. Click the orange "Store" button to save the Secret +2. For "Secret Type" select "Other type of secret" +3. Enter all required secret key-value pairs. Notably, the keys should be the secret names as listed (case-sensitive) in the [job specs](./job_spec/) that will be deployed +4. Click the orange "Next" button +5. Give the secret the same name that you plan to give to the HyP3 CloudFormation stack when you deploy it (below) +6. Click the orange "Next" button +7. Click the orange "Next" button (we won't configure rotation) +8. Click the orange "Store" button to save the Secret #### Request SSL cert @@ -239,13 +241,16 @@ even if you're deploying HyP3 to `us-west-2`.* To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL): -AWS console -> AWS Certificate Manager -> Request a public certificate: -- The domain name should be something like `hyp3-foobar.asf.alaska.edu` -- Choose "DNS validation" +AWS console -> AWS Certificate Manager -> Request certificate:\ +1. Select "Request a public certificate" +2. Click the orange "Next" button +3. Choose a "Fully qualified domain name". Domain name should be something like `hyp3-foobar.asf.alaska.edu` or for a test deployment `hyp3-foobar-test.asf.alaska.edu`. +3. Choose "DNS validation" +4. Copy the "CNAME name" and "CNAME value" -Then add the validation record to +Then copy past the add the validation record to a row of https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy_dns/files/asf.alaska.edu.db -(see previous records for examples). +in the format ` in CNAME .` (see previous records for examples). ### Create the GitHub environment @@ -254,29 +259,30 @@ https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy > repository, which is generally only possible for ASF employees on HyP3 development teams. 1. Go to https://github.com/ASFHyP3/hyp3/settings/environments -> New Environment -2. Check "required reviewers" and add the appropriate team(s) or user(s) -3. Change "Deployment branches and tags" to "Selected branches and tags" and +2. Name the environment like your chosen domain name i.e. `hyp3-foobar` or `hyp3-foobar-test` +3. Check "required reviewers" and add the appropriate team(s) or user(s) +4. Change "Deployment branches and tags" to "Selected branches and tags" and - add a deployment branch or tag rule - use "Ref Type: Branch" and write the name of the branch it will be deploying out of. (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) -4. Add the following environment secrets: +5. Add the following environment secrets: - `AWS_REGION` - e.g. `us-west-2` - - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate that you created manually + - `CERTIFICATE_ARN` (ASF and JPL only) - ARN of the AWS Certificate Manager certificate that you created manually, e.g. `arn:aws:acm:us-west-2:XXXXXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX` - `CLOUDFORMATION_ROLE_ARN` (ASF only) - part of the `hyp3-ci` stack that you deployed, e.g. `arn:aws:iam::xxxxxxxxxxxx:role/hyp3-ci-CloudformationDeploymentRole-XXXXXXXXXXXXX` - - `SECRET_ARN` - ARN for the AWS Secrets Manager Secret that you created manually + - `SECRET_ARN` - ARN for the AWS Secrets Manager Secret that you created manually, e.g. `arn:aws:secretsmanager:us-west-X:XXXXXXXXXXXX:secret:hyp3-foobar-XXXXXX` - `V2_AWS_ACCESS_KEY_ID` - AWS access key ID: - - ASF: for the `github-actions` user + - ASF: for the `github-actions` user (created in step "Enable CI/CD above") - JPL: for the service user - EDC: created by an ASF developer via Kion - `V2_AWS_SECRET_ACCESS_KEY` - The corresponding secret access key - - `VPC_ID` - ID of the default VPC for this AWS account and region (aws console -> vpc -> your VPCs, e.g. `vpc-xxxxxxxxxxxxxxxxx`) - - `SUBNET_IDS` - Comma delimited list (no spaces) of the default subnets for the VPC specified in `VPC_ID` (aws console -> vpc -> subnets, e.g. `subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx`) + - `VPC_ID` - ID of the default VPC for this AWS account and region (aws console -> VPC -> Your VPCs, e.g. `vpc-xxxxxxxxxxxxxxxxx`) + - `SUBNET_IDS` - Comma delimited list (no spaces) of the default subnets for the VPC specified in `VPC_ID` (aws console -> VPC -> Subnets, e.g. `subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx,subnet-xxxxxxxxxxxxxxxxx`) ### Create the HyP3 deployment -You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow or create +You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow (.github/workflows/) or create a new one for the deployment. If you need to create a new one, we recommend copying one of the -existing workflows, and then updating all of the fields +existing workflows, and then updating all of the fields. as appropriate for your deployment. Also make sure to update the top-level `name` of the workflow and the name of the branch to deploy from. (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) @@ -284,6 +290,9 @@ of the branch to deploy from. (This is typically `main` for prod deployments, `d > If you're deploying from a feature branch, make sure to [protect](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches) > it from accidental deletion. +> [!TIP] +> If your CI/CD workflow fails. Delete the "Rolled Back" stack (AWS Manager -> CloudFormation -> Stacks) before re-running the failed job. + The deployment workflow will run as soon as you merge your changes into the branch specified in the workflow file. ### Finishing touches @@ -296,7 +305,8 @@ Once HyP3 is deployed, there are a few follow on tasks you may need to do for a > This step must be done by an ASF employee. Open a PR adding a line to https://gitlab.asf.alaska.edu/operations/puppet/-/blob/production/modules/legacy_dns/files/asf.alaska.edu.db -for the new custom domain name (AWS console -> api gateway -> custom domain names -> "API Gateway domain name"). +for the new custom domain name (AWS console -> api gateway -> custom domain names -> "API Gateway domain name") of the format +`hyp3-foobar in CNAME .`. Follow similar examples. Ask someone from ASF support to review/merge the PR. @@ -315,6 +325,19 @@ Update the [AWS Accounts and HyP3 Deployments](https://docs.google.com/spreadshe > ``` > Remember to remove this after the DNS PR is merged! +#### Testing and adding user credits to your hyp3 deployment + +After successful deployment and DNS record merging you can test you deployment by accessing the Swagger UI and using the POST /user tab to +check if your user is approved and has credits for running jobs on the deployment. This should automatically add your user to the DynamoDB +table. + +AWS console -> DynamoDB -> Explore items +1. Find the table with format like `hyp3-foobar-UsersTable-XXXXXXXXXXXXX` +2. Edit your user (only present after using the Swagger UI in some way) +3. Add credits if neccessary + +You can then return the Swagger UI and use the POST /jobs to run a test job and confirm it completes. + #### Optional
From 1612c5b30bec416f04a315ea17a50b4dacbb0d5d Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 12:29:00 -0700 Subject: [PATCH 27/94] add in attempt at motivation statement --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 89093f099..976387262 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ A processing environment for HyP3 Plugins in AWS. ## Deployment +### Why would you set up a hyp3 deployment? + +A HyP3 deployment stack provides a reproducible cloud processing environment that bundles AWS infrastructure, execution logic, and cost controls, enabling scalable, on-demand computation with clear operational costs. + > [!IMPORTANT] > It's not currently possible to deploy HyP3 fully independent of ASF due to our integration with > [ASF Vertex](https://search.alaska.edu). If you'd like your own deployment of HyP3, please open an issue here or @@ -126,7 +130,7 @@ In order to integrate an ASF deployment we'll need: These can be done by deploying the [ASF CI stack](cicd-stacks/ASF-deployment-ci-cf.yml). -*Warning: This stack should only be deployed once per AWS account. This stack also +*Warning: This stack only needs to be deployed once per AWS account. This stack also assumes you are only deploying into a single AWS Region. If you are deploying into multiple regions in the same AWS account, you'll need to adjust the IAM permissions that are limited to a single region.* @@ -149,6 +153,7 @@ which we will use to deploy HyP3 via CI/CD tooling: 2. Select "Other" for key usage 3. Add tag value to describe the key such as `hyp3-foobar-access` 4. Store the access key ID and secret access key using your team's password manager. You will use them below in "Create the GitHub environment" + as `V2_AWS_ACCESS_KEY_ID` and `V2_AWS_SECRET_ACCESS_KEY`.
From 9481980b8db883cbffea7a9804637744425fc1cd Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 12:30:27 -0700 Subject: [PATCH 28/94] changelog of readme edits --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b984027be..520cd999b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. +- Changed readme to reflect current state of AWS, motivation for deploying a hyp3 stack, and clarify naming and formatting. ## [10.12.1] From 8656537451fab790814b5dd6db7352d5de08e982 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 15:21:28 -0700 Subject: [PATCH 29/94] create new job_spec yaml for tdbp processing slimsar data. Adding to hyp3-slimsar-test deployment CI CD deployment matrix and changelog --- .github/workflows/deploy-custom-test.yml | 1 + CHANGELOG.md | 1 + job_spec/SLIMSAR_TDBP.yml | 67 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 job_spec/SLIMSAR_TDBP.yml diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index ae2073115..ff27037e9 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -138,6 +138,7 @@ jobs: cost_profile: DEFAULT job_files: >- job_spec/INSAR_ISCE_BURST.yml + job_spec/SLIMSAR_TDBP.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 expanded_max_vcpus: 640 diff --git a/CHANGELOG.md b/CHANGELOG.md index b984027be..c3174fa25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added the custom hyp3-slimsar-test deployment +- Added new job_spec for slimsar time-domain backprojection processing ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml new file mode 100644 index 000000000..d3b74cdb1 --- /dev/null +++ b/job_spec/SLIMSAR_TDBP.yml @@ -0,0 +1,67 @@ +SLIMSAR_TDBP: + required_parameters: + - data_fps + - nav_fps + - dem_fp + parameters: + data_fps: + api_schema: + type: array + minItems: 1 + maxItems: 6 + example: + - s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar + items: + description: Path to uncompressed slimsar echoes file on s3 bucket + type: string + pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.sar$" + example: s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar + nav_fp: + api_schema: + type: string + example: s3://asf-frostbyte/slimsar/NMZT23520001E_100_proj.dat + pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.dat$" + description: Path to post-processed nav .dat file on s3 bucket + dem_fp: + api_schema: + type: string + example: s3://asf-frostbyte/slimsar/repeat_sub.tif + description: Path to DEM file on S3 bucket + validators: [] + cost_profiles: + DEFAULT: + cost: 1.0 + steps: + - name: '' + image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest + command: + - docker + - run + - --rm + - --gpus + - all + - -v + - /mnt/ephemeral:/work + - -e + - AWS_DEFAULT_REGION=us-west-2 + - 123456789012.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest + - --data_fps + - Ref::data_fps + - --nav_fps + - Ref::nav_fps + - --dem_fp + - Ref::dem_fp + - --s3_bucket + - asf-frostbyte + - --s3_prefix + - Ref::job_id + timeout: 126000 # 35 hours + compute_environment: Default + vcpu: 1 + gpu: 1 + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD + - PUBLISH_ACCESS_KEY_ID + - PUBLISH_SECRET_ACCESS_KEY + - PUBLISH_BUCKET From 89b29c555b1105b2e4d1236cc0982be8d1272e5d Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 15:23:40 -0700 Subject: [PATCH 30/94] generalize bucket names and ecr address for now --- job_spec/SLIMSAR_TDBP.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index d3b74cdb1..5b126ccb6 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -10,7 +10,7 @@ SLIMSAR_TDBP: minItems: 1 maxItems: 6 example: - - s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar + - s3://asf-bucket/slimsar/SAR_11072025_110035.sar items: description: Path to uncompressed slimsar echoes file on s3 bucket type: string @@ -19,13 +19,13 @@ SLIMSAR_TDBP: nav_fp: api_schema: type: string - example: s3://asf-frostbyte/slimsar/NMZT23520001E_100_proj.dat + example: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.dat$" description: Path to post-processed nav .dat file on s3 bucket dem_fp: api_schema: type: string - example: s3://asf-frostbyte/slimsar/repeat_sub.tif + example: s3://asf-bucket/slimsar/repeat_sub.tif description: Path to DEM file on S3 bucket validators: [] cost_profiles: @@ -33,7 +33,7 @@ SLIMSAR_TDBP: cost: 1.0 steps: - name: '' - image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest + image: 123456789.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest command: - docker - run @@ -44,7 +44,7 @@ SLIMSAR_TDBP: - /mnt/ephemeral:/work - -e - AWS_DEFAULT_REGION=us-west-2 - - 123456789012.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest + - 123456789.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest - --data_fps - Ref::data_fps - --nav_fps From 4c67ed33bc1b7c9790bcab29769712ad74e625a7 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 16:32:47 -0700 Subject: [PATCH 31/94] adding defaults to all api_schema parameters --- job_spec/SLIMSAR_TDBP.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 5b126ccb6..226427e9e 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -9,6 +9,7 @@ SLIMSAR_TDBP: type: array minItems: 1 maxItems: 6 + default: [s3://asf-bucket/slimsar/SAR_11072025_110035.sar] example: - s3://asf-bucket/slimsar/SAR_11072025_110035.sar items: @@ -21,11 +22,14 @@ SLIMSAR_TDBP: type: string example: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.dat$" + default: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat description: Path to post-processed nav .dat file on s3 bucket dem_fp: api_schema: type: string + default: s3://asf-bucket/slimsar/repeat_sub.tif example: s3://asf-bucket/slimsar/repeat_sub.tif + pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.tif$" description: Path to DEM file on S3 bucket validators: [] cost_profiles: @@ -48,7 +52,7 @@ SLIMSAR_TDBP: - --data_fps - Ref::data_fps - --nav_fps - - Ref::nav_fps + - Ref::nav_fp - --dem_fp - Ref::dem_fp - --s3_bucket @@ -59,6 +63,7 @@ SLIMSAR_TDBP: compute_environment: Default vcpu: 1 gpu: 1 + memory: 30500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From 844f55ba71ebb1d5c1d49fc609bb0cf0f26b30bc Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 16:46:18 -0700 Subject: [PATCH 32/94] add slimsar compute environment and fix job_spec for slimsar to it --- job_spec/SLIMSAR_TDBP.yml | 4 ++-- job_spec/config/compute_environments.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 226427e9e..2a29f36fb 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -8,7 +8,7 @@ SLIMSAR_TDBP: api_schema: type: array minItems: 1 - maxItems: 6 + maxItems: 10 default: [s3://asf-bucket/slimsar/SAR_11072025_110035.sar] example: - s3://asf-bucket/slimsar/SAR_11072025_110035.sar @@ -60,7 +60,7 @@ SLIMSAR_TDBP: - --s3_prefix - Ref::job_id timeout: 126000 # 35 hours - compute_environment: Default + compute_environment: SlimSAR vcpu: 1 gpu: 1 memory: 30500 diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index dfc3460bd..9de8f5e3d 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -27,3 +27,6 @@ compute_environments: ItsLiveMeta: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + SlimSAR: + instance_types: g4dn.xlarge + allocation_type: EC2 \ No newline at end of file From d1c5fcf922b87e107efb7a974b1f0feeb1ddb8b7 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 16:47:21 -0700 Subject: [PATCH 33/94] remove unused parameters from job_spec --- job_spec/SLIMSAR_TDBP.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 2a29f36fb..5a56e8eaf 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -55,10 +55,6 @@ SLIMSAR_TDBP: - Ref::nav_fp - --dem_fp - Ref::dem_fp - - --s3_bucket - - asf-frostbyte - - --s3_prefix - - Ref::job_id timeout: 126000 # 35 hours compute_environment: SlimSAR vcpu: 1 @@ -67,6 +63,6 @@ SLIMSAR_TDBP: secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD - - PUBLISH_ACCESS_KEY_ID - - PUBLISH_SECRET_ACCESS_KEY - - PUBLISH_BUCKET + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - S3_PUBLISH_BUCKET From e4d4b3e6bba7b52cff1120edef069ab9309ed39b Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 16:53:44 -0700 Subject: [PATCH 34/94] removing unneccesary defaults from api_specs --- job_spec/SLIMSAR_TDBP.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 5a56e8eaf..cf93ddd30 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -1,7 +1,7 @@ SLIMSAR_TDBP: required_parameters: - data_fps - - nav_fps + - nav_fp - dem_fp parameters: data_fps: @@ -9,7 +9,6 @@ SLIMSAR_TDBP: type: array minItems: 1 maxItems: 10 - default: [s3://asf-bucket/slimsar/SAR_11072025_110035.sar] example: - s3://asf-bucket/slimsar/SAR_11072025_110035.sar items: @@ -22,12 +21,10 @@ SLIMSAR_TDBP: type: string example: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.dat$" - default: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat description: Path to post-processed nav .dat file on s3 bucket dem_fp: api_schema: type: string - default: s3://asf-bucket/slimsar/repeat_sub.tif example: s3://asf-bucket/slimsar/repeat_sub.tif pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.tif$" description: Path to DEM file on S3 bucket From 09c78c77e81047af5841b9093c1002947b67a3dc Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 17:10:35 -0700 Subject: [PATCH 35/94] fix command to accept necessary arguments --- job_spec/SLIMSAR_TDBP.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index cf93ddd30..db3a33d21 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -34,24 +34,18 @@ SLIMSAR_TDBP: cost: 1.0 steps: - name: '' - image: 123456789.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest + image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test command: - - docker - - run - - --rm - - --gpus - - all - - -v - - /mnt/ephemeral:/work - - -e - - AWS_DEFAULT_REGION=us-west-2 - - 123456789.dkr.ecr.us-west-2.amazonaws.com/slimsar-test:latest - --data_fps - Ref::data_fps - - --nav_fps + - --nav_fp - Ref::nav_fp - --dem_fp - Ref::dem_fp + - --bucket + - '!Ref Bucket' + - --bucket_prefix + - Ref::job_id timeout: 126000 # 35 hours compute_environment: SlimSAR vcpu: 1 From 1c99a2061c2add24eea9f7188e1090d5a7445679 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 14 Jan 2026 17:17:28 -0700 Subject: [PATCH 36/94] add in polarization selection paramater and remove publish bucket for now --- job_spec/SLIMSAR_TDBP.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index db3a33d21..5748391d2 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -28,6 +28,21 @@ SLIMSAR_TDBP: example: s3://asf-bucket/slimsar/repeat_sub.tif pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.tif$" description: Path to DEM file on S3 bucket + pols: + api_schema: + type: array + minItems: 1 + maxItems: 4 + example: + - VV + - VH + - HV + - HH + default: [VV, VH, HV, HH] + items: + description: Polarization to process + maxLength: 2 + minLength: 2 validators: [] cost_profiles: DEFAULT: @@ -42,6 +57,8 @@ SLIMSAR_TDBP: - Ref::nav_fp - --dem_fp - Ref::dem_fp + - --pols + - Ref:pols - --bucket - '!Ref Bucket' - --bucket_prefix @@ -53,7 +70,4 @@ SLIMSAR_TDBP: memory: 30500 secrets: - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - S3_PUBLISH_BUCKET + - EARTHDATA_PASSWORD \ No newline at end of file From affbf55b36c65abce84fb8e973d4c6a7aa173a71 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Thu, 15 Jan 2026 09:21:20 -0700 Subject: [PATCH 37/94] add in large g4 gpus instances and volume and gpu commands to docker --- job_spec/SLIMSAR_TDBP.yml | 4 ++++ job_spec/config/compute_environments.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 5748391d2..1987f4177 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -51,6 +51,10 @@ SLIMSAR_TDBP: - name: '' image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test command: + - -v + - ${PWD}:/work + - --gpus + - all - --data_fps - Ref::data_fps - --nav_fp diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 9de8f5e3d..46024b14f 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -28,5 +28,5 @@ compute_environments: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id SlimSAR: - instance_types: g4dn.xlarge + instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, allocation_type: EC2 \ No newline at end of file From b9def7d5833faa5bb9f2722b3278799a230f8ccd Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Thu, 15 Jan 2026 15:12:12 -0700 Subject: [PATCH 38/94] remove unnessary docker commands from job spec --- job_spec/SLIMSAR_TDBP.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 1987f4177..5748391d2 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -51,10 +51,6 @@ SLIMSAR_TDBP: - name: '' image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test command: - - -v - - ${PWD}:/work - - --gpus - - all - --data_fps - Ref::data_fps - --nav_fp From 028d88318b147fa06eb71ef9da9e246c5830870d Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Tue, 20 Jan 2026 13:49:56 -0700 Subject: [PATCH 39/94] change data_fps to singular fp, add in pattern for pols, and change image to conda version --- job_spec/SLIMSAR_TDBP.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 5748391d2..e2910a352 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -4,18 +4,14 @@ SLIMSAR_TDBP: - nav_fp - dem_fp parameters: - data_fps: + data_fp: api_schema: - type: array - minItems: 1 - maxItems: 10 - example: - - s3://asf-bucket/slimsar/SAR_11072025_110035.sar - items: - description: Path to uncompressed slimsar echoes file on s3 bucket - type: string - pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.sar$" - example: s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar + type: string + example: s3://asf-bucket/slimsar/SAR_11072025_110035.sar + description: Path to uncompressed slimsar echo/raw file on s3 bucket + type: string + pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.sar$" + example: s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar nav_fp: api_schema: type: string @@ -43,16 +39,17 @@ SLIMSAR_TDBP: description: Polarization to process maxLength: 2 minLength: 2 + pattern: "^(VV|VH|HV|HH)$" validators: [] cost_profiles: DEFAULT: cost: 1.0 steps: - name: '' - image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/slimsar-test + image: 242399506617.dkr.ecr.us-west-2.amazonaws.com/hyp3-slimsar command: - - --data_fps - - Ref::data_fps + - --data_fp + - Ref::data_fp - --nav_fp - Ref::nav_fp - --dem_fp From 6b2500baeda72272503c41b8a12e461e64a95cba Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Tue, 20 Jan 2026 14:06:43 -0700 Subject: [PATCH 40/94] fixing required param list from data_fps to data_fp --- job_spec/SLIMSAR_TDBP.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index e2910a352..8949b9146 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -1,6 +1,6 @@ SLIMSAR_TDBP: required_parameters: - - data_fps + - data_fp - nav_fp - dem_fp parameters: From 954004fee8a4d27c1ca8d0112041c04d8e80b1bf Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 21 Jan 2026 18:31:01 -0900 Subject: [PATCH 41/94] whitespace --- job_spec/SLIMSAR_TDBP.yml | 2 +- job_spec/config/compute_environments.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 8949b9146..9bc2c1302 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -67,4 +67,4 @@ SLIMSAR_TDBP: memory: 30500 secrets: - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD \ No newline at end of file + - EARTHDATA_PASSWORD diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 46024b14f..d8c1b62d5 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -29,4 +29,4 @@ compute_environments: ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, - allocation_type: EC2 \ No newline at end of file + allocation_type: EC2 From ac22307c7fe7b495d87f1c8f42e084e3922008b8 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 21 Jan 2026 20:14:56 -0900 Subject: [PATCH 42/94] Keep push event trigger for tests workflow --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bddcc4919..70a110fb5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,7 @@ name: Run tests on: + push: pull_request: branches: - main From 33fde8da094b2d364a59bfa4053d2899b87002a5 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 21 Jan 2026 20:20:40 -0900 Subject: [PATCH 43/94] Update static analysis workflow to include pull requests --- .github/workflows/static-analysis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 306b7abc3..850235120 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -3,7 +3,13 @@ name: Static code analysis permissions: contents: read -on: push +on: + push: + pull_request: + branches: + - main + - develop + env: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_HYP3: v0.0.0 From bc67b2dfaed2d845227c5e63634224b1255a5b16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Jan 2026 05:30:18 +0000 Subject: [PATCH 44/94] Bump the github-actions-deps group with 3 updates Bumps the github-actions-deps group with 3 updates: [actions/checkout](https://github.com/actions/checkout), [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) and [actions/setup-python](https://github.com/actions/setup-python). Updates `actions/checkout` from 4.2.2 to 6.0.0 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.2...v6.0.0) Updates `aws-actions/configure-aws-credentials` from 4 to 5 - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/v4...v5) Updates `actions/setup-python` from 5 to 6 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-deps - dependency-name: aws-actions/configure-aws-credentials dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-deps - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-deps ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy-custom-prod.yml | 2 +- .github/workflows/deploy-custom-test.yml | 2 +- .github/workflows/deploy-daac-prod.yml | 2 +- .github/workflows/deploy-daac-test.yml | 2 +- .github/workflows/deploy-edc-sandbox.yml | 6 +++--- .github/workflows/deploy-plus-prod.yml | 2 +- .github/workflows/deploy-plus-test.yml | 2 +- .github/workflows/static-analysis.yml | 8 ++++---- .github/workflows/tests.yml | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy-custom-prod.yml b/.github/workflows/deploy-custom-prod.yml index c4dbe59ec..821e7d3e5 100644 --- a/.github/workflows/deploy-custom-prod.yml +++ b/.github/workflows/deploy-custom-prod.yml @@ -310,7 +310,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index ae2073115..987fa4cec 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -150,7 +150,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-prod.yml b/.github/workflows/deploy-daac-prod.yml index c1dda339a..cec1fff20 100644 --- a/.github/workflows/deploy-daac-prod.yml +++ b/.github/workflows/deploy-daac-prod.yml @@ -42,7 +42,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index edd61597c..0d0a882f3 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -44,7 +44,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index a2f8fec75..4da050764 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -45,18 +45,18 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v4.2.2 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@v5 with: aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} aws-region: ${{ secrets.AWS_REGION }} - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: 3.13 diff --git a/.github/workflows/deploy-plus-prod.yml b/.github/workflows/deploy-plus-prod.yml index c3a21ae5e..682d487a5 100644 --- a/.github/workflows/deploy-plus-prod.yml +++ b/.github/workflows/deploy-plus-prod.yml @@ -40,7 +40,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-plus-test.yml b/.github/workflows/deploy-plus-test.yml index ada84df0c..261930ee8 100644 --- a/.github/workflows/deploy-plus-test.yml +++ b/.github/workflows/deploy-plus-test.yml @@ -41,7 +41,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 850235120..32282014f 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -29,7 +29,7 @@ jobs: matrix: security_environment: [ASF, EDC, JPL, JPL-public] steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -43,7 +43,7 @@ jobs: openapi-spec-validator: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -56,7 +56,7 @@ jobs: statelint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 @@ -76,7 +76,7 @@ jobs: snyk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: snyk/actions/setup@v1.0.0 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 70a110fb5..b6894427d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: actions/setup-python@v6 with: From 15d43d91d3179c426787ddbdeb2c6c071e4daef3 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:58:59 -0700 Subject: [PATCH 45/94] Update job_spec/SLIMSAR_TDBP.yml fixing reget pattern for s3 raw file Co-authored-by: Joseph H Kennedy --- job_spec/SLIMSAR_TDBP.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 9bc2c1302..fbef94727 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -10,7 +10,7 @@ SLIMSAR_TDBP: example: s3://asf-bucket/slimsar/SAR_11072025_110035.sar description: Path to uncompressed slimsar echo/raw file on s3 bucket type: string - pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.sar$" + pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.sar$" example: s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar nav_fp: api_schema: From 1e92cfd06bb336c6c4691ff6aab621a687cd9d6a Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:59:37 -0700 Subject: [PATCH 46/94] Update job_spec/SLIMSAR_TDBP.yml fixing regex on .dat file Co-authored-by: Joseph H Kennedy --- job_spec/SLIMSAR_TDBP.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index fbef94727..4fe88216d 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -16,7 +16,7 @@ SLIMSAR_TDBP: api_schema: type: string example: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat - pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.dat$" + pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.dat$" description: Path to post-processed nav .dat file on s3 bucket dem_fp: api_schema: From 0471a5e073c1ec53139748761d9753936d1ccbe9 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:00:07 -0700 Subject: [PATCH 47/94] Update job_spec/SLIMSAR_TDBP.yml change to more explicit enum on polarizations Co-authored-by: Joseph H Kennedy --- job_spec/SLIMSAR_TDBP.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 4fe88216d..1e57cf41f 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -39,7 +39,12 @@ SLIMSAR_TDBP: description: Polarization to process maxLength: 2 minLength: 2 - pattern: "^(VV|VH|HV|HH)$" + type: string + enum: + - VV + - VH + - HV + - HH validators: [] cost_profiles: DEFAULT: From 13853788e51ca654fed9628c13a7ecebc4714829 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:00:38 -0700 Subject: [PATCH 48/94] Update job_spec/SLIMSAR_TDBP.yml fix regex on dem geotiff argument Co-authored-by: Joseph H Kennedy --- job_spec/SLIMSAR_TDBP.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index 1e57cf41f..f1ea70ba6 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -22,7 +22,7 @@ SLIMSAR_TDBP: api_schema: type: string example: s3://asf-bucket/slimsar/repeat_sub.tif - pattern: "^s3://[a-z0-9.-]+(/[^/]+)*/[^/]+\\.tif$" + pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.tif$" description: Path to DEM file on S3 bucket pols: api_schema: From 65894a962269e08f45b17b4cddc70a0d010c55ca Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Thu, 22 Jan 2026 11:05:04 -0700 Subject: [PATCH 49/94] add in ami_id for compute id from current ec2 --- job_spec/config/compute_environments.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index d8c1b62d5..b3422460b 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -30,3 +30,4 @@ compute_environments: SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, allocation_type: EC2 + ami_id: ami-03aa99ddf5498ceb91 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id From 5b870dda4947945ded5a3619c76c1437d387a85d Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Thu, 22 Jan 2026 11:49:06 -0700 Subject: [PATCH 50/94] fixing double quote to single quote to match new regex style --- job_spec/SLIMSAR_TDBP.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index f1ea70ba6..e1b5f8b3f 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -10,19 +10,19 @@ SLIMSAR_TDBP: example: s3://asf-bucket/slimsar/SAR_11072025_110035.sar description: Path to uncompressed slimsar echo/raw file on s3 bucket type: string - pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.sar$" + pattern: '^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.sar$' example: s3://asf-frostbyte/slimsar/SAR_11072025_110035.sar nav_fp: api_schema: type: string example: s3://asf-bucket/slimsar/NMZT23520001E_100_proj.dat - pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.dat$" + pattern: '^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.dat$' description: Path to post-processed nav .dat file on s3 bucket dem_fp: api_schema: type: string example: s3://asf-bucket/slimsar/repeat_sub.tif - pattern: "^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.tif$" + pattern: '^s3:\/\/[a-z0-9.-]+(\/[^/]+)*\/[^/]+\.tif$' description: Path to DEM file on S3 bucket pols: api_schema: From 34a0c7ac691b19fa741efb4e67cdbc0ef6ac8dbf Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Thu, 22 Jan 2026 12:28:06 -0700 Subject: [PATCH 51/94] extra 1 at end of ami_id for some reason --- job_spec/config/compute_environments.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index b3422460b..7f384d935 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -30,4 +30,4 @@ compute_environments: SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, allocation_type: EC2 - ami_id: ami-03aa99ddf5498ceb91 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id + ami_id: ami-03aa99ddf5498ceb9 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id From 6072b78a799973ae908b470475dbba35d8848d2a Mon Sep 17 00:00:00 2001 From: cmspeed Date: Thu, 22 Jan 2026 16:16:42 -0800 Subject: [PATCH 52/94] Update ARIA_AUTORIFT.yml to support custom chip-size and search-range --- job_spec/ARIA_AUTORIFT.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/job_spec/ARIA_AUTORIFT.yml b/job_spec/ARIA_AUTORIFT.yml index eef6a2f60..9c09b669e 100644 --- a/job_spec/ARIA_AUTORIFT.yml +++ b/job_spec/ARIA_AUTORIFT.yml @@ -106,6 +106,20 @@ AUTORIFT: description: Shapefile for determining the correct search parameters by geographic location. Path to shapefile must be understood by GDAL. type: string default: '/vsicurl/https://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_solidearth_0120m.shp' + chip_size: + api_schema: + description: Custom chip size in pixels for AutoRIFT (overrides parameter_file if provided) + type: integer + nullable: true + default: null + example: 32 + search_range: + api_schema: + description: Custom search range in pixels for AutoRIFT (overrides parameter_file if provided) + type: integer + nullable: true + default: null + example: 64 cost_profiles: DEFAULT: cost: 1.0 @@ -129,10 +143,14 @@ AUTORIFT: - Ref::reference - --secondary - Ref::secondary + - --chip-size + - Ref::chip_size + - --search-range + - Ref::search_range timeout: 10800 compute_environment: AriaAutorift vcpu: 1 memory: 8 # Memory is always overridden in the step function by set-batch-overrides secrets: - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD + - EARTHDATA_PASSWORD \ No newline at end of file From 17909c0854d2056218268719249fc17151e7a6be Mon Sep 17 00:00:00 2001 From: cmspeed Date: Thu, 22 Jan 2026 16:37:41 -0800 Subject: [PATCH 53/94] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b984027be..daca501b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.12.3] + +### Added +- Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. + ## [10.12.2] ### Added From 756792806d30a6c60f32d09c317e0c156fb1e2fa Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 22 Jan 2026 16:23:51 -0900 Subject: [PATCH 54/94] Update CHANGELOG with new job_spec and changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3174fa25..660d32d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added the custom hyp3-slimsar-test deployment -- Added new job_spec for slimsar time-domain backprojection processing +- Added the new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. From 3f0b2a9bcd3a76cdfc6128b46d278cdff66456b7 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 22 Jan 2026 16:31:48 -0900 Subject: [PATCH 55/94] fix missing newline at end of file --- job_spec/ARIA_AUTORIFT.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/ARIA_AUTORIFT.yml b/job_spec/ARIA_AUTORIFT.yml index 9c09b669e..882ad88eb 100644 --- a/job_spec/ARIA_AUTORIFT.yml +++ b/job_spec/ARIA_AUTORIFT.yml @@ -153,4 +153,4 @@ AUTORIFT: memory: 8 # Memory is always overridden in the step function by set-batch-overrides secrets: - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD \ No newline at end of file + - EARTHDATA_PASSWORD From 2fe2883d25b9c2e740f08f3e2bb2e116073090ca Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 22 Jan 2026 16:52:05 -0900 Subject: [PATCH 56/94] Add condition to run Snyk action only for non-forked PRs fixes #1476 --- .github/workflows/static-analysis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 32282014f..2823d5316 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -75,6 +75,7 @@ jobs: snyk: runs-on: ubuntu-latest + if: ! github.event.pull_request.head.repo.fork steps: - uses: actions/checkout@v6.0.1 - uses: snyk/actions/setup@v1.0.0 From 904b8dc2075d4e051333451517e6e6e905e8d0ba Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 22 Jan 2026 16:57:40 -0900 Subject: [PATCH 57/94] Apply suggestion from @jhkennedy --- .github/workflows/static-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 2823d5316..7e3173d5d 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -75,7 +75,7 @@ jobs: snyk: runs-on: ubuntu-latest - if: ! github.event.pull_request.head.repo.fork + if: ${{ ! github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@v6.0.1 - uses: snyk/actions/setup@v1.0.0 From f8f368f4a1ffbc3c2d3c05626105d0ca60de018a Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 22 Jan 2026 17:00:43 -0900 Subject: [PATCH 58/94] Remove premature version entry in CHANGELOG Removed premature entry for version 10.12.3 in CHANGELOG. --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daca501b4..86c319634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [10.12.3] - -### Added -- Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. - ## [10.12.2] ### Added +- Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added the custom hyp3-slimsar-test deployment From 4866b1d9eb943b866adb255dc7250201f5cfee23 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 15 Oct 2025 18:23:30 -0800 Subject: [PATCH 59/94] ITS_LIVE monitoring needs to be able to do cloudformation operations across regions --- cicd-stacks/JPL-deployment-policy-cf.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cicd-stacks/JPL-deployment-policy-cf.yml b/cicd-stacks/JPL-deployment-policy-cf.yml index c5f18a7f5..3b7c6d452 100644 --- a/cicd-stacks/JPL-deployment-policy-cf.yml +++ b/cicd-stacks/JPL-deployment-policy-cf.yml @@ -49,6 +49,7 @@ Resources: - Effect: Allow Action: - cloudformation:SetStackPolicy + - cloudformation:ListStacks - cloudformation:CreateStack - cloudformation:UpdateStack - cloudformation:DeleteStack @@ -57,7 +58,7 @@ Resources: - cloudformation:ExecuteChangeSet - cloudformation:DeleteChangeSet - cloudformation:GetTemplateSummary - Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/*" + Resource: !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/*" ApiGatewayLoggingRole: Type: Custom::JplRole From 6f11ac5fe824a1de1efb7154d28b0c3d18a39f87 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 15 Oct 2025 22:39:09 -0800 Subject: [PATCH 60/94] readme and changelog --- CHANGELOG.md | 1 + README.md | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c319634..a9124a373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. +- Expanded the hyp3-ci stack permissions for JPL deployments to allow listing CloudFormation stacks and CloudFormation permissions were expended to any region to support deploying HyP3-based monitoring stacks. ## [10.12.1] diff --git a/README.md b/README.md index 1ab2619ea..b4d6d320b 100644 --- a/README.md +++ b/README.md @@ -184,10 +184,7 @@ aws cloudformation deploy \ --template-file cicd-stacks/JPL-deployment-policy-cf.yml ``` -*Warning: This stack should only be deployed once per AWS account. This stack also -assumes you are only deploying into a single AWS Region. If you are deploying into -multiple regions in the same AWS account, you'll need to adjust the IAM permissions -that are limited to a single region.* +*Warning: This stack should only be deployed once per AWS account.* Then open a [Cloud Team Service Desk](https://itsd-jira.jpl.nasa.gov/servicedesk/customer/portal/13) request for a service user account here: From e6f8f80779212c28014429fd0c5a15bf0e24814d Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 23 Oct 2025 00:06:43 -0800 Subject: [PATCH 61/94] Add ECR --- CHANGELOG.md | 5 ++++- cicd-stacks/JPL-deployment-policy-cf.yml | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9124a373..a0060c78c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. -- Expanded the hyp3-ci stack permissions for JPL deployments to allow listing CloudFormation stacks and CloudFormation permissions were expended to any region to support deploying HyP3-based monitoring stacks. +- The hyp3-ci stack permission for JPL deployments were expended to support deploying HyP3-based monitoring stacks: + - Listing CloudFormation stacks is now allowed. + - CloudFormation permissions were expended to any region from just us-west-2. + - ECR actions are now allowed. ## [10.12.1] diff --git a/cicd-stacks/JPL-deployment-policy-cf.yml b/cicd-stacks/JPL-deployment-policy-cf.yml index 3b7c6d452..c3f06f86b 100644 --- a/cicd-stacks/JPL-deployment-policy-cf.yml +++ b/cicd-stacks/JPL-deployment-policy-cf.yml @@ -18,6 +18,7 @@ Resources: - dynamodb:* - ec2:* - ecs:* + - ecr:GetAuthorizationToken - events:* - iam:CreateServiceLinkedRole - iam:DeleteServiceLinkedRole @@ -60,6 +61,20 @@ Resources: - cloudformation:GetTemplateSummary Resource: !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/*" + - Effect: Allow + Action: + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:DescribeRepositories + - ecr:ListImages + - ecr:DescribeImages + - ecr:BatchGetImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + - ecr:PutImage + Resource: !Sub "arn:aws:ecr:*:${AWS::AccountId}:repository/*" + ApiGatewayLoggingRole: Type: Custom::JplRole Properties: From c14550b820096c573d703c70574a8ead56e903fb Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 23 Jan 2026 12:53:48 -0900 Subject: [PATCH 62/94] Update CHANGELOG.md Co-authored-by: Andrew Player --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0060c78c..361123c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. -- The hyp3-ci stack permission for JPL deployments were expended to support deploying HyP3-based monitoring stacks: +- The hyp3-ci stack permission for JPL deployments were expanded to support deploying HyP3-based monitoring stacks: - Listing CloudFormation stacks is now allowed. - - CloudFormation permissions were expended to any region from just us-west-2. + - CloudFormation permissions were expanded to any region from just us-west-2. - ECR actions are now allowed. ## [10.12.1] From a9b751478dbe5fd4b3812b2f96521c260f0014d9 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:06:37 -0700 Subject: [PATCH 63/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 976387262..ba822a6ee 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ which we will use to deploy HyP3 via CI/CD tooling: 1. Go to AWS console -> IAM -> Users -> github-actions -> security credentials tab -> "create access key". 2. Select "Other" for key usage -3. Add tag value to describe the key such as `hyp3-foobar-access` +3. (Optional)Add tag value to describe the key, such as "For GitHub Actions CI/CD pipelines" 4. Store the access key ID and secret access key using your team's password manager. You will use them below in "Create the GitHub environment" as `V2_AWS_ACCESS_KEY_ID` and `V2_AWS_SECRET_ACCESS_KEY`.
From 7da9ef237166c281b01f5c78faeb76423a4e77e3 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:06:52 -0700 Subject: [PATCH 64/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba822a6ee..ed3ca07a4 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ in the format ` in CNAME .` (see previous records for ### Create the HyP3 deployment -You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow (.github/workflows/) or create +You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow located in the `.github/workflows/` directory, or create a new one for the deployment. If you need to create a new one, we recommend copying one of the existing workflows, and then updating all of the fields. as appropriate for your deployment. Also make sure to update the top-level `name` of the workflow and the name From 94a2dcc3e354fcf51402d969fdec9eef5a446c12 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:07:30 -0700 Subject: [PATCH 65/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed3ca07a4..e88bc1f07 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ in the format ` in CNAME .` (see previous records for You will need to add the deployment to the matrix in an existing GitHub Actions `deploy-*.yml` workflow located in the `.github/workflows/` directory, or create a new one for the deployment. If you need to create a new one, we recommend copying one of the -existing workflows, and then updating all of the fields. +existing workflows, and then updating all of the fields as appropriate for your deployment. Also make sure to update the top-level `name` of the workflow and the name of the branch to deploy from. (This is typically `main` for prod deployments, `develop` for test deployments, or a feature branch name for sandbox deployments.) From d3e8dc75bae3c33efc98ab827be9183a803e78cf Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:07:37 -0700 Subject: [PATCH 66/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e88bc1f07..5a273cc6f 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ Update the [AWS Accounts and HyP3 Deployments](https://docs.google.com/spreadshe #### Testing and adding user credits to your hyp3 deployment -After successful deployment and DNS record merging you can test you deployment by accessing the Swagger UI and using the POST /user tab to +After successfully deploying HyP3 and your new DNS record has taken effect (or you've edited your local DNS name resolution), you can test your deployment by accessing the Swagger UI and using the POST `/user` tab to check if your user is approved and has credits for running jobs on the deployment. This should automatically add your user to the DynamoDB table. From 467193cb02ec80755ecb8a837ab57013dd4b5278 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:07:43 -0700 Subject: [PATCH 67/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a273cc6f..d8438b00e 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,7 @@ AWS console -> DynamoDB -> Explore items 2. Edit your user (only present after using the Swagger UI in some way) 3. Add credits if neccessary -You can then return the Swagger UI and use the POST /jobs to run a test job and confirm it completes. +You can then return the Swagger UI and use the POST `/jobs` to run a test job and confirm it completes. #### Optional From ad639bdded4609a8b243ab4c68c0e930b52e2c3f Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Mon, 26 Jan 2026 11:08:08 -0700 Subject: [PATCH 68/94] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d8438b00e..cc985d248 100644 --- a/README.md +++ b/README.md @@ -333,13 +333,11 @@ Update the [AWS Accounts and HyP3 Deployments](https://docs.google.com/spreadshe #### Testing and adding user credits to your hyp3 deployment After successfully deploying HyP3 and your new DNS record has taken effect (or you've edited your local DNS name resolution), you can test your deployment by accessing the Swagger UI and using the POST `/user` tab to -check if your user is approved and has credits for running jobs on the deployment. This should automatically add your user to the DynamoDB -table. +check if your user is approved and has credits for running jobs on the deployment. You will need to be authenticated by either providing an Earthdata Login Bearer Token using the "Authorize" button, or by having a valid `asf-urs` cookie, typically logging into [Vertex](https://search.asf.alaska.edu). Interacting with HyP3 should automatically add your user to the DynamoDB table with the default number of credits (typically 0). -AWS console -> DynamoDB -> Explore items -1. Find the table with format like `hyp3-foobar-UsersTable-XXXXXXXXXXXXX` -2. Edit your user (only present after using the Swagger UI in some way) -3. Add credits if neccessary +To add credits to your (or any) user, log in to the AWS console and navigate to DynamoDB -> Explore items, then: +1. Find the table with a format like `hyp3-foobar-UsersTable-XXXXXXXXXXXXX` +2. Edit your user record (only present after using the Swagger UI in some way) You can then return the Swagger UI and use the POST `/jobs` to run a test job and confirm it completes. From 9bcab8b2e305af2edc2fbcaf1c13a8b9765cdb10 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Mon, 26 Jan 2026 11:10:45 -0700 Subject: [PATCH 69/94] addressing Joe's last comment --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc985d248..a4c4c4346 100644 --- a/README.md +++ b/README.md @@ -253,9 +253,9 @@ AWS console -> AWS Certificate Manager -> Request certificate:\ 3. Choose "DNS validation" 4. Copy the "CNAME name" and "CNAME value" -Then copy past the add the validation record to a row of +Then create a validation record in https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy_dns/files/asf.alaska.edu.db -in the format ` in CNAME .` (see previous records for examples). +of the form ` in CNAME `, stripping the `.asf.alaska.edu` from the CNAME name (see previous records for examples). ### Create the GitHub environment From d1e48225586b15992cc4885bba3cd9e176049543 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 21:19:37 +0000 Subject: [PATCH 70/94] Bump actions/checkout in the github-actions-deps group Bumps the github-actions-deps group with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 6.0.1 to 6.0.2 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6.0.1...v6.0.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions-deps ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy-custom-prod.yml | 2 +- .github/workflows/deploy-custom-test.yml | 2 +- .github/workflows/deploy-daac-prod.yml | 2 +- .github/workflows/deploy-daac-test.yml | 2 +- .github/workflows/deploy-edc-sandbox.yml | 2 +- .github/workflows/deploy-plus-prod.yml | 2 +- .github/workflows/deploy-plus-test.yml | 2 +- .github/workflows/static-analysis.yml | 8 ++++---- .github/workflows/tests.yml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-custom-prod.yml b/.github/workflows/deploy-custom-prod.yml index 821e7d3e5..1fed51728 100644 --- a/.github/workflows/deploy-custom-prod.yml +++ b/.github/workflows/deploy-custom-prod.yml @@ -310,7 +310,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index 49527fe46..3d86e0aa6 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -151,7 +151,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-prod.yml b/.github/workflows/deploy-daac-prod.yml index cec1fff20..4917a50e9 100644 --- a/.github/workflows/deploy-daac-prod.yml +++ b/.github/workflows/deploy-daac-prod.yml @@ -42,7 +42,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index 0d0a882f3..c9104bc47 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -44,7 +44,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index 4da050764..fbe25deed 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -45,7 +45,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-plus-prod.yml b/.github/workflows/deploy-plus-prod.yml index 682d487a5..070523784 100644 --- a/.github/workflows/deploy-plus-prod.yml +++ b/.github/workflows/deploy-plus-prod.yml @@ -40,7 +40,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-plus-test.yml b/.github/workflows/deploy-plus-test.yml index 261930ee8..c20000506 100644 --- a/.github/workflows/deploy-plus-test.yml +++ b/.github/workflows/deploy-plus-test.yml @@ -41,7 +41,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 7e3173d5d..a0b067fb4 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -29,7 +29,7 @@ jobs: matrix: security_environment: [ASF, EDC, JPL, JPL-public] steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -43,7 +43,7 @@ jobs: openapi-spec-validator: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -56,7 +56,7 @@ jobs: statelint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 @@ -77,7 +77,7 @@ jobs: runs-on: ubuntu-latest if: ${{ ! github.event.pull_request.head.repo.fork }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: snyk/actions/setup@v1.0.0 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b6894427d..960e03bf7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: actions/setup-python@v6 with: From f5e1b02957b95d205f5f27a41b22c55135c97c3e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 8 Jan 2026 09:57:09 -0900 Subject: [PATCH 71/94] drop publish-stac-prefix as not necessary anymore --- job_spec/ITS_LIVE_AUTORIFT.yml | 7 ------- job_spec/ITS_LIVE_META.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index 7561fab85..d5c0ce06e 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -116,11 +116,6 @@ AUTORIFT: - "its-live-data" - "its-live-data-test" default: null - publish_stac_prefix: - api_schema: - description: Publish the resulting STAC JSON item under this prefix in the ITS_LIVE AWS Open Data (or test) S3 Bucket - type: string - default: stac-ingest use_static_files: api_schema: description: Use static topographic correction files for ISCE3 processing if available (Sentinel-1 only) @@ -177,8 +172,6 @@ AUTORIFT: - Ref::job_id - --publish-bucket - Ref::publish_bucket - - --publish-prefix - - Ref::publish_stac_prefix timeout: 10800 compute_environment: ItsLiveMeta vcpu: 1 diff --git a/job_spec/ITS_LIVE_META.yml b/job_spec/ITS_LIVE_META.yml index d079361fd..9842709cf 100644 --- a/job_spec/ITS_LIVE_META.yml +++ b/job_spec/ITS_LIVE_META.yml @@ -17,11 +17,6 @@ ITS_LIVE_META: - "its-live-data" - "its-live-data-test" default: null - publish_stac_prefix: - api_schema: - description: Publish the resulting STAC JSON item under this prefix in the ITS_LIVE AWS Open Data (or test) S3 Bucket - type: string - default: stac-ingest cost_profiles: DEFAULT: cost: 1.0 @@ -57,8 +52,6 @@ ITS_LIVE_META: - Ref::job_id - --publish-bucket - Ref::publish_bucket - - --publish-prefix - - Ref::publish_stac_prefix timeout: 10800 compute_environment: ItsLiveMeta vcpu: 1 From b2d1131288d5ae3cc11847d9f853ce5056ff3ebc Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 9 Jan 2026 15:08:07 -0900 Subject: [PATCH 72/94] Add a bulk metadata gen job for ITS_LIVE --- .github/workflows/deploy-custom-prod.yml | 1 + .github/workflows/deploy-custom-test.yml | 1 + CHANGELOG.md | 7 +- job_spec/ITS_LIVE_META.yml | 2 + job_spec/ITS_LIVE_META_BULK.yml | 85 ++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 job_spec/ITS_LIVE_META_BULK.yml diff --git a/.github/workflows/deploy-custom-prod.yml b/.github/workflows/deploy-custom-prod.yml index 821e7d3e5..8acc82a9d 100644 --- a/.github/workflows/deploy-custom-prod.yml +++ b/.github/workflows/deploy-custom-prod.yml @@ -25,6 +25,7 @@ jobs: job_files: >- job_spec/ITS_LIVE_AUTORIFT.yml job_spec/ITS_LIVE_META.yml + job_spec/ITS_LIVE_META_BULK.yml instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge default_max_vcpus: 2000 # Max: 10,406 expanded_max_vcpus: 2000 # Max: 10,406 diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index 49527fe46..7f32b7bd2 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -66,6 +66,7 @@ jobs: job_files: >- job_spec/ITS_LIVE_AUTORIFT.yml job_spec/ITS_LIVE_META.yml + job_spec/ITS_LIVE_META_BULK.yml instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge default_max_vcpus: 640 # Max: 10,406 expanded_max_vcpus: 640 # Max: 10,406 diff --git a/CHANGELOG.md b/CHANGELOG.md index a1391af63..e8350928d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. -- Added the custom hyp3-slimsar-test deployment -- Added the new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing - +- Added a new custom hyp3-slimsar-test deployment. +- Added the new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. +- Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for already published ITS_LIVE products. +- Added the `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. - The hyp3-ci stack permission for JPL deployments were expanded to support deploying HyP3-based monitoring stacks: diff --git a/job_spec/ITS_LIVE_META.yml b/job_spec/ITS_LIVE_META.yml index 9842709cf..dfb96b425 100644 --- a/job_spec/ITS_LIVE_META.yml +++ b/job_spec/ITS_LIVE_META.yml @@ -44,6 +44,8 @@ ITS_LIVE_META: - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata command: + - ++plugin + - meta - --granule-uri - Ref::granule_uri - --bucket diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml new file mode 100644 index 000000000..5d96fb81e --- /dev/null +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -0,0 +1,85 @@ +ITS_LIVE_META: + required_parameters: + - granules_parquet + parameters: + granules_parquet: + api_schema: + description: S3 URI for an ITS_LIVE velocity granule to generate metadata for + type: string + pattern: '^s3:\/\/.*\.nc$' + start_idx: + api_schema: + description: Start index (inclusive) of the contiguous subset of granules in granules_parquet to generate metadata for + type: integer + minimum: 0 + default: 0 + stop_idx: + api_schema: + description: Stop index (exclusive) of the contiguous subset of granules in granules_parquet to generate metadata for + type: integer + nullable: true + minimum: 1 + default: null + publish_bucket: + api_schema: + description: Publish the resulting metadata files to the ITS_LIVE AWS Open Data (or test) S3 Bucket + type: string + nullable: true + enum: + - null + - "its-live-data" + - "its-live-data-test" + default: null + cost_profiles: + DEFAULT: + cost: 1.0 + validators: [] + steps: + - name: '' + image: ghcr.io/asfhyp3/hyp3-autorift + command: + - ++process + - bulk_crop_netcdf_product + - --granules-parquet + - Ref::granules_parquet + - --start-idx + - Ref::start_idx + - --stop-idx + - Ref::stop_idx + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::job_id + - --publish-bucket + - Ref::publish_bucket + timeout: 10800 + compute_environment: Default + vcpu: 1 + memory: 7875 + secrets: + - PUBLISH_ACCESS_KEY_ID + - PUBLISH_SECRET_ACCESS_KEY + - name: METADATA + image: ghcr.io/asfhyp3/itslive-metadata + command: + - ++plugin + - bulk_meta + - --granules-parquet + - Ref::granules_parquet + - --start-idx + - Ref::start_idx + - --stop-idx + - Ref::stop_idx + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::job_id + - --publish-bucket + - Ref::publish_bucket + timeout: 10800 + compute_environment: ItsLiveMeta + vcpu: 1 + memory: 7875 + secrets: + - PUBLISH_ACCESS_KEY_ID + - PUBLISH_SECRET_ACCESS_KEY From 572b0ea928d4c1565a784986d9ef2bd78a7b2a06 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 9 Jan 2026 15:27:48 -0900 Subject: [PATCH 73/94] add removed prarmeter to changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8350928d..08caa7d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. - Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for already published ITS_LIVE products. - Added the `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. + ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. - The hyp3-ci stack permission for JPL deployments were expanded to support deploying HyP3-based monitoring stacks: @@ -20,6 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CloudFormation permissions were expanded to any region from just us-west-2. - ECR actions are now allowed. +### Removed +- Removed the `publish_stac_prefix` job parameter from the `ITS_LIVE_AUTORIFT` and `ITS_LIVE_META` job specs as it's no longer used by the ITS_LIVE plugins. + ## [10.12.1] ### Fixed From 8023907e5dc829d688893d770a29044489294f41 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 9 Jan 2026 16:29:12 -0900 Subject: [PATCH 74/94] add tags for test builds of containers --- job_spec/ITS_LIVE_META_BULK.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 5d96fb81e..ef8be1595 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -37,6 +37,7 @@ ITS_LIVE_META: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-autorift + image_tag: "0.26.2.dev10_gc799a46" command: - ++process - bulk_crop_netcdf_product @@ -61,6 +62,7 @@ ITS_LIVE_META: - PUBLISH_SECRET_ACCESS_KEY - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata + image_tag: "0.6.1.dev8_g305e5c531" command: - ++plugin - bulk_meta From b8754d6d27752f0b5a1556691a3537c8296d729d Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 14:43:44 -0900 Subject: [PATCH 75/94] use on demand instances and set timout for 6 hours for its-live bulk meta --- job_spec/ITS_LIVE_META_BULK.yml | 10 +++++----- job_spec/config/compute_environments.yml | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index ef8be1595..ee3c11154 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -37,7 +37,7 @@ ITS_LIVE_META: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-autorift - image_tag: "0.26.2.dev10_gc799a46" + image_tag: "0.26.2.dev12_g202d2aa" command: - ++process - bulk_crop_netcdf_product @@ -53,8 +53,8 @@ ITS_LIVE_META: - Ref::job_id - --publish-bucket - Ref::publish_bucket - timeout: 10800 - compute_environment: Default + timeout: 21600 + compute_environment: ItsLiveAutoRIFT vcpu: 1 memory: 7875 secrets: @@ -62,7 +62,7 @@ ITS_LIVE_META: - PUBLISH_SECRET_ACCESS_KEY - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata - image_tag: "0.6.1.dev8_g305e5c531" + image_tag: "0.6.1.dev11_ga449bc" command: - ++plugin - bulk_meta @@ -78,7 +78,7 @@ ITS_LIVE_META: - Ref::job_id - --publish-bucket - Ref::publish_bucket - timeout: 10800 + timeout: 21600 compute_environment: ItsLiveMeta vcpu: 1 memory: 7875 diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 7f384d935..368c80b02 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -24,9 +24,16 @@ compute_environments: instance_types: m6id.2xlarge,m6id.4xlarge,m6id.8xlarge allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE + ItsLiveAutoRIFT: + instance_types: r7gd.medium,r7gd.large,r7gd.xlarge,r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge + ami_id: ami-0692356050549192d # /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/image_id + allocation_type: EC2 + allocation_strategy: BEST_FIT_PROGRESSIVE ItsLiveMeta: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + allocation_type: EC2 + allocation_strategy: BEST_FIT_PROGRESSIVE SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, allocation_type: EC2 From d5bc76b9cc167c44fdc1a8c7c0d5e8d1c2006d69 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 15:12:29 -0900 Subject: [PATCH 76/94] fix meta bulk job name --- job_spec/ITS_LIVE_META_BULK.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index ee3c11154..33f15a8b8 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -1,4 +1,4 @@ -ITS_LIVE_META: +ITS_LIVE_META_BULK: required_parameters: - granules_parquet parameters: From c28d6b2b5b880a4cf5129c1857b7a1a0c875d094 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 15:37:12 -0900 Subject: [PATCH 77/94] parquet files hsould end in parquet --- job_spec/ITS_LIVE_META_BULK.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 33f15a8b8..758ac94e2 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -6,7 +6,7 @@ ITS_LIVE_META_BULK: api_schema: description: S3 URI for an ITS_LIVE velocity granule to generate metadata for type: string - pattern: '^s3:\/\/.*\.nc$' + pattern: '^s3:\/\/.*\.parquet$' start_idx: api_schema: description: Start index (inclusive) of the contiguous subset of granules in granules_parquet to generate metadata for From 25dde118592519b1c2c4ddb2146ec8fd66be1335 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 21:44:02 -0900 Subject: [PATCH 78/94] update autorift image tag --- job_spec/ITS_LIVE_META_BULK.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 758ac94e2..ae3e583bb 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -37,7 +37,7 @@ ITS_LIVE_META_BULK: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-autorift - image_tag: "0.26.2.dev12_g202d2aa" + image_tag: "0.26.2.dev14_g4b1425d" command: - ++process - bulk_crop_netcdf_product From 400f0d9f8f4e1bf3e74e28a85a8ed1de584fd836 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 22:04:31 -0900 Subject: [PATCH 79/94] increase job memory --- job_spec/ITS_LIVE_META_BULK.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index ae3e583bb..e2bde2e56 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -56,7 +56,7 @@ ITS_LIVE_META_BULK: timeout: 21600 compute_environment: ItsLiveAutoRIFT vcpu: 1 - memory: 7875 + memory: 31500 secrets: - PUBLISH_ACCESS_KEY_ID - PUBLISH_SECRET_ACCESS_KEY @@ -81,7 +81,7 @@ ITS_LIVE_META_BULK: timeout: 21600 compute_environment: ItsLiveMeta vcpu: 1 - memory: 7875 + memory: 31500 secrets: - PUBLISH_ACCESS_KEY_ID - PUBLISH_SECRET_ACCESS_KEY From e50e742f1f6a285a9324bb364d7bf9166757327b Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sun, 11 Jan 2026 22:27:40 -0900 Subject: [PATCH 80/94] fix itslive meatadata image tag --- job_spec/ITS_LIVE_META_BULK.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index e2bde2e56..dfdc4581c 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -62,7 +62,7 @@ ITS_LIVE_META_BULK: - PUBLISH_SECRET_ACCESS_KEY - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata - image_tag: "0.6.1.dev11_ga449bc" + image_tag: "0.6.1.dev11_ga449bc0a4" command: - ++plugin - bulk_meta From a1d4950c0454d6fe082ed5d5ad1e9134507f99b0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 14 Jan 2026 16:20:27 -0900 Subject: [PATCH 81/94] add ITS_LIVE_STAC_BULK and update itslive-metadata to handle missing S1 frame_img? attributes --- job_spec/ITS_LIVE_META_BULK.yml | 2 +- job_spec/ITS_LIVE_STAC_BULK.yml | 62 +++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 job_spec/ITS_LIVE_STAC_BULK.yml diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index dfdc4581c..27d704556 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -62,7 +62,7 @@ ITS_LIVE_META_BULK: - PUBLISH_SECRET_ACCESS_KEY - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata - image_tag: "0.6.1.dev11_ga449bc0a4" + image_tag: "0.6.1.dev12_gb8283ca3f" command: - ++plugin - bulk_meta diff --git a/job_spec/ITS_LIVE_STAC_BULK.yml b/job_spec/ITS_LIVE_STAC_BULK.yml new file mode 100644 index 000000000..930797a01 --- /dev/null +++ b/job_spec/ITS_LIVE_STAC_BULK.yml @@ -0,0 +1,62 @@ +ITS_LIVE_STAC_BULK: + required_parameters: + - granules_parquet + parameters: + granules_parquet: + api_schema: + description: S3 URI for an ITS_LIVE velocity granule to generate metadata for + type: string + pattern: '^s3:\/\/.*\.parquet$' + start_idx: + api_schema: + description: Start index (inclusive) of the contiguous subset of granules in granules_parquet to generate metadata for + type: integer + minimum: 0 + default: 0 + stop_idx: + api_schema: + description: Stop index (exclusive) of the contiguous subset of granules in granules_parquet to generate metadata for + type: integer + nullable: true + minimum: 1 + default: null + publish_bucket: + api_schema: + description: Publish the resulting metadata files to the ITS_LIVE AWS Open Data (or test) S3 Bucket + type: string + nullable: true + enum: + - null + - "its-live-data" + - "its-live-data-test" + default: null + cost_profiles: + DEFAULT: + cost: 1.0 + validators: [] + steps: + - name: '' + image: ghcr.io/asfhyp3/itslive-metadata + image_tag: "0.6.1.dev12_gb8283ca3f" + command: + - ++plugin + - bulk_meta + - --granules-parquet + - Ref::granules_parquet + - --start-idx + - Ref::start_idx + - --stop-idx + - Ref::stop_idx + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::job_id + - --publish-bucket + - Ref::publish_bucket + timeout: 21600 + compute_environment: ItsLiveMeta + vcpu: 1 + memory: 31500 + secrets: + - PUBLISH_ACCESS_KEY_ID + - PUBLISH_SECRET_ACCESS_KEY From af9847ab736b20a37ca9715f8347ae0bb4a141cd Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 21 Jan 2026 21:29:22 -0900 Subject: [PATCH 82/94] Clean up ITS_LIVE job specs --- .github/workflows/deploy-custom-prod.yml | 3 +- .github/workflows/deploy-custom-test.yml | 3 +- CHANGELOG.md | 10 ++++--- .../{ITS_LIVE_META.yml => ITS_LIVE_CROP.yml} | 2 +- ...E_STAC_BULK.yml => ITS_LIVE_CROP_BULK.yml} | 29 +++++++++++++++++-- job_spec/ITS_LIVE_META_BULK.yml | 28 +----------------- job_spec/config/compute_environments.yml | 5 +++- 7 files changed, 42 insertions(+), 38 deletions(-) rename job_spec/{ITS_LIVE_META.yml => ITS_LIVE_CROP.yml} (99%) rename job_spec/{ITS_LIVE_STAC_BULK.yml => ITS_LIVE_CROP_BULK.yml} (70%) diff --git a/.github/workflows/deploy-custom-prod.yml b/.github/workflows/deploy-custom-prod.yml index 8acc82a9d..d666b86c7 100644 --- a/.github/workflows/deploy-custom-prod.yml +++ b/.github/workflows/deploy-custom-prod.yml @@ -24,7 +24,8 @@ jobs: cost_profile: DEFAULT job_files: >- job_spec/ITS_LIVE_AUTORIFT.yml - job_spec/ITS_LIVE_META.yml + job_spec/ITS_LIVE_CROP.yml + job_spec/ITS_LIVE_CROP_BULK.yml job_spec/ITS_LIVE_META_BULK.yml instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge default_max_vcpus: 2000 # Max: 10,406 diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index 7f32b7bd2..20ed60a2d 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -65,7 +65,8 @@ jobs: cost_profile: DEFAULT job_files: >- job_spec/ITS_LIVE_AUTORIFT.yml - job_spec/ITS_LIVE_META.yml + job_spec/ITS_LIVE_CROP.yml + job_spec/ITS_LIVE_CROP_BULK.yml job_spec/ITS_LIVE_META_BULK.yml instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge default_max_vcpus: 640 # Max: 10,406 diff --git a/CHANGELOG.md b/CHANGELOG.md index 08caa7d2f..65c145a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,19 +10,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added a new custom hyp3-slimsar-test deployment. -- Added the new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. -- Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for already published ITS_LIVE products. -- Added the `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. +- Added a new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. +- Added a new `ITS_LIVE_CROP_BULK` job spec which re-crops existing ITS_LIVE products which are specified in a parquet file to ensure they are chunk-aligned and have a time dimension, and then it generates STAC JSON and other metadata files. +- Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for existing ITS_LIVE products which are specified in a parquet file. +- Added the `ITS_LIVE_CROP_BULK` and `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. +- The `ITS_LIVE_META` job spec has been renamed `ITS_LIVE_CROP` as it re-crops an existing ITS_LIVE product to ensure it is chunk-aligned and has a time dimension, and then it generates STAC JSON and other metadata files. - The hyp3-ci stack permission for JPL deployments were expanded to support deploying HyP3-based monitoring stacks: - Listing CloudFormation stacks is now allowed. - CloudFormation permissions were expanded to any region from just us-west-2. - ECR actions are now allowed. ### Removed -- Removed the `publish_stac_prefix` job parameter from the `ITS_LIVE_AUTORIFT` and `ITS_LIVE_META` job specs as it's no longer used by the ITS_LIVE plugins. +- Removed the `publish_stac_prefix` job parameter from the `ITS_LIVE_AUTORIFT` and `ITS_LIVE_CROP` (previously `ITS_LIVE_META`) job specs as it's no longer used by the ITS_LIVE plugins. ## [10.12.1] diff --git a/job_spec/ITS_LIVE_META.yml b/job_spec/ITS_LIVE_CROP.yml similarity index 99% rename from job_spec/ITS_LIVE_META.yml rename to job_spec/ITS_LIVE_CROP.yml index dfb96b425..730c4819f 100644 --- a/job_spec/ITS_LIVE_META.yml +++ b/job_spec/ITS_LIVE_CROP.yml @@ -1,4 +1,4 @@ -ITS_LIVE_META: +ITS_LIVE_CROP: required_parameters: - granule_uri parameters: diff --git a/job_spec/ITS_LIVE_STAC_BULK.yml b/job_spec/ITS_LIVE_CROP_BULK.yml similarity index 70% rename from job_spec/ITS_LIVE_STAC_BULK.yml rename to job_spec/ITS_LIVE_CROP_BULK.yml index 930797a01..cb881a777 100644 --- a/job_spec/ITS_LIVE_STAC_BULK.yml +++ b/job_spec/ITS_LIVE_CROP_BULK.yml @@ -1,4 +1,4 @@ -ITS_LIVE_STAC_BULK: +ITS_LIVE_CROP_BULK: required_parameters: - granules_parquet parameters: @@ -36,8 +36,31 @@ ITS_LIVE_STAC_BULK: validators: [] steps: - name: '' + image: ghcr.io/asfhyp3/hyp3-autorift + command: + - ++process + - bulk_crop_netcdf_product + - --granules-parquet + - Ref::granules_parquet + - --start-idx + - Ref::start_idx + - --stop-idx + - Ref::stop_idx + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::job_id + - --publish-bucket + - Ref::publish_bucket + timeout: 21600 + compute_environment: ItsLiveAutoRIFTOnDemand + vcpu: 1 + memory: 31500 + secrets: + - PUBLISH_ACCESS_KEY_ID + - PUBLISH_SECRET_ACCESS_KEY + - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata - image_tag: "0.6.1.dev12_gb8283ca3f" command: - ++plugin - bulk_meta @@ -54,7 +77,7 @@ ITS_LIVE_STAC_BULK: - --publish-bucket - Ref::publish_bucket timeout: 21600 - compute_environment: ItsLiveMeta + compute_environment: ItsLiveMetaOnDemand vcpu: 1 memory: 31500 secrets: diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 27d704556..0034dd53b 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -36,33 +36,7 @@ ITS_LIVE_META_BULK: validators: [] steps: - name: '' - image: ghcr.io/asfhyp3/hyp3-autorift - image_tag: "0.26.2.dev14_g4b1425d" - command: - - ++process - - bulk_crop_netcdf_product - - --granules-parquet - - Ref::granules_parquet - - --start-idx - - Ref::start_idx - - --stop-idx - - Ref::stop_idx - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::job_id - - --publish-bucket - - Ref::publish_bucket - timeout: 21600 - compute_environment: ItsLiveAutoRIFT - vcpu: 1 - memory: 31500 - secrets: - - PUBLISH_ACCESS_KEY_ID - - PUBLISH_SECRET_ACCESS_KEY - - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata - image_tag: "0.6.1.dev12_gb8283ca3f" command: - ++plugin - bulk_meta @@ -79,7 +53,7 @@ ITS_LIVE_META_BULK: - --publish-bucket - Ref::publish_bucket timeout: 21600 - compute_environment: ItsLiveMeta + compute_environment: ItsLiveMetaOnDemand vcpu: 1 memory: 31500 secrets: diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 368c80b02..689f9bc35 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -24,7 +24,7 @@ compute_environments: instance_types: m6id.2xlarge,m6id.4xlarge,m6id.8xlarge allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE - ItsLiveAutoRIFT: + ItsLiveAutoRIFTOnDemand: instance_types: r7gd.medium,r7gd.large,r7gd.xlarge,r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge ami_id: ami-0692356050549192d # /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/image_id allocation_type: EC2 @@ -32,6 +32,9 @@ compute_environments: ItsLiveMeta: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + ItsLiveMetaOnDemand: + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE SlimSAR: From b16f99af291654ba3686c3112b9ce731aa5032cb Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 23 Jan 2026 15:43:25 -0900 Subject: [PATCH 83/94] Apply suggestions from code review Co-authored-by: Andrew Player --- job_spec/ITS_LIVE_CROP_BULK.yml | 4 ++-- job_spec/ITS_LIVE_META_BULK.yml | 2 +- job_spec/config/compute_environments.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/job_spec/ITS_LIVE_CROP_BULK.yml b/job_spec/ITS_LIVE_CROP_BULK.yml index cb881a777..398cae7f5 100644 --- a/job_spec/ITS_LIVE_CROP_BULK.yml +++ b/job_spec/ITS_LIVE_CROP_BULK.yml @@ -53,7 +53,7 @@ ITS_LIVE_CROP_BULK: - --publish-bucket - Ref::publish_bucket timeout: 21600 - compute_environment: ItsLiveAutoRIFTOnDemand + compute_environment: ItsLiveCropBulk vcpu: 1 memory: 31500 secrets: @@ -77,7 +77,7 @@ ITS_LIVE_CROP_BULK: - --publish-bucket - Ref::publish_bucket timeout: 21600 - compute_environment: ItsLiveMetaOnDemand + compute_environment: ItsLiveMetaBulk vcpu: 1 memory: 31500 secrets: diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 0034dd53b..b5ffbc134 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -53,7 +53,7 @@ ITS_LIVE_META_BULK: - --publish-bucket - Ref::publish_bucket timeout: 21600 - compute_environment: ItsLiveMetaOnDemand + compute_environment: ItsLiveMetaBulk vcpu: 1 memory: 31500 secrets: diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 689f9bc35..2222e6f40 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -24,7 +24,7 @@ compute_environments: instance_types: m6id.2xlarge,m6id.4xlarge,m6id.8xlarge allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE - ItsLiveAutoRIFTOnDemand: + ItsLiveCropBulk: instance_types: r7gd.medium,r7gd.large,r7gd.xlarge,r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge ami_id: ami-0692356050549192d # /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/image_id allocation_type: EC2 @@ -32,7 +32,7 @@ compute_environments: ItsLiveMeta: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id - ItsLiveMetaOnDemand: + ItsLiveMetaBulk: instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge ami_id: ami-0aece254fc7c27a77 # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id allocation_type: EC2 From d5fda41baa419acdc5eb8009b76c06fe3b28b056 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 26 Jan 2026 14:16:32 -0900 Subject: [PATCH 84/94] bump to a minor release due to the number of additions and changes in the release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c145a5a..3590a70d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [10.12.2] +## [10.13.0] ### Added - Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. From d950c6ae313d6cdbcd3133be55341cd2160a9bf1 Mon Sep 17 00:00:00 2001 From: cmspeed Date: Mon, 26 Jan 2026 09:25:38 -0800 Subject: [PATCH 85/94] Update ARIA_AUTORIFT.yml to support processing of S1C/D and S2C/D granules --- job_spec/ARIA_AUTORIFT.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/job_spec/ARIA_AUTORIFT.yml b/job_spec/ARIA_AUTORIFT.yml index 882ad88eb..8f2276e22 100644 --- a/job_spec/ARIA_AUTORIFT.yml +++ b/job_spec/ARIA_AUTORIFT.yml @@ -14,7 +14,7 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 IW SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 @@ -26,7 +26,7 @@ AUTORIFT: example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 @@ -49,7 +49,7 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 IW SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 @@ -61,7 +61,7 @@ AUTORIFT: example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 @@ -85,13 +85,13 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 From 341f2d49203e255ffffe299b608f4fb5ba0ed7e4 Mon Sep 17 00:00:00 2001 From: cmspeed Date: Mon, 26 Jan 2026 09:31:35 -0800 Subject: [PATCH 86/94] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3590a70d1..92110caef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. +- Updated `ARIA_AUTORIFT.yml` validation schema to support processing of Sentinel-1D and Sentinel-2C/D granules. - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added a new custom hyp3-slimsar-test deployment. - Added a new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. From 8d2f63db70b22c9f0dc10babd0b01b49595d9587 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 26 Jan 2026 15:37:47 -0900 Subject: [PATCH 87/94] add S1 D and S2 C+D to autorift and its-live-autorift job specs --- CHANGELOG.md | 2 +- job_spec/AUTORIFT.yml | 4 ++-- job_spec/ITS_LIVE_AUTORIFT.yml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92110caef..61c9e0ca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added optional `chip_size` and `search_range` parameters to the `ARIA_AUTORIFT.yml` job specification to enable user-defined `chip-size` and `search-range`. -- Updated `ARIA_AUTORIFT.yml` validation schema to support processing of Sentinel-1D and Sentinel-2C/D granules. +- Updated `AUTORIFT.yml`, `ARIA_AUTORIFT.yml`, and `ITS_LIVE_AUTORIFT.yml` validation schema to support processing of Sentinel-1D and Sentinel-2C/D granules. - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added a new custom hyp3-slimsar-test deployment. - Added a new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. diff --git a/job_spec/AUTORIFT.yml b/job_spec/AUTORIFT.yml index 358507aa4..03f6efd30 100644 --- a/job_spec/AUTORIFT.yml +++ b/job_spec/AUTORIFT.yml @@ -14,13 +14,13 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index d5c0ce06e..7db46c735 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -14,7 +14,7 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 IW SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 @@ -26,7 +26,7 @@ AUTORIFT: example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 @@ -49,7 +49,7 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 IW SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 @@ -61,7 +61,7 @@ AUTORIFT: example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 @@ -85,13 +85,13 @@ AUTORIFT: anyOf: - description: The name of the Sentinel-1 SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + pattern: "^S1[ABCD]_IW_SLC__1S[SD][VH]" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 - description: The name of the Sentinel-2 granule to process (ESA naming convention) type: string - pattern: "^S2[AB]_MSIL1C_" + pattern: "^S2[ABCD]_MSIL1C_" minLength: 60 maxLength: 60 example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 From 14f35f1002bd7a7e7f0d213233b79a6161e85fef Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Tue, 27 Jan 2026 15:10:23 -0700 Subject: [PATCH 88/94] add in allocation strategy for slimsarCompute environment and changelog --- CHANGELOG.md | 1 + job_spec/config/compute_environments.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c9e0ca7..87b1e4858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added a new custom hyp3-slimsar-test deployment. - Added a new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. +- Added a new `SlimSAR` compute environment for slimsar processing. - Added a new `ITS_LIVE_CROP_BULK` job spec which re-crops existing ITS_LIVE products which are specified in a parquet file to ensure they are chunk-aligned and have a time dimension, and then it generates STAC JSON and other metadata files. - Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for existing ITS_LIVE products which are specified in a parquet file. - Added the `ITS_LIVE_CROP_BULK` and `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 2222e6f40..3e524cd45 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -39,5 +39,6 @@ compute_environments: allocation_strategy: BEST_FIT_PROGRESSIVE SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, - allocation_type: EC2 ami_id: ami-03aa99ddf5498ceb9 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id + allocation_type: EC2 + allocation_strategy: BEST_FIT_PROGRESSIVE From 2bd0271b0d740ae4c9a6d16323b6e1969316bce6 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:51:27 -0700 Subject: [PATCH 89/94] Update SlimSAR instance types in compute_environments.yml Removed additional instance types for SlimSAR configuration. --- job_spec/config/compute_environments.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 3e524cd45..a696ff7fe 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -38,7 +38,7 @@ compute_environments: allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE SlimSAR: - instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.6xlarge, + instance_types: g4dn.2xlarge ami_id: ami-03aa99ddf5498ceb9 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE From 764b846e3cca5b953c8e427a5ea57cd5be1fe8c8 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen Date: Wed, 28 Jan 2026 13:32:31 -0700 Subject: [PATCH 90/94] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87b1e4858..2cc6d8b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `model_context_length` parameter to the `OPERA_DIST_S1` job specification. - Added a new custom hyp3-slimsar-test deployment. - Added a new `SLIMSAR_TDBP` job_spec for slimsar time-domain backprojection processing. -- Added a new `SlimSAR` compute environment for slimsar processing. +- Added a new `SlimSAR` compute environment for slimsar processing with correct EC2 instances. - Added a new `ITS_LIVE_CROP_BULK` job spec which re-crops existing ITS_LIVE products which are specified in a parquet file to ensure they are chunk-aligned and have a time dimension, and then it generates STAC JSON and other metadata files. - Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for existing ITS_LIVE products which are specified in a parquet file. - Added the `ITS_LIVE_CROP_BULK` and `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. From 644af0b2e813f31f5c79c13cae4218c4b5302782 Mon Sep 17 00:00:00 2001 From: Zachary Hoppinen <56941602+ZachHoppinen@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:02:01 -0700 Subject: [PATCH 91/94] Update job_spec/config/compute_environments.yml add in correct instances Co-authored-by: Joseph H Kennedy --- job_spec/config/compute_environments.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index a696ff7fe..a0a30f7cd 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -38,7 +38,7 @@ compute_environments: allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE SlimSAR: - instance_types: g4dn.2xlarge + instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.8xlarge,g4dn.16xlarge ami_id: ami-03aa99ddf5498ceb9 # /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id allocation_type: EC2 allocation_strategy: BEST_FIT_PROGRESSIVE From ef5ff01f71133bccd98b73931f66efa83eaec81f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 29 Jan 2026 14:53:22 -0900 Subject: [PATCH 92/94] Update ITS_LIVE autorift and crop jobs to allow automatically publishing/updating a STAC item to/in a collection --- CHANGELOG.md | 1 + job_spec/ITS_LIVE_AUTORIFT.yml | 20 +++++++++++++++++++- job_spec/ITS_LIVE_CROP.yml | 18 +++++++++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc6d8b19..1a3672543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a new `ITS_LIVE_CROP_BULK` job spec which re-crops existing ITS_LIVE products which are specified in a parquet file to ensure they are chunk-aligned and have a time dimension, and then it generates STAC JSON and other metadata files. - Added a new `ITS_LIVE_META_BULK` job spec which generates STAC JSON and other metadata files for existing ITS_LIVE products which are specified in a parquet file. - Added the `ITS_LIVE_CROP_BULK` and `ITS_LIVE_META_BULK` job spec to the ITS_LIVE deployments. +- Added the `stac_items_endpoint` and `stac_exists_okay` job parameters to the `ITS_LIVE_AUTORIFT` and `ITS_LIVE_CROP` job specs to allow directly publishing STAC items to the ITS_LIVE STAC catalog. ### Changed - Increased the maximum `stride_for_norm_param_estimation` to 32 from 16 for the `OPERA_DIST_S1` job specification to handle models with a larger input size. diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index 7db46c735..40c0e10ed 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -108,7 +108,7 @@ AUTORIFT: default: '/vsicurl/https://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp' publish_bucket: api_schema: - description: Publish the resulting product to the ITS_LIVE AWS Open Data (or test) S3 Bucket + description: Publish the resulting product files to the ITS_LIVE AWS Open Data (or test) S3 Bucket type: string nullable: true enum: @@ -127,6 +127,17 @@ AUTORIFT: default: null nullable: true type: string + stac_items_endpoint: + api_schema: + description: STAC items endpoint URL for the collection you want to add items to + type: string + format: uri + default: 'https://stac.itslive.cloud/collections/itslive-granules/items' + stac_exists_ok: + api_schema: + description: Allows updating existing STAC items + type: boolean + default: false cost_profiles: DEFAULT: cost: 1.0 @@ -166,16 +177,23 @@ AUTORIFT: - name: METADATA image: ghcr.io/asfhyp3/itslive-metadata command: + - ++plugin + - meta - --bucket - '!Ref Bucket' - --bucket-prefix - Ref::job_id - --publish-bucket - Ref::publish_bucket + - --stac-items-endpoint + - Ref::stac_items_endpoint + - --stac-exists-ok + - Ref::stac_exists_ok timeout: 10800 compute_environment: ItsLiveMeta vcpu: 1 memory: 7875 secrets: + - STAC_API_TOKEN - PUBLISH_ACCESS_KEY_ID - PUBLISH_SECRET_ACCESS_KEY diff --git a/job_spec/ITS_LIVE_CROP.yml b/job_spec/ITS_LIVE_CROP.yml index 730c4819f..a8d8016d1 100644 --- a/job_spec/ITS_LIVE_CROP.yml +++ b/job_spec/ITS_LIVE_CROP.yml @@ -9,7 +9,7 @@ ITS_LIVE_CROP: pattern: '^s3:\/\/.*\.nc$' publish_bucket: api_schema: - description: Publish the resulting metadata files to the ITS_LIVE AWS Open Data (or test) S3 Bucket + description: Publish the resulting product files to the ITS_LIVE AWS Open Data (or test) S3 Bucket type: string nullable: true enum: @@ -17,6 +17,17 @@ ITS_LIVE_CROP: - "its-live-data" - "its-live-data-test" default: null + stac_items_endpoint: + api_schema: + description: STAC items endpoint URL for the collection you want to add items to + type: string + format: uri + default: 'https://stac.itslive.cloud/collections/itslive-granules/items' + stac_exists_ok: + api_schema: + description: Allows updating existing STAC items + type: boolean + default: true cost_profiles: DEFAULT: cost: 1.0 @@ -54,10 +65,15 @@ ITS_LIVE_CROP: - Ref::job_id - --publish-bucket - Ref::publish_bucket + - --stac-items-endpoint + - Ref::stac_items_endpoint + - --stac-exists-ok + - Ref::stac_exists_ok timeout: 10800 compute_environment: ItsLiveMeta vcpu: 1 memory: 7875 secrets: + - STAC_API_TOKEN - PUBLISH_ACCESS_KEY_ID - PUBLISH_SECRET_ACCESS_KEY From ae95e297516f0d7845a6ee122770388e64f1d812 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 30 Jan 2026 10:59:24 -0900 Subject: [PATCH 93/94] make stac endpoint nullable --- job_spec/ITS_LIVE_AUTORIFT.yml | 3 ++- job_spec/ITS_LIVE_CROP.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index 40c0e10ed..247fb8bea 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -131,8 +131,9 @@ AUTORIFT: api_schema: description: STAC items endpoint URL for the collection you want to add items to type: string + nullable: true format: uri - default: 'https://stac.itslive.cloud/collections/itslive-granules/items' + default: null stac_exists_ok: api_schema: description: Allows updating existing STAC items diff --git a/job_spec/ITS_LIVE_CROP.yml b/job_spec/ITS_LIVE_CROP.yml index a8d8016d1..b34790092 100644 --- a/job_spec/ITS_LIVE_CROP.yml +++ b/job_spec/ITS_LIVE_CROP.yml @@ -21,8 +21,9 @@ ITS_LIVE_CROP: api_schema: description: STAC items endpoint URL for the collection you want to add items to type: string + nullable: true format: uri - default: 'https://stac.itslive.cloud/collections/itslive-granules/items' + default: null stac_exists_ok: api_schema: description: Allows updating existing STAC items From b3e60e7a6922d44b4e6b5f56c625cdb0d78c95f0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 30 Jan 2026 14:55:27 -0900 Subject: [PATCH 94/94] Fix formatting and enhance clarity in README Corrected formatting and improved clarity in the README. --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0dca78bdc..87ac49f89 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ which we will use to deploy HyP3 via CI/CD tooling: 1. Go to AWS console -> IAM -> Users -> github-actions -> security credentials tab -> "create access key". 2. Select "Other" for key usage -3. (Optional)Add tag value to describe the key, such as "For GitHub Actions CI/CD pipelines" +3. (Optional) Add tag value to describe the key, such as "For GitHub Actions CI/CD pipelines" 4. Store the access key ID and secret access key using your team's password manager. You will use them below in "Create the GitHub environment" as `V2_AWS_ACCESS_KEY_ID` and `V2_AWS_SECRET_ACCESS_KEY`.
@@ -237,13 +237,14 @@ Go to AWS console -> Secrets Manager, then: #### Request SSL cert -*Note: For EDC accounts, you should create the cert in the `us-east-1` region -for use with the CloudFront distribution that you will create later, -even if you're deploying HyP3 to `us-west-2`.* +To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL), which we can request from AWS. -To allow HTTPS connections, HyP3 needs an SSL certificate that is valid for its deployment domain name (URL): +[!NOTE] +> For EDC accounts, you should create the cert in the `us-east-1` region +> for use with the CloudFront distribution that you will create later, +> even if you're deploying HyP3 to `us-west-2`.* -AWS console -> AWS Certificate Manager -> Request certificate:\ +Go to the AWS console -> AWS Certificate Manager -> Request certificate and then: 1. Select "Request a public certificate" 2. Click the orange "Next" button 3. Choose a "Fully qualified domain name". Domain name should be something like `hyp3-foobar.asf.alaska.edu` or for a test deployment `hyp3-foobar-test.asf.alaska.edu`. @@ -252,7 +253,7 @@ AWS console -> AWS Certificate Manager -> Request certificate:\ Then create a validation record in https://gitlab.asf.alaska.edu/operations/puppet/-/edit/production/modules/legacy_dns/files/asf.alaska.edu.db -of the form ` in CNAME `, stripping the `.asf.alaska.edu` from the CNAME name (see previous records for examples). +of the form ` in CNAME `, stripping `.asf.alaska.edu` from the `CNAME_name` (see previous records for examples). ### Create the GitHub environment @@ -329,12 +330,15 @@ Update the [AWS Accounts and HyP3 Deployments](https://docs.google.com/spreadshe #### Testing and adding user credits to your hyp3 deployment -After successfully deploying HyP3 and your new DNS record has taken effect (or you've edited your local DNS name resolution), you can test your deployment by accessing the Swagger UI and using the POST `/user` tab to -check if your user is approved and has credits for running jobs on the deployment. You will need to be authenticated by either providing an Earthdata Login Bearer Token using the "Authorize" button, or by having a valid `asf-urs` cookie, typically logging into [Vertex](https://search.asf.alaska.edu). Interacting with HyP3 should automatically add your user to the DynamoDB table with the default number of credits (typically 0). +After successfully deploying HyP3 and your new DNS record has taken effect (or you've edited your local DNS name resolution), you can test your +deployment by accessing the Swagger UI and using the POST `/user` tab to check if your user is approved and has credits for running jobs on the +deployment. You will need to be authenticated by either providing an Earthdata Login Bearer Token using the "Authorize" button, or by having a +valid `asf-urs` browser cookie, typically obtained by logging into [Vertex](https://search.asf.alaska.edu). Interacting with HyP3 should +automatically add your user to the DynamoDB table with the default number of credits (typically 0). To add credits to your (or any) user, log in to the AWS console and navigate to DynamoDB -> Explore items, then: 1. Find the table with a format like `hyp3-foobar-UsersTable-XXXXXXXXXXXXX` -2. Edit your user record (only present after using the Swagger UI in some way) +2. Edit your user record if present (after using the Swagger UI in some way) or duplicate an existing reccord updaing the `user_id`. You can then return the Swagger UI and use the POST `/jobs` to run a test job and confirm it completes.