From df8052af8bb325f114cc66f3001d59dacf88ae3d Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 13:01:24 -0400 Subject: [PATCH 01/12] initial commit: added cicd --- .github/workflows/cicd.yaml | 220 ++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 .github/workflows/cicd.yaml diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 000000000..3f2ff68c4 --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,220 @@ +name: CI/CD Pipeline + +on: + pull_request: + branches: [main, development] + push: + branches: [main, development] + tags: ['v*.*.*'] + +permissions: + contents: read + packages: write + security-events: write + +env: + REGISTRY: ghcr.io + PYTHON_VERSION: '3.13' + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + image_base: ${{ steps.vars.outputs.image_base }} + pr_tag: ${{ steps.vars.outputs.pr_tag }} + commit_sha: ${{ steps.vars.outputs.commit_sha }} + commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} + test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + steps: + - name: Compute image vars + id: vars + shell: bash + run: | + set -euo pipefail + ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + REPO="$(basename "${GITHUB_REPOSITORY}")" + IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + PR_NUM="${{ github.event.pull_request.number }}" + PR_TAG="pr-${PR_NUM}-build" + echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + fi + + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + COMMIT_SHA="${GITHUB_SHA}" + SHORT_SHA="${COMMIT_SHA:0:12}" + echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + fi + + build-and-scan-ingest: + name: Build and Scan Ingest Container + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + - name: Build ingest image for scanning + id: build-ingest + uses: docker/build-push-action@v6 + with: + context: ./Ingest + file: ./Ingest/docker/Dockerfile.ingest + # Load the image to the local Docker daemon, but do not push it + load: true + tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + - name: Scan ingest container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + # Scan the locally available image + image-ref: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-ingest.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Ingest Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-ingest.sarif' + category: 'ingest-container' + + build-and-scan-rnr: + name: Build and Scan RnR Container + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + - name: Build RnR image for scanning + id: build-rnr + uses: docker/build-push-action@v6 + with: + context: ./RnR + file: ./RnR/docker/Dockerfile.troute + load: true + tags: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} + - name: Scan RnR container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + image-ref: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-rnr.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload RnR Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-rnr.sarif' + category: 'rnr-container' + + codeql-scan: + name: CodeQL Scan + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + # The 'unit-test' job is disabled for now. When you re-enable it, uncomment the line below. + # needs: unit-test + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ env.PYTHON_VERSION }} + cache-dependency-glob: "**/uv.lock **/pyproject.toml" + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + - name: Install Ingest dependencies + run: | + if [ -f "Source/Ingest/pyproject.toml" ]; then + cd Source/Ingest && uv sync + fi + - name: Install RnR dependencies + run: | + if [ -f "Source/RnR/pyproject.toml" ]; then + cd Source/RnR && uv sync + fi + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + publish-ingest: + name: Publish Ingest to Registry + if: > + github.event_name == 'push' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/development' || + startsWith(github.ref, 'refs/tags/v') + ) + runs-on: ubuntu-latest + needs: [setup, build-and-scan-ingest, codeql-scan] + steps: + - uses: actions/checkout@v4 + - name: Prepare ingest image tags + id: prep_tags + run: | + # Always start with the unique commit SHA tag for traceability + TAGS="${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.commit_sha_short }}" + + # If it's a push to the main branch, also add the 'latest' tag + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:latest" + fi + + # If the trigger was a version tag, add that version as a tag + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + # github.ref_name holds the tag name (e.g., "v1.0.0") + VERSION_TAG=${{ github.ref_name }} + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:${VERSION_TAG}" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push ingest image + uses: docker/build-push-action@v6 + with: + context: ./Ingest + file: ./Ingest/docker/Dockerfile.ingest + push: true + tags: ${{ steps.prep_tags.outputs.tags }} + + publish-rnr: + name: Publish RnR to Registry + if: > + github.event_name == 'push' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/development' || + startsWith(github.ref, 'refs/tags/v') + ) + runs-on: ubuntu-latest + needs: [setup, build-and-scan-rnr, codeql-scan] + steps: + - uses: actions/checkout@v4 + - name: Prepare RnR image tags + id: prep_tags + run: | + # Always start with the unique commit SHA tag for traceability + TAGS="${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.commit_sha_short }}" + + # If it's a push to the main branch, also add the 'latest' tag + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:latest" + fi + + # If the trigger was a version tag, add that version as a tag + if [[ "${{ github.ref }}" == refs/tags/v From d4f0ad3b276e8d3c83244aeadafc505df09725b3 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 13:07:12 -0400 Subject: [PATCH 02/12] added: a new PR branch req to test docker builds --- .github/workflows/cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 3f2ff68c4..cb959b204 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -2,7 +2,7 @@ name: CI/CD Pipeline on: pull_request: - branches: [main, development] + branches: [main, development, pi-7-NGWPC-6258] push: branches: [main, development] tags: ['v*.*.*'] From ef598106164c8164194f4b0ceefad55b51dd4537 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 13:10:18 -0400 Subject: [PATCH 03/12] patch: fixing path issues --- .github/workflows/cicd.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index cb959b204..f373eb545 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -61,8 +61,8 @@ jobs: id: build-ingest uses: docker/build-push-action@v6 with: - context: ./Ingest - file: ./Ingest/docker/Dockerfile.ingest + context: ./Source/Ingest + file: docker/Dockerfile.ingest # Load the image to the local Docker daemon, but do not push it load: true tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} @@ -91,8 +91,8 @@ jobs: id: build-rnr uses: docker/build-push-action@v6 with: - context: ./RnR - file: ./RnR/docker/Dockerfile.troute + context: ./Source/RnR + file: docker/Dockerfile.troute load: true tags: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} - name: Scan RnR container with Trivy From b67b933e8e5fcff1a55737f1026d632f6a491ecb Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 13:22:00 -0400 Subject: [PATCH 04/12] patch fixed another path error for builds --- .github/workflows/cicd.yaml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index f373eb545..f3c52f06f 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -188,8 +188,8 @@ jobs: - name: Build & push ingest image uses: docker/build-push-action@v6 with: - context: ./Ingest - file: ./Ingest/docker/Dockerfile.ingest + context: ./Source/Ingest + file: docker/Dockerfile.ingest push: true tags: ${{ steps.prep_tags.outputs.tags }} @@ -217,4 +217,23 @@ jobs: fi # If the trigger was a version tag, add that version as a tag - if [[ "${{ github.ref }}" == refs/tags/v + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + # github.ref_name holds the tag name (e.g., "v1.0.0") + VERSION_TAG=${{ github.ref_name }} + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:${VERSION_TAG}" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push RnR image + uses: docker/build-push-action@v6 + with: + context: ./Source/RnR + file: docker/Dockerfile.troute + push: true + tags: ${{ steps.prep_tags.outputs.tags }} From accd21980ce7eabdd0fd8d0f3ea0c8a468b015a4 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 13:38:34 -0400 Subject: [PATCH 05/12] patch: added additional dependencies or codeql --- .github/workflows/cicd.yaml | 23 +++++++++++------------ Source/Ingest/docker/Dockerfile.ingest | 9 ++++++++- Source/RnR/docker/Dockerfile.troute | 19 ++++++++++++++----- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index f3c52f06f..e5f454178 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -113,8 +113,6 @@ jobs: name: CodeQL Scan if: github.event_name == 'pull_request' || github.event_name == 'push' runs-on: ubuntu-latest - # The 'unit-test' job is disabled for now. When you re-enable it, uncomment the line below. - # needs: unit-test permissions: actions: read contents: read @@ -135,16 +133,17 @@ jobs: uses: github/codeql-action/init@v3 with: languages: python - - name: Install Ingest dependencies - run: | - if [ -f "Source/Ingest/pyproject.toml" ]; then - cd Source/Ingest && uv sync - fi - - name: Install RnR dependencies - run: | - if [ -f "Source/RnR/pyproject.toml" ]; then - cd Source/RnR && uv sync - fi + # Commenting out as binary/wheels are missing + # - name: Install Ingest dependencies + # run: | + # if [ -f "Source/Ingest/pyproject.toml" ]; then + # cd Source/Ingest && uv sync + # fi + # - name: Install RnR dependencies + # run: | + # if [ -f "Source/RnR/pyproject.toml" ]; then + # cd Source/RnR && uv sync + # fi - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 diff --git a/Source/Ingest/docker/Dockerfile.ingest b/Source/Ingest/docker/Dockerfile.ingest index 7cd632ba5..2efcfd79c 100644 --- a/Source/Ingest/docker/Dockerfile.ingest +++ b/Source/Ingest/docker/Dockerfile.ingest @@ -1,7 +1,14 @@ FROM python:3.12-slim # Install curl for UV installation -RUN apt-get update && apt-get install -y curl +RUN apt-get update && apt-get install -y \ + curl \ + libhdf5-dev \ + libnetcdf-dev \ + pkg-config \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* # Install UV properly by copying from the official image COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/ diff --git a/Source/RnR/docker/Dockerfile.troute b/Source/RnR/docker/Dockerfile.troute index 181a5ce00..e31797012 100644 --- a/Source/RnR/docker/Dockerfile.troute +++ b/Source/RnR/docker/Dockerfile.troute @@ -1,11 +1,20 @@ FROM rockylinux:9 # Install system dependencies -RUN yum update -y -RUN yum install -y epel-release -RUN yum install -y gcc -RUN yum install -y netcdf netcdf-fortran netcdf-fortran-devel netcdf-openmpi -RUN yum install -y git cmake +RUN yum update -y && \ + yum install -y epel-release && \ + yum install -y \ + gcc \ + gcc-c++ \ + netcdf \ + netcdf-devel \ + netcdf-fortran \ + netcdf-fortran-devel \ + hdf5-devel \ + pkg-config \ + git \ + cmake && \ + yum clean all # Install UV by copying the binaries directly from the official image COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/ From 060820a06215640598a0acca7412c17dbecca6be Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 14:14:38 -0400 Subject: [PATCH 06/12] patch: more path problems --- .github/workflows/cicd.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index e5f454178..644ae05a4 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -62,7 +62,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./Source/Ingest - file: docker/Dockerfile.ingest + file: ./Source/Ingest/docker/Dockerfile.ingest # Load the image to the local Docker daemon, but do not push it load: true tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} @@ -92,7 +92,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./Source/RnR - file: docker/Dockerfile.troute + file: ./Source/RnR/docker/Dockerfile.troute load: true tags: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} - name: Scan RnR container with Trivy @@ -123,12 +123,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - python-version: ${{ env.PYTHON_VERSION }} - cache-dependency-glob: "**/uv.lock **/pyproject.toml" + # - name: Install uv + # uses: astral-sh/setup-uv@v5 + # with: + # enable-cache: true + # python-version: ${{ env.PYTHON_VERSION }} + # cache-dependency-glob: "**/uv.lock **/pyproject.toml" - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: @@ -188,7 +188,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./Source/Ingest - file: docker/Dockerfile.ingest + file: ./Source/Ingest/docker/Dockerfile.ingest push: true tags: ${{ steps.prep_tags.outputs.tags }} @@ -233,6 +233,6 @@ jobs: uses: docker/build-push-action@v6 with: context: ./Source/RnR - file: docker/Dockerfile.troute + file: ./Source/RnR/docker/Dockerfile.troute push: true tags: ${{ steps.prep_tags.outputs.tags }} From d12d383ffebf0ba7ad1ae7804f0cc0fced5f5969 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 8 Sep 2025 14:17:54 -0400 Subject: [PATCH 07/12] patch: fixed t-route dependencies in Dockerfile --- Source/RnR/docker/Dockerfile.troute | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Source/RnR/docker/Dockerfile.troute b/Source/RnR/docker/Dockerfile.troute index e31797012..181a5ce00 100644 --- a/Source/RnR/docker/Dockerfile.troute +++ b/Source/RnR/docker/Dockerfile.troute @@ -1,20 +1,11 @@ FROM rockylinux:9 # Install system dependencies -RUN yum update -y && \ - yum install -y epel-release && \ - yum install -y \ - gcc \ - gcc-c++ \ - netcdf \ - netcdf-devel \ - netcdf-fortran \ - netcdf-fortran-devel \ - hdf5-devel \ - pkg-config \ - git \ - cmake && \ - yum clean all +RUN yum update -y +RUN yum install -y epel-release +RUN yum install -y gcc +RUN yum install -y netcdf netcdf-fortran netcdf-fortran-devel netcdf-openmpi +RUN yum install -y git cmake # Install UV by copying the binaries directly from the official image COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/ From 6585289ada26f8312170a35f5463e5351da03ab0 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Tue, 16 Sep 2025 14:42:51 -0400 Subject: [PATCH 08/12] patch: changed ingestion to be synchronous --- Source/Ingest/pyproject.toml | 18 +---- Source/Ingest/scripts/read_hml.py | 4 +- Source/Ingest/src/hml_reader/publish.py | 95 +++++++++++++----------- Source/Ingest/src/hml_reader/settings.py | 18 +++-- 4 files changed, 67 insertions(+), 68 deletions(-) diff --git a/Source/Ingest/pyproject.toml b/Source/Ingest/pyproject.toml index eda35a150..6c11623ee 100644 --- a/Source/Ingest/pyproject.toml +++ b/Source/Ingest/pyproject.toml @@ -8,26 +8,16 @@ authors = [ dependencies = [ "pydantic==2.7.1", "httpx==0.27.0", - "geopandas==1.0.1", - "numpy==1.26.3", - "pandas==2.2.2", - "pyarrow==15.0.0", - "xarray==2024.1.1", - "isort==4.3.21", - "aio-pika==9.4.3", + "pika==1.3.2", "pydantic-settings==2.3.4", + "aioboto3==15.1.0", + "aio-pika==9.4.3", "redis==5.0.7", - "pytest==8.3.2", - "pytest-asyncio==0.23.8", - "matplotlib==3.9.0", - "netcdf4==1.6.5", - "jinja2==3.1.4", "rdflib==7.1.3", "tqdm==4.64.1", "xmltodict==0.12.0", ] -readme = "README.md" -requires-python = ">= 3.10" +requires-python = ">= 3.10, < 3.13" [build-system] requires = ["hatchling"] diff --git a/Source/Ingest/scripts/read_hml.py b/Source/Ingest/scripts/read_hml.py index 5491e01a2..04cc2431c 100644 --- a/Source/Ingest/scripts/read_hml.py +++ b/Source/Ingest/scripts/read_hml.py @@ -1,6 +1,4 @@ -import asyncio - from hml_reader import fetch_data if __name__ == "__main__": - asyncio.run(fetch_data()) + fetch_data() diff --git a/Source/Ingest/src/hml_reader/publish.py b/Source/Ingest/src/hml_reader/publish.py index a95679c89..9734856c6 100644 --- a/Source/Ingest/src/hml_reader/publish.py +++ b/Source/Ingest/src/hml_reader/publish.py @@ -3,10 +3,11 @@ import json from typing import Any -import aio_pika +import pika import httpx import redis import redis.exceptions +from pika.exceptions import AMQPConnectionError, UnroutableError from rdflib import Graph from tqdm import tqdm import xmltodict @@ -56,55 +57,59 @@ def fetch_weather_products(headers) -> list[Any]: raise httpx.HTTPError(f"Error fetching data: {response.status_code}") -async def publish(channel: aio_pika.channel, hml: HML) -> None: +def publish(channel, hml, settings) -> None: if not channel: raise RuntimeError( "Message could not be sent as there is no RabbitMQ Connection" ) - async with channel.transaction(): - msg = hml.json().encode() - try: - await channel.default_exchange.publish( - aio_pika.Message(body=msg), - routing_key=settings.flooded_data_queue, - mandatory=True - ) - except aio_pika.exceptions.DeliveryError as e: - raise e("Message rejected") + + msg = hml.model_dump_json().encode() + try: + channel.basic_publish( + exchange='', + routing_key=settings.flooded_data_queue, + body=msg, + properties=pika.BasicProperties( + delivery_mode=2, + ), + mandatory=True + ) + except UnroutableError as e: + raise RuntimeError("Message rejected") from e -async def fetch_data() -> None: - connection = await aio_pika.connect_robust( - settings.aio_pika_url, - heartbeat=30 - ) - - async with connection: - channel = await connection.channel(publisher_confirms=False) - await channel.declare_queue( - settings.flooded_data_queue, +def fetch_data() -> None: + try: + connection = pika.BlockingConnection(settings.connection_params) + channel = connection.channel() + channel.queue_declare( + queue=settings.flooded_data_queue, durable=True ) - print("Successfully connected to RabbitMQ") - headers = { - 'Accept': 'application/ld+json', - 'User-Agent': '(water.noaa.gov, user@rtx.com)' - } - hml_data = fetch_weather_products(headers) - try: - r = redis.Redis( - host=settings.redis_url, - port=settings.redis_port, - decode_responses=True - ) - hml_data = sorted(hml_data, key=lambda x: datetime.fromisoformat(x["issuanceTime"])) - for hml in tqdm(hml_data, desc="reading through api.weather.gov HML outputs"): - hml_id = hml["id"] - if r.get(hml_id) is None: - hml_obj = HML(**hml) - await publish(channel, hml_obj) - r.set(hml_id, hml_obj.json()) - r.expire(hml_id, 604800) # exires after a week - except redis.exceptions.ConnectionError as e: - raise e("Cannot run Redis service") - \ No newline at end of file + except AMQPConnectionError as e: + print(f"RabbitMQ connection error: {e}") + raise RuntimeError("Cannot connect to RabbitMQ service") from e + print("Successfully connected to RabbitMQ") + headers = { + 'Accept': 'application/ld+json', + 'User-Agent': '(water.noaa.gov, user@rtx.com)' + } + hml_data = fetch_weather_products(headers) + try: + r = redis.Redis( + host=settings.redis_url, + port=settings.redis_port, + decode_responses=True + ) + hml_data = sorted(hml_data, key=lambda x: datetime.fromisoformat(x["issuanceTime"])) + for hml in tqdm(hml_data, desc="reading through api.weather.gov HML outputs"): + hml_id = hml["id"] + if r.get(hml_id) is None: + hml_obj = HML(**hml) + publish(channel, hml_obj, settings) + r.set(hml_id, hml_obj.model_dump_json()) + r.expire(hml_id, 604800) # exires after a week + except redis.exceptions.ConnectionError as e: + raise e("Cannot run Redis service") + connection.close() + \ No newline at end of file diff --git a/Source/Ingest/src/hml_reader/settings.py b/Source/Ingest/src/hml_reader/settings.py index e6313d089..4230cdd62 100644 --- a/Source/Ingest/src/hml_reader/settings.py +++ b/Source/Ingest/src/hml_reader/settings.py @@ -1,6 +1,7 @@ import os +import pika from pydantic import ConfigDict from pydantic_settings import BaseSettings @@ -21,7 +22,6 @@ class Settings(BaseSettings): rabbitmq_default_host: str = "localhost" rabbitmq_default_port: int = 5672 - aio_pika_url: str = "amqp://{}:{}@{}:{}/" redis_url: str = "localhost" redis_port: int = 6379 @@ -39,9 +39,15 @@ def __init__(self, **data): if os.getenv("REDIS_HOST") is not None: self.redis_url = os.getenv("REDIS_HOST") - self.aio_pika_url = self.aio_pika_url.format( - self.rabbitmq_default_username, - self.rabbitmq_default_password, - self.rabbitmq_default_host, - self.rabbitmq_default_port, + creds = pika.PlainCredentials( + self.rabbitmq_default_username, + self.rabbitmq_default_password + ) + + self.connection_params = pika.ConnectionParameters( + host=self.rabbitmq_default_host, + port=self.rabbitmq_default_port, + credentials=creds, + heartbeat=30, + blocked_connection_timeout=300, ) From 5d4f40bb9800ce842c20d1bd5c6783abc14bfae7 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 22 Sep 2025 11:25:03 -0400 Subject: [PATCH 09/12] fix: added new references to venv --- Source/RnR/docker/Dockerfile.troute | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/RnR/docker/Dockerfile.troute b/Source/RnR/docker/Dockerfile.troute index 181a5ce00..a2c296a45 100644 --- a/Source/RnR/docker/Dockerfile.troute +++ b/Source/RnR/docker/Dockerfile.troute @@ -21,6 +21,7 @@ RUN ln -s /usr/lib64/gfortran/modules/netcdf.mod /usr/include/openmpi-x86_64/net # # Create venv and set environment to use it RUN uv venv --python 3.11 ENV PATH="/app/.venv/bin:$PATH" +ENV UV_PROJECT_ENVIRONMENT="/t-route/.venv" # # Install the main package in development mode RUN uv pip install -e . From 45e675faed2fbd93af98e6915fa195fb41e961e0 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 22 Sep 2025 11:37:01 -0400 Subject: [PATCH 10/12] Remove corrupted submodule Source/iac --- .gitmodules | 3 --- Source/iac | 1 - 2 files changed, 4 deletions(-) delete mode 160000 Source/iac diff --git a/.gitmodules b/.gitmodules index e69a06ec0..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "Source/iac"] - path = Source/iac - url = https://github.com/NGWPC/rnr-aws diff --git a/Source/iac b/Source/iac deleted file mode 160000 index db395c551..000000000 --- a/Source/iac +++ /dev/null @@ -1 +0,0 @@ -Subproject commit db395c55147012df9279fe97bd68d6ffa0af1ea8 From 78796003146b68d019ad57bd2a98fc9c2f56d8ed Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 22 Sep 2025 11:55:33 -0400 Subject: [PATCH 11/12] chore: updated IaC reference --- .gitmodules | 3 +++ Source/iac | 1 + 2 files changed, 4 insertions(+) create mode 160000 Source/iac diff --git a/.gitmodules b/.gitmodules index e69de29bb..e69a06ec0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Source/iac"] + path = Source/iac + url = https://github.com/NGWPC/rnr-aws diff --git a/Source/iac b/Source/iac new file mode 160000 index 000000000..7a75af2c6 --- /dev/null +++ b/Source/iac @@ -0,0 +1 @@ +Subproject commit 7a75af2c642049b76a0bf450bd0834dcc2cc5e8f From 75579126a0e025793a5c44f7afd5def74a032b4b Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Mon, 29 Sep 2025 11:31:04 -0400 Subject: [PATCH 12/12] patch: update submodule reference --- Source/iac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/iac b/Source/iac index 7a75af2c6..bc96781e7 160000 --- a/Source/iac +++ b/Source/iac @@ -1 +1 @@ -Subproject commit 7a75af2c642049b76a0bf450bd0834dcc2cc5e8f +Subproject commit bc96781e704cbaaee8f6b031bc5faacf399ec2bb