diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 000000000..644ae05a4 --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,238 @@ +name: CI/CD Pipeline + +on: + pull_request: + branches: [main, development, pi-7-NGWPC-6258] + push: + branches: [main, development] + tags: ['v*.*.*'] + +permissions: + contents: read + packages: write + security-events: write + +env: + REGISTRY: ghcr.io + PYTHON_VERSION: '3.13' + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + image_base: ${{ steps.vars.outputs.image_base }} + pr_tag: ${{ steps.vars.outputs.pr_tag }} + commit_sha: ${{ steps.vars.outputs.commit_sha }} + commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} + test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + steps: + - name: Compute image vars + id: vars + shell: bash + run: | + set -euo pipefail + ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + REPO="$(basename "${GITHUB_REPOSITORY}")" + IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + PR_NUM="${{ github.event.pull_request.number }}" + PR_TAG="pr-${PR_NUM}-build" + echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + fi + + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + COMMIT_SHA="${GITHUB_SHA}" + SHORT_SHA="${COMMIT_SHA:0:12}" + echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + fi + + build-and-scan-ingest: + name: Build and Scan Ingest Container + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + - name: Build ingest image for scanning + id: build-ingest + uses: docker/build-push-action@v6 + with: + context: ./Source/Ingest + file: ./Source/Ingest/docker/Dockerfile.ingest + # Load the image to the local Docker daemon, but do not push it + load: true + tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + - name: Scan ingest container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + # Scan the locally available image + image-ref: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-ingest.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Ingest Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-ingest.sarif' + category: 'ingest-container' + + build-and-scan-rnr: + name: Build and Scan RnR Container + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + - name: Build RnR image for scanning + id: build-rnr + uses: docker/build-push-action@v6 + with: + context: ./Source/RnR + file: ./Source/RnR/docker/Dockerfile.troute + load: true + tags: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} + - name: Scan RnR container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + image-ref: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-rnr.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload RnR Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-rnr.sarif' + category: 'rnr-container' + + codeql-scan: + name: CodeQL Scan + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + # - name: Install uv + # uses: astral-sh/setup-uv@v5 + # with: + # enable-cache: true + # python-version: ${{ env.PYTHON_VERSION }} + # cache-dependency-glob: "**/uv.lock **/pyproject.toml" + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + # Commenting out as binary/wheels are missing + # - name: Install Ingest dependencies + # run: | + # if [ -f "Source/Ingest/pyproject.toml" ]; then + # cd Source/Ingest && uv sync + # fi + # - name: Install RnR dependencies + # run: | + # if [ -f "Source/RnR/pyproject.toml" ]; then + # cd Source/RnR && uv sync + # fi + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + publish-ingest: + name: Publish Ingest to Registry + if: > + github.event_name == 'push' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/development' || + startsWith(github.ref, 'refs/tags/v') + ) + runs-on: ubuntu-latest + needs: [setup, build-and-scan-ingest, codeql-scan] + steps: + - uses: actions/checkout@v4 + - name: Prepare ingest image tags + id: prep_tags + run: | + # Always start with the unique commit SHA tag for traceability + TAGS="${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.commit_sha_short }}" + + # If it's a push to the main branch, also add the 'latest' tag + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:latest" + fi + + # If the trigger was a version tag, add that version as a tag + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + # github.ref_name holds the tag name (e.g., "v1.0.0") + VERSION_TAG=${{ github.ref_name }} + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:${VERSION_TAG}" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push ingest image + uses: docker/build-push-action@v6 + with: + context: ./Source/Ingest + file: ./Source/Ingest/docker/Dockerfile.ingest + push: true + tags: ${{ steps.prep_tags.outputs.tags }} + + publish-rnr: + name: Publish RnR to Registry + if: > + github.event_name == 'push' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/development' || + startsWith(github.ref, 'refs/tags/v') + ) + runs-on: ubuntu-latest + needs: [setup, build-and-scan-rnr, codeql-scan] + steps: + - uses: actions/checkout@v4 + - name: Prepare RnR image tags + id: prep_tags + run: | + # Always start with the unique commit SHA tag for traceability + TAGS="${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.commit_sha_short }}" + + # If it's a push to the main branch, also add the 'latest' tag + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:latest" + fi + + # If the trigger was a version tag, add that version as a tag + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + # github.ref_name holds the tag name (e.g., "v1.0.0") + VERSION_TAG=${{ github.ref_name }} + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:${VERSION_TAG}" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push RnR image + uses: docker/build-push-action@v6 + with: + context: ./Source/RnR + file: ./Source/RnR/docker/Dockerfile.troute + push: true + tags: ${{ steps.prep_tags.outputs.tags }} diff --git a/Source/Ingest/docker/Dockerfile.ingest b/Source/Ingest/docker/Dockerfile.ingest index 7cd632ba5..2efcfd79c 100644 --- a/Source/Ingest/docker/Dockerfile.ingest +++ b/Source/Ingest/docker/Dockerfile.ingest @@ -1,7 +1,14 @@ FROM python:3.12-slim # Install curl for UV installation -RUN apt-get update && apt-get install -y curl +RUN apt-get update && apt-get install -y \ + curl \ + libhdf5-dev \ + libnetcdf-dev \ + pkg-config \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* # Install UV properly by copying from the official image COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/ diff --git a/Source/Ingest/pyproject.toml b/Source/Ingest/pyproject.toml index eda35a150..6c11623ee 100644 --- a/Source/Ingest/pyproject.toml +++ b/Source/Ingest/pyproject.toml @@ -8,26 +8,16 @@ authors = [ dependencies = [ "pydantic==2.7.1", "httpx==0.27.0", - "geopandas==1.0.1", - "numpy==1.26.3", - "pandas==2.2.2", - "pyarrow==15.0.0", - "xarray==2024.1.1", - "isort==4.3.21", - "aio-pika==9.4.3", + "pika==1.3.2", "pydantic-settings==2.3.4", + "aioboto3==15.1.0", + "aio-pika==9.4.3", "redis==5.0.7", - "pytest==8.3.2", - "pytest-asyncio==0.23.8", - "matplotlib==3.9.0", - "netcdf4==1.6.5", - "jinja2==3.1.4", "rdflib==7.1.3", "tqdm==4.64.1", "xmltodict==0.12.0", ] -readme = "README.md" -requires-python = ">= 3.10" +requires-python = ">= 3.10, < 3.13" [build-system] requires = ["hatchling"] diff --git a/Source/Ingest/scripts/read_hml.py b/Source/Ingest/scripts/read_hml.py index 5491e01a2..04cc2431c 100644 --- a/Source/Ingest/scripts/read_hml.py +++ b/Source/Ingest/scripts/read_hml.py @@ -1,6 +1,4 @@ -import asyncio - from hml_reader import fetch_data if __name__ == "__main__": - asyncio.run(fetch_data()) + fetch_data() diff --git a/Source/Ingest/src/hml_reader/publish.py b/Source/Ingest/src/hml_reader/publish.py index a95679c89..9734856c6 100644 --- a/Source/Ingest/src/hml_reader/publish.py +++ b/Source/Ingest/src/hml_reader/publish.py @@ -3,10 +3,11 @@ import json from typing import Any -import aio_pika +import pika import httpx import redis import redis.exceptions +from pika.exceptions import AMQPConnectionError, UnroutableError from rdflib import Graph from tqdm import tqdm import xmltodict @@ -56,55 +57,59 @@ def fetch_weather_products(headers) -> list[Any]: raise httpx.HTTPError(f"Error fetching data: {response.status_code}") -async def publish(channel: aio_pika.channel, hml: HML) -> None: +def publish(channel, hml, settings) -> None: if not channel: raise RuntimeError( "Message could not be sent as there is no RabbitMQ Connection" ) - async with channel.transaction(): - msg = hml.json().encode() - try: - await channel.default_exchange.publish( - aio_pika.Message(body=msg), - routing_key=settings.flooded_data_queue, - mandatory=True - ) - except aio_pika.exceptions.DeliveryError as e: - raise e("Message rejected") + + msg = hml.model_dump_json().encode() + try: + channel.basic_publish( + exchange='', + routing_key=settings.flooded_data_queue, + body=msg, + properties=pika.BasicProperties( + delivery_mode=2, + ), + mandatory=True + ) + except UnroutableError as e: + raise RuntimeError("Message rejected") from e -async def fetch_data() -> None: - connection = await aio_pika.connect_robust( - settings.aio_pika_url, - heartbeat=30 - ) - - async with connection: - channel = await connection.channel(publisher_confirms=False) - await channel.declare_queue( - settings.flooded_data_queue, +def fetch_data() -> None: + try: + connection = pika.BlockingConnection(settings.connection_params) + channel = connection.channel() + channel.queue_declare( + queue=settings.flooded_data_queue, durable=True ) - print("Successfully connected to RabbitMQ") - headers = { - 'Accept': 'application/ld+json', - 'User-Agent': '(water.noaa.gov, user@rtx.com)' - } - hml_data = fetch_weather_products(headers) - try: - r = redis.Redis( - host=settings.redis_url, - port=settings.redis_port, - decode_responses=True - ) - hml_data = sorted(hml_data, key=lambda x: datetime.fromisoformat(x["issuanceTime"])) - for hml in tqdm(hml_data, desc="reading through api.weather.gov HML outputs"): - hml_id = hml["id"] - if r.get(hml_id) is None: - hml_obj = HML(**hml) - await publish(channel, hml_obj) - r.set(hml_id, hml_obj.json()) - r.expire(hml_id, 604800) # exires after a week - except redis.exceptions.ConnectionError as e: - raise e("Cannot run Redis service") - \ No newline at end of file + except AMQPConnectionError as e: + print(f"RabbitMQ connection error: {e}") + raise RuntimeError("Cannot connect to RabbitMQ service") from e + print("Successfully connected to RabbitMQ") + headers = { + 'Accept': 'application/ld+json', + 'User-Agent': '(water.noaa.gov, user@rtx.com)' + } + hml_data = fetch_weather_products(headers) + try: + r = redis.Redis( + host=settings.redis_url, + port=settings.redis_port, + decode_responses=True + ) + hml_data = sorted(hml_data, key=lambda x: datetime.fromisoformat(x["issuanceTime"])) + for hml in tqdm(hml_data, desc="reading through api.weather.gov HML outputs"): + hml_id = hml["id"] + if r.get(hml_id) is None: + hml_obj = HML(**hml) + publish(channel, hml_obj, settings) + r.set(hml_id, hml_obj.model_dump_json()) + r.expire(hml_id, 604800) # exires after a week + except redis.exceptions.ConnectionError as e: + raise e("Cannot run Redis service") + connection.close() + \ No newline at end of file diff --git a/Source/Ingest/src/hml_reader/settings.py b/Source/Ingest/src/hml_reader/settings.py index e6313d089..4230cdd62 100644 --- a/Source/Ingest/src/hml_reader/settings.py +++ b/Source/Ingest/src/hml_reader/settings.py @@ -1,6 +1,7 @@ import os +import pika from pydantic import ConfigDict from pydantic_settings import BaseSettings @@ -21,7 +22,6 @@ class Settings(BaseSettings): rabbitmq_default_host: str = "localhost" rabbitmq_default_port: int = 5672 - aio_pika_url: str = "amqp://{}:{}@{}:{}/" redis_url: str = "localhost" redis_port: int = 6379 @@ -39,9 +39,15 @@ def __init__(self, **data): if os.getenv("REDIS_HOST") is not None: self.redis_url = os.getenv("REDIS_HOST") - self.aio_pika_url = self.aio_pika_url.format( - self.rabbitmq_default_username, - self.rabbitmq_default_password, - self.rabbitmq_default_host, - self.rabbitmq_default_port, + creds = pika.PlainCredentials( + self.rabbitmq_default_username, + self.rabbitmq_default_password + ) + + self.connection_params = pika.ConnectionParameters( + host=self.rabbitmq_default_host, + port=self.rabbitmq_default_port, + credentials=creds, + heartbeat=30, + blocked_connection_timeout=300, ) diff --git a/Source/RnR/docker/Dockerfile.troute b/Source/RnR/docker/Dockerfile.troute index 181a5ce00..a2c296a45 100644 --- a/Source/RnR/docker/Dockerfile.troute +++ b/Source/RnR/docker/Dockerfile.troute @@ -21,6 +21,7 @@ RUN ln -s /usr/lib64/gfortran/modules/netcdf.mod /usr/include/openmpi-x86_64/net # # Create venv and set environment to use it RUN uv venv --python 3.11 ENV PATH="/app/.venv/bin:$PATH" +ENV UV_PROJECT_ENVIRONMENT="/t-route/.venv" # # Install the main package in development mode RUN uv pip install -e . diff --git a/Source/iac b/Source/iac index db395c551..bc96781e7 160000 --- a/Source/iac +++ b/Source/iac @@ -1 +1 @@ -Subproject commit db395c55147012df9279fe97bd68d6ffa0af1ea8 +Subproject commit bc96781e704cbaaee8f6b031bc5faacf399ec2bb