initial commit: added cicd

.github/workflows/cicd.yaml

@@ -0,0 +1,238 @@
+name: CI/CD Pipeline
+
+on:
+    pull_request:
+        branches: [main, development, pi-7-NGWPC-6258]
+    push:
+        branches: [main, development]
+        tags: ['v*.*.*']
+
+permissions:
+    contents: read
+    packages: write
+    security-events: write
+
+env:
+    REGISTRY: ghcr.io
+    PYTHON_VERSION: '3.13'
+
+jobs:
+    setup:
+        runs-on: ubuntu-latest
+        outputs:
+            image_base: ${{ steps.vars.outputs.image_base }}
+            pr_tag: ${{ steps.vars.outputs.pr_tag }}
+            commit_sha: ${{ steps.vars.outputs.commit_sha }}
+            commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
+            test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
+        steps:
+            - name: Compute image vars
+              id: vars
+              shell: bash
+              run: |
+                set -euo pipefail
+                ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
+                REPO="$(basename "${GITHUB_REPOSITORY}")"
+                IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
+                echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
+
+                if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
+                    PR_NUM="${{ github.event.pull_request.number }}"
+                    PR_TAG="pr-${PR_NUM}-build"
+                    echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
+                    echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
+                fi
+
+                if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+                    COMMIT_SHA="${GITHUB_SHA}"
+                    SHORT_SHA="${COMMIT_SHA:0:12}"
+                    echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+                    echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+                    echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+                fi
+
+    build-and-scan-ingest:
+        name: Build and Scan Ingest Container
+        runs-on: ubuntu-latest
+        needs: setup
+        steps:
+          - uses: actions/checkout@v4
+          - name: Build ingest image for scanning
+            id: build-ingest
+            uses: docker/build-push-action@v6
+            with:
+              context: ./Source/Ingest
+              file: ./Source/Ingest/docker/Dockerfile.ingest
+              # Load the image to the local Docker daemon, but do not push it
+              load: true
+              tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }}
+          - name: Scan ingest container with Trivy
+            uses: aquasecurity/trivy-action@0.20.0
+            with:
+              # Scan the locally available image
+              image-ref: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }}
+              format: 'template'
+              template: '@/contrib/sarif.tpl'
+              output: 'trivy-results-ingest.sarif'
+              severity: 'CRITICAL,HIGH'
+          - name: Upload Ingest Trivy SARIF
+            uses: github/codeql-action/upload-sarif@v3
+            with:
+              sarif_file: 'trivy-results-ingest.sarif'
+              category: 'ingest-container'
+
+    build-and-scan-rnr:
+        name: Build and Scan RnR Container
+        runs-on: ubuntu-latest
+        needs: setup
+        steps:
+          - uses: actions/checkout@v4
+          - name: Build RnR image for scanning
+            id: build-rnr
+            uses: docker/build-push-action@v6
+            with:
+              context: ./Source/RnR
+              file: ./Source/RnR/docker/Dockerfile.troute
+              load: true
+              tags: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }}
+          - name: Scan RnR container with Trivy
+            uses: aquasecurity/trivy-action@0.20.0
+            with:
+              image-ref: ${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.test_image_tag }}
+              format: 'template'
+              template: '@/contrib/sarif.tpl'
+              output: 'trivy-results-rnr.sarif'
+              severity: 'CRITICAL,HIGH'
+          - name: Upload RnR Trivy SARIF
+            uses: github/codeql-action/upload-sarif@v3
+            with:
+              sarif_file: 'trivy-results-rnr.sarif'
+              category: 'rnr-container'
+
+    codeql-scan:
+        name: CodeQL Scan
+        if: github.event_name == 'pull_request' || github.event_name == 'push'
+        runs-on: ubuntu-latest
+        permissions:
+            actions: read
+            contents: read
+            security-events: write
+        steps:
+          - uses: actions/checkout@v4
+          - name: Set up Python
+            uses: actions/setup-python@v5
+            with:
+              python-version: ${{ env.PYTHON_VERSION }}
+        #   - name: Install uv
+        #     uses: astral-sh/setup-uv@v5
+        #     with:
+        #       enable-cache: true
+        #       python-version: ${{ env.PYTHON_VERSION }}
+        #       cache-dependency-glob: "**/uv.lock **/pyproject.toml"
+          - name: Initialize CodeQL
+            uses: github/codeql-action/init@v3
+            with:
+              languages: python
+        #  Commenting out as binary/wheels are missing
+        #   - name: Install Ingest dependencies
+        #     run: |
+        #       if [ -f "Source/Ingest/pyproject.toml" ]; then
+        #         cd Source/Ingest && uv sync
+        #       fi
+        #   - name: Install RnR dependencies  
+        #     run: |
+        #       if [ -f "Source/RnR/pyproject.toml" ]; then
+        #         cd Source/RnR && uv sync
+        #       fi
+          - name: Perform CodeQL Analysis
+            uses: github/codeql-action/analyze@v3
+
+    publish-ingest:
+        name: Publish Ingest to Registry
+        if: >
+          github.event_name == 'push' && (
+            github.ref == 'refs/heads/main' ||
+            github.ref == 'refs/heads/development' ||
+            startsWith(github.ref, 'refs/tags/v')
+          )
+        runs-on: ubuntu-latest
+        needs: [setup, build-and-scan-ingest, codeql-scan]
+        steps:
+          - uses: actions/checkout@v4
+          - name: Prepare ingest image tags
+            id: prep_tags
+            run: |
+              # Always start with the unique commit SHA tag for traceability
+              TAGS="${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.commit_sha_short }}"
+
+              # If it's a push to the main branch, also add the 'latest' tag
+              if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+                TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:latest"
+              fi
+
+              # If the trigger was a version tag, add that version as a tag
+              if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+                # github.ref_name holds the tag name (e.g., "v1.0.0")
+                VERSION_TAG=${{ github.ref_name }}
+                TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:${VERSION_TAG}"
+              fi
+
+              echo "tags=${TAGS}" >> "$GITHUB_OUTPUT"
+          - name: Log in to registry
+            uses: docker/login-action@v3
+            with:
+              registry: ${{ env.REGISTRY }}
+              username: ${{ github.actor }}
+              password: ${{ secrets.GITHUB_TOKEN }}
+          - name: Build & push ingest image
+            uses: docker/build-push-action@v6
+            with:
+              context: ./Source/Ingest
+              file: ./Source/Ingest/docker/Dockerfile.ingest
+              push: true
+              tags: ${{ steps.prep_tags.outputs.tags }}
+
+    publish-rnr:
+        name: Publish RnR to Registry
+        if: >
+          github.event_name == 'push' && (
+            github.ref == 'refs/heads/main' ||
+            github.ref == 'refs/heads/development' ||
+            startsWith(github.ref, 'refs/tags/v')
+          )
+        runs-on: ubuntu-latest
+        needs: [setup, build-and-scan-rnr, codeql-scan]
+        steps:
+          - uses: actions/checkout@v4
+          - name: Prepare RnR image tags
+            id: prep_tags
+            run: |
+              # Always start with the unique commit SHA tag for traceability
+              TAGS="${{ needs.setup.outputs.image_base }}/rnr:${{ needs.setup.outputs.commit_sha_short }}"
+
+              # If it's a push to the main branch, also add the 'latest' tag
+              if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+                TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:latest"
+              fi
+
+              # If the trigger was a version tag, add that version as a tag
+              if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+                # github.ref_name holds the tag name (e.g., "v1.0.0")
+                VERSION_TAG=${{ github.ref_name }}
+                TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/rnr:${VERSION_TAG}"
+              fi
+
+              echo "tags=${TAGS}" >> "$GITHUB_OUTPUT"
+          - name: Log in to registry
+            uses: docker/login-action@v3
+            with:
+              registry: ${{ env.REGISTRY }}
+              username: ${{ github.actor }}
+              password: ${{ secrets.GITHUB_TOKEN }}
+          - name: Build & push RnR image
+            uses: docker/build-push-action@v6
+            with:
+              context: ./Source/RnR
+              file: ./Source/RnR/docker/Dockerfile.troute
+              push: true
+              tags: ${{ steps.prep_tags.outputs.tags }}

Source/Ingest/docker/Dockerfile.ingest

@@ -1,7 +1,14 @@
 FROM python:3.12-slim
 
 # Install curl for UV installation
-RUN apt-get update && apt-get install -y curl
+RUN apt-get update && apt-get install -y \
+    curl \
+    libhdf5-dev \
+    libnetcdf-dev \
+    pkg-config \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install UV properly by copying from the official image
 COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/

Source/Ingest/pyproject.toml

@@ -8,26 +8,16 @@ authors = [
 dependencies = [
     "pydantic==2.7.1",
     "httpx==0.27.0",
-    "geopandas==1.0.1",
-    "numpy==1.26.3",
-    "pandas==2.2.2",
-    "pyarrow==15.0.0",
-    "xarray==2024.1.1",
-    "isort==4.3.21",
-    "aio-pika==9.4.3",
+    "pika==1.3.2",
     "pydantic-settings==2.3.4",
+    "aioboto3==15.1.0",
+    "aio-pika==9.4.3",
     "redis==5.0.7",
-    "pytest==8.3.2",
-    "pytest-asyncio==0.23.8",
-    "matplotlib==3.9.0",
-    "netcdf4==1.6.5",
-    "jinja2==3.1.4",
     "rdflib==7.1.3",
     "tqdm==4.64.1",
     "xmltodict==0.12.0",
 ]
-readme = "README.md"
-requires-python = ">= 3.10"
+requires-python = ">= 3.10, < 3.13"
 
 [build-system]
 requires = ["hatchling"]

Source/Ingest/scripts/read_hml.py

@@ -1,6 +1,4 @@
-import asyncio
-
 from hml_reader import fetch_data
 
 if __name__ == "__main__":
-    asyncio.run(fetch_data())
+    fetch_data()