PR on develop - Finalising and checking before production

.github/workflows/acceptance_test_cd.yml

@@ -4,32 +4,33 @@ on:
   # Manual trigger
   workflow_dispatch:
 
-  # Run the test when the new PR to develop is created
-  # pull_request:
-  #   branches:
-  #     - develop
-  #   paths:
-  #     - 'backend/**'
-  #     - 'frontend/**'
-  #     - 'k8s/staging/**'
-  #     - 'infrastructure/staging/**'
-  #     - '.github/workflows/*staging*.yml'
-
-  workflow_call:
-    inputs:
-      frontend_url:
-        required: true
-        type: string
-      product_service_url:
-        required: true
-        type: string
+  push:
+    branches:
+      - "feature/*staging*"
+      - "fix/*staging*"
+    paths: 
+      - "playwright-python/**"
+      - ".github/workflows/*acceptance*.yml"
+
+  # Run the test when the new PR to develop or main is created
+  pull_request:
+    branches:
+      - develop
+      - main
+    paths:
+      - 'backend/**'
+      - 'frontend/**'
+      - 'k8s/staging/**'
+      - 'infrastructure/staging/**'
+      - '.github/workflows/*staging*.yml'
 
 env:
   PYTHON_VERSION: "3.10"
+  FRONTEND_URL: http://localhost:3000
+  USERS_SERVICE_URL: http://localhost:5000
+  NOTES_SERVICE_URL: http://localhost:5001
 
 jobs:
-  # Test Individual Services (Already triggered on feature_test workflows)
-
   # Acceptance Tests (End-to-End)
   acceptance-tests:
     name: Acceptance Tests - End-to-end user flow
@@ -52,23 +53,25 @@ jobs:
       - name: Wait for services to be ready
         run: |
           echo "Waiting for services to start..."
-          timeout 60 bash -c 'until curl -s http://localhost:5000/health > /dev/null; do sleep 2; done'
-          timeout 60 bash -c 'until curl -s http://localhost:5001/health > /dev/null; do sleep 2; done'
-          timeout 60 bash -c 'until curl -s http://localhost:3000 > /dev/null; do sleep 2; done'
+          timeout 60 bash -c 'until curl -s ${{ env.USERS_SERVICE_URL }}/health > /dev/null; do sleep 2; done'
+          timeout 60 bash -c 'until curl -s ${{ env.NOTES_SERVICE_URL }}/health > /dev/null; do sleep 2; done'
+          timeout 60 bash -c 'until curl -s ${{ env.FRONTEND_URL }} > /dev/null; do sleep 2; done'
           echo "Services are ready!"
 
       - name: Install Playwright
         run: |
-          cd ./playwright-python
           echo "Installing Playwright..."
           pip install pytest-playwright
           playwright install
-          pip install -r requirements.txt
+          pip install -r ./playwright-python/requirements.txt
 
       - name: Run acceptance tests
+        env:
+          FRONTEND_URL: ${{ env.FRONTEND_URL }}
+          USERS_SERVICE_URL: ${{ env.USERS_SERVICE_URL }}
+          NOTES_SERVICE_URL: ${{ env.NOTES_SERVICE_URL }}
         run: |
           echo "Runing acceptance tests with Playwright..."
-          cd ./playwright-python
           pytest ./playwright-python/tests/test_acceptance.py -v
 
       - name: Stop services

.github/workflows/cd-staging-deploy.yml

@@ -5,10 +5,11 @@ on:
   workflow_dispatch:
 
 
-  # Run the workflow when the new PR to develop is approved and merged
+  # Run the workflow when the new PR to develop or main is approved and merged
   push:
     branches:
       - develop
+      - main
     paths:
       - "backend/**"
       - "frontend/**"
@@ -136,27 +137,27 @@ jobs:
         run: |
           echo "Setting up infrastructure with OpenTofu"
 
-      # - name: Setup OpenTofu
-      #   uses: opentofu/setup-opentofu@v1
-      #   with:
-      #     tofu_version: '1.6.0'
+      - name: Setup OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: '1.6.0'
 
-      # - name: Log in to Azure
-      #   uses: azure/login@v1
-      #   with:
-      #     creds: {{ secrets.AZURE_CREDENTIALS }}
+      - name: Log in to Azure
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
 
-      # - name: OpenTofu Init
-      #   run: tofu init
+      - name: OpenTofu Init
+        run: tofu init
 
-      # - name: OpenTofu Plan
-      #   run: |
-      #     tofu plan \
-      #       -var="git_sha={{ github.sha }}" \
-      #       -out=staging.tfplan
+      - name: OpenTofu Plan
+        run: |
+          tofu plan \
+            -var="git_sha={{ github.sha }}" \
+            -out=staging.tfplan
 
-      # - name: OpenTofu Apply
-      #   run: tofu apply -auto-approve staging.tfplan
+      - name: OpenTofu Apply
+        run: tofu apply -auto-approve staging.tfplan
 
   # Deploy services to staging AKS
   deploy-to-staging:
@@ -241,6 +242,10 @@ jobs:
           echo "Updating image tag in deployment manifest..."
           sed -i "s|_IMAGE_NAME_WITH_TAG_|${{ env.SHARED_ACR_LOGIN_SERVER }}/${{ needs.build-images.outputs.FRONTEND_IMAGE }}|g" k8s/staging/frontend-deployment.yaml
 
+          # Student Subscription only allow 2 public IP address, so as a demo, I remove the notes service
+          kubectl delete -f k8s/staging/notes-service-deployment.yaml
+
+          # Apply frontend deployment
           echo "Deploying frontend to AKS..."
           kubectl apply -f k8s/staging/frontend-deployment.yaml
 
@@ -317,18 +322,20 @@ jobs:
     steps:   
       - name: Checkout repository
         uses: actions/checkout@v4
-
-      - name: OpenTofu Init
-        run: |
-          echo "Init OpenTofu..."
-
-      - name: OpenTofu Destroy
-        run: |
-          echo "Destroying staging infrastructure..."
 
-      - name: Deployment summary
-        if: success()
+      - name: Log in to Azure
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+          enable-AzPSSession: true
+
+      - name: Delete staging environment
         run: |
-          echo "Staging deployment successful!"
-          echo "Smoke tests passed!"
-          echo "Staging environment cleaned up!"
+          az group delete \
+            --name ${{ env.RESOURCE_GROUP_STAGING }} \
+            --yes \
+            --no-wait
+
+      - name: Logout from Azure
+        run: az logout
+

.github/workflows/feature_test_notes_service.yml

@@ -14,9 +14,10 @@ on:
       - ".github/workflows/*notes_service*.yml"
 
   # Re-run the test when the new PR to develop is created
-  # pull_request:
-  #   branches:
-  #     - "develop"
+  pull_request:
+    branches:
+      - develop
+      - main
 
 jobs:
   quality-checks:

.github/workflows/feature_test_users_service.yml

@@ -14,9 +14,10 @@ on:
       - ".github/workflows/*users_service*.yml"
 
   # Re-run the test when the new PR to develop is created
-  # pull_request:
-  #   branches:
-  #     - "develop"
+  pull_request:
+    branches:
+      - develop
+      - main
 
 jobs:
   quality-checks:

README.md

@@ -1,3 +1,42 @@
 # Microsoft Azure - Project with DevOps Feature
 
-This project is a part of HD task for SIT722 - Software Deployment and Operations, focusing on learning DevOps Cycle and pipelines
+This project is a part of HD task for SIT722 - Software Deployment and Operations, focusing on learning DevOps Cycle and pipelines
+
+## Setup
+
+To run this CI/CD project, we must initialize some existing resource (as in real production, these resource always available)
+- Initialize shared infrastructure, refer to section [Shared Azure Resource](#shared-existing)
+- Initialize production infrastructure, refer to section [Production Azure Resource](#production-existing)
+
+
+## Azure Infrastructure and Resources
+### Staging (Dynamic and Automation) 
+The staging resource can either:
+- Ephemeral environment where it is created, deploy, test and removed after the staging complete
+- Remains active as a 1-1 replica of production for manual testing and troubleshooting
+
+To reduce cost for learning purpose only, this project follows the first approach. The staging infrastructure information can be found at `infrastructure/staging`, resources include:
+- Staging resource group
+- Staging AKS, with related deployment information (Kubernetes manifest) can be found at `k8s/staging`
+
+### Shared (Existing)
+Shared resource is the existing resource on Azure, contains the resources that shared between staging and production. It is not created during CI-CD pipeline, and it requires manual review and manage since it relates to production.
+
+Shared resource setup can be found at `infrastructure/shared`, resources include:
+- Shared resource group
+- Shared container registry
+
+Commands
+```bash
+cd infrastructure/shared
+tofu init
+tofu plan
+tofu apply 
+```
+
+### Production (Existing)
+Production environment is where we deliver the product to the user, it must pass the manual approvals and should only be merge with develop branch, after all tests and check passed.
+
+The production infrastructure information can be found at `infrastructure/production`, resources include:
+- Staging resource group
+- Staging AKS, with related deployment information (Kubernetes manifest) can be found at `k8s/production`

docker-compose.yml

@@ -1,6 +1,18 @@
 version: '3.8'
 
 services:
+  postgres-notes:
+    image: postgres:15-alpine
+    container_name: postgres-notes
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=notes
+    ports:
+      - "5532:5432"
+    volumes:
+      - notes_db_data:/var/lib/postgresql/data
+
   notes-service:
     build: ./backend/notes_service
     ports:
@@ -9,25 +21,25 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=notes
-      - POSTGRES_HOST=postgres
+      - POSTGRES_HOST=postgres-notes
       - POSTGRES_PORT=5432
     depends_on:
       - postgres-notes
     command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
     volumes:
       - ./backend/notes_service/app:/code/app
 
-  postgres-notes:
+  postgres-users:
     image: postgres:15-alpine
-    container_name: postgres-notes
+    container_name: postgres-users
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=notes
+      - POSTGRES_DB=users
     ports:
-      - "5432:5432"
+      - "5533:5432"  # Different host port to avoid conflict
     volumes:
-      - notes_db_data:/var/lib/postgresql/data
+      - users_db_data:/var/lib/postgresql/data
 
   users-service:
     build: ./backend/users_service
@@ -37,28 +49,17 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=users
-      - POSTGRES_HOST=postgres
-      - POSTGRES_PORT=5434
+      - POSTGRES_HOST=postgres-users
+      - POSTGRES_PORT=5432
     depends_on:
       - postgres-users
     command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
     volumes:
       - ./backend/users_service/app:/code/app
 
-  postgres-users:
-    image: postgres:15-alpine
-    container_name: postgres-users
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=users
-    ports:
-      - "5434:5432"  # Different host port to avoid conflict
-    volumes:
-      - users_db_data:/var/lib/postgresql/data
-
   frontend:
     build: ./frontend
+    container_name: frontend
     ports:
       - "3000:80"
     depends_on:

infrastructure/production/container_registry.tf

@@ -0,0 +1,7 @@
+# infrastructure/production/container_registry.tf
+
+# Reference the shared ACR from the shared resource group
+data "azurerm_container_registry" "shared_acr" {
+  name                = "${var.prefix}acr"
+  resource_group_name = "${var.prefix}-shared-rg"
+}