From 58e0f34f5629767ad5ccf76bb68105bad9bd6a6a Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Tue, 16 Sep 2025 14:12:36 -0400 Subject: [PATCH 001/144] initial test pipeline --- .github/workflows/test-deployer.yml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/test-deployer.yml diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml new file mode 100644 index 00000000..5a716641 --- /dev/null +++ b/.github/workflows/test-deployer.yml @@ -0,0 +1,46 @@ +name: Test Deployer + +on: + push: + branches: + - pipeline-test + +env: + TF_DIRECTORY: ./tofu_aws + ANSIBLE_HOST_KEY_CHECKING: "False" + +jobs: + test-deployer: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set design parameter + id: design + run: echo "design=aio" >> $GITHUB_OUTPUT + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: latest + + - name: Install Ansible + run: | + python -m pip install --upgrade pip + pip install ansible + ansible-galaxy collection install itential.deployer + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} From 95456c1f7f1522c4a98712c09b21ade4e57063d9 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Tue, 16 Sep 2025 14:37:07 -0400 Subject: [PATCH 002/144] add session token --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 5a716641..47a6254d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -43,4 +43,5 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} From d9235de5fcfc485fde6829eec198125e095f1604 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 14:50:33 -0400 Subject: [PATCH 003/144] add themis clone step --- .github/workflows/test-deployer.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 47a6254d..61101691 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -45,3 +45,16 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + + - name: Setup SSH for GitLab + run: | + mkdir -p ~/.ssh + echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts + + - name: Clone Themis repository + run: | + git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" ~/themis + cd ~/themis + ls -la From 83f9dc839e2a344a718d25dd74cbb4305f19001c Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 14:59:01 -0400 Subject: [PATCH 004/144] add tofu ec2 provision, hosts generation --- .github/workflows/test-deployer.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 61101691..0d1b26a8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -58,3 +58,13 @@ jobs: git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" ~/themis cd ~/themis ls -la + + - name: Provision EC2 instances for Minimal design + working-directory: ~/themis/tofu_aws + run: | + tofu init + tofu apply -var-file=tfvars/minimal.tfvars + + - name: Generate Ansible inventory hosts file + working-directory: ~/themis/tofu_aws + run: python3 ../scripts/generate_inventory.py From 1c95379fe7891de0b44ce848c4a390bc11605941 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 15:01:06 -0400 Subject: [PATCH 005/144] fix working dirs --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 0d1b26a8..685fe959 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -60,11 +60,11 @@ jobs: ls -la - name: Provision EC2 instances for Minimal design - working-directory: ~/themis/tofu_aws + working-directory: /home/runner/themis/tofu_aws run: | tofu init tofu apply -var-file=tfvars/minimal.tfvars - name: Generate Ansible inventory hosts file - working-directory: ~/themis/tofu_aws + working-directory: /home/runner/themis/tofu_aws run: python3 ../scripts/generate_inventory.py From f0a67532d688ae3a9e072d28df517b06fe611d72 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 15:06:38 -0400 Subject: [PATCH 006/144] add auto approval, aws profile name --- .github/workflows/test-deployer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 685fe959..0037d5a8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -6,6 +6,7 @@ on: - pipeline-test env: + AWS_PROFILE: ${{ secrets.AWS_PROFILE }} TF_DIRECTORY: ./tofu_aws ANSIBLE_HOST_KEY_CHECKING: "False" @@ -63,7 +64,7 @@ jobs: working-directory: /home/runner/themis/tofu_aws run: | tofu init - tofu apply -var-file=tfvars/minimal.tfvars + tofu apply -var-file=tfvars/minimal.tfvars -auto-approve - name: Generate Ansible inventory hosts file working-directory: /home/runner/themis/tofu_aws From e7623ead3f808a2ecbcd67a488f2e94a1a9afb37 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 15:09:00 -0400 Subject: [PATCH 007/144] remove AWS_PROFILE, split tofu into 2 steps --- .github/workflows/test-deployer.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 0037d5a8..080a6943 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -6,7 +6,6 @@ on: - pipeline-test env: - AWS_PROFILE: ${{ secrets.AWS_PROFILE }} TF_DIRECTORY: ./tofu_aws ANSIBLE_HOST_KEY_CHECKING: "False" @@ -60,11 +59,13 @@ jobs: cd ~/themis ls -la + - name: Initialize OpenTofu + working-directory: /home/runner/themis/tofu_aws + run: tofu init + - name: Provision EC2 instances for Minimal design working-directory: /home/runner/themis/tofu_aws - run: | - tofu init - tofu apply -var-file=tfvars/minimal.tfvars -auto-approve + run: tofu apply -var-file=tfvars/minimal.tfvars -auto-approve - name: Generate Ansible inventory hosts file working-directory: /home/runner/themis/tofu_aws From 2317441207e9b061875bc3a0a8d3552b7f3f94ef Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 16 Sep 2025 15:24:03 -0400 Subject: [PATCH 008/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 080a6943..078501cf 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -36,7 +36,9 @@ jobs: run: | python -m pip install --upgrade pip pip install ansible - ansible-galaxy collection install itential.deployer + + - name: Install this collection + run: ansible-galaxy collection install . - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -69,4 +71,15 @@ jobs: - name: Generate Ansible inventory hosts file working-directory: /home/runner/themis/tofu_aws - run: python3 ../scripts/generate_inventory.py + run: python3 ../scripts/generate_inventory.py > hosts.json + + - name: Wait for EC2 instances to be ready + working-directory: /home/runner/themis/tofu_aws + run: sleep 60s + + - name: Run deployer on created instances + run: ansible-playbook itential.deployer.site -i /home/runner/themis/tofu_aws/hosts.json -v + + - name: Tear down EC2 instances + working-directory: /home/runner/themis/tofu_aws + run: tofu destroy -var-file=tfvars/minimal.tfvars -auto-approve From bc28b6a96c72abbea649bf9f06f26a8dba32ff2d Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 17 Sep 2025 11:21:54 -0400 Subject: [PATCH 009/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 078501cf..a08089db 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -71,7 +71,13 @@ jobs: - name: Generate Ansible inventory hosts file working-directory: /home/runner/themis/tofu_aws - run: python3 ../scripts/generate_inventory.py > hosts.json + run: > + python3 ../scripts/generate_inventory.py + --platform-release 6.0 + --platform-packages itential-platform-6.0.7-1.noarch.rpm + --gateway-release 2023.1 + --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl + -o hosts.json - name: Wait for EC2 instances to be ready working-directory: /home/runner/themis/tofu_aws From 127e3c25fb95b54bfff1130acc11f83a8a79b5de Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 17 Sep 2025 11:24:06 -0400 Subject: [PATCH 010/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a08089db..23b26a86 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -71,13 +71,7 @@ jobs: - name: Generate Ansible inventory hosts file working-directory: /home/runner/themis/tofu_aws - run: > - python3 ../scripts/generate_inventory.py - --platform-release 6.0 - --platform-packages itential-platform-6.0.7-1.noarch.rpm - --gateway-release 2023.1 - --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl - -o hosts.json + run: python3 ../scripts/generate_inventory.py --platform-release 6.0 --platform-packages itential-platform-6.0.7-1.noarch.rpm --gateway-release 2023.1 --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl -o hosts.json - name: Wait for EC2 instances to be ready working-directory: /home/runner/themis/tofu_aws From 949dcb69d99e568f53be5d9648050c4637ee0be1 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 17 Sep 2025 11:37:02 -0400 Subject: [PATCH 011/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 23b26a86..df3c0789 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -48,11 +48,13 @@ jobs: aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - - name: Setup SSH for GitLab + - name: Setup SSH for GitLab and EC2 run: | mkdir -p ~/.ssh echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem chmod 600 ~/.ssh/id_rsa + chmod 600 ~/.ssh/pet-east1.open.pem ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - name: Clone Themis repository @@ -78,7 +80,9 @@ jobs: run: sleep 60s - name: Run deployer on created instances - run: ansible-playbook itential.deployer.site -i /home/runner/themis/tofu_aws/hosts.json -v + env: + ANSIBLE_HOST_KEY_CHECKING: "False" + run: ansible-playbook itential.deployer.site -i /home/runner/themis/tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem - name: Tear down EC2 instances working-directory: /home/runner/themis/tofu_aws From ad6b27a06e5124b4398e539c4dbb346ebc42abf3 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:18:31 -0400 Subject: [PATCH 012/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index df3c0789..aa14ad68 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -11,17 +11,13 @@ env: jobs: test-deployer: - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 60 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set design parameter - id: design - run: echo "design=aio" >> $GITHUB_OUTPUT - - name: Setup Python uses: actions/setup-python@v4 with: @@ -85,5 +81,6 @@ jobs: run: ansible-playbook itential.deployer.site -i /home/runner/themis/tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem - name: Tear down EC2 instances + if: always() working-directory: /home/runner/themis/tofu_aws run: tofu destroy -var-file=tfvars/minimal.tfvars -auto-approve From 1351c9e2bbd2049fd1d40b4d554e4021c7722751 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:22:00 -0400 Subject: [PATCH 013/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index aa14ad68..e35b8ec5 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -60,19 +60,19 @@ jobs: ls -la - name: Initialize OpenTofu - working-directory: /home/runner/themis/tofu_aws + working-directory: $HOME/themis/tofu_aws run: tofu init - name: Provision EC2 instances for Minimal design - working-directory: /home/runner/themis/tofu_aws + working-directory: $HOME/themis/tofu_aws run: tofu apply -var-file=tfvars/minimal.tfvars -auto-approve - name: Generate Ansible inventory hosts file - working-directory: /home/runner/themis/tofu_aws + working-directory: $HOME/themis/tofu_aws run: python3 ../scripts/generate_inventory.py --platform-release 6.0 --platform-packages itential-platform-6.0.7-1.noarch.rpm --gateway-release 2023.1 --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl -o hosts.json - name: Wait for EC2 instances to be ready - working-directory: /home/runner/themis/tofu_aws + working-directory: $HOME/themis/tofu_aws run: sleep 60s - name: Run deployer on created instances @@ -82,5 +82,5 @@ jobs: - name: Tear down EC2 instances if: always() - working-directory: /home/runner/themis/tofu_aws + working-directory: $HOME/themis/tofu_aws run: tofu destroy -var-file=tfvars/minimal.tfvars -auto-approve From 6879e3d2aecc01cba2a49789df9f50d1761ed05f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:25:28 -0400 Subject: [PATCH 014/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e35b8ec5..fecd2658 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -54,33 +54,35 @@ jobs: ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - name: Clone Themis repository + working-directory: ".." run: | - git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" ~/themis - cd ~/themis + git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" + cd themis ls -la - name: Initialize OpenTofu - working-directory: $HOME/themis/tofu_aws + working-directory: ../themis/tofu_aws run: tofu init - name: Provision EC2 instances for Minimal design - working-directory: $HOME/themis/tofu_aws + working-directory: ../themis/tofu_aws run: tofu apply -var-file=tfvars/minimal.tfvars -auto-approve - name: Generate Ansible inventory hosts file - working-directory: $HOME/themis/tofu_aws + working-directory: ../themis/tofu_aws run: python3 ../scripts/generate_inventory.py --platform-release 6.0 --platform-packages itential-platform-6.0.7-1.noarch.rpm --gateway-release 2023.1 --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl -o hosts.json - name: Wait for EC2 instances to be ready - working-directory: $HOME/themis/tofu_aws + working-directory: ../themis/tofu_aws run: sleep 60s - name: Run deployer on created instances + working-directory: ../themis/tofu_aws env: ANSIBLE_HOST_KEY_CHECKING: "False" - run: ansible-playbook itential.deployer.site -i /home/runner/themis/tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i hosts.json -v --key-file=~/.ssh/pet-east1.open.pem - name: Tear down EC2 instances if: always() - working-directory: $HOME/themis/tofu_aws + working-directory: ../themis/tofu_aws run: tofu destroy -var-file=tfvars/minimal.tfvars -auto-approve From b167d7787f243da34fa0aaee3216728c9c68f796 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:34:13 -0400 Subject: [PATCH 015/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index fecd2658..f548fb87 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -18,10 +18,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' + #- name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: '3.11' - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From d120d1d595b986ecfd0c9540fa86a1c9ef068f71 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:37:32 -0400 Subject: [PATCH 016/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index f548fb87..173f4da0 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -23,6 +23,10 @@ jobs: # with: # python-version: '3.11' + - name: Setup Python + run: | + sudo dnf install python3-pip -y + - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 with: From 2ccbbc1ccc1fc0710efd5afe565903fcc5161582 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:42:06 -0400 Subject: [PATCH 017/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 173f4da0..34887b33 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -23,8 +23,9 @@ jobs: # with: # python-version: '3.11' - - name: Setup Python + - name: Install dnf packages run: | + sudo dnf groupinstall 'Development Tools' -y sudo dnf install python3-pip -y - name: Install OpenTofu From 0f7575b76d8d2f11aa6faf3af5ba90e385ccfe0e Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:45:51 -0400 Subject: [PATCH 018/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 34887b33..95ef40dc 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -27,6 +27,7 @@ jobs: run: | sudo dnf groupinstall 'Development Tools' -y sudo dnf install python3-pip -y + sudo dnf install nodejs -y - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From fcf159a88dd44eefc78082b02a4858f29d792784 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 12:48:00 -0400 Subject: [PATCH 019/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 95ef40dc..6c035b35 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -62,6 +62,7 @@ jobs: - name: Clone Themis repository working-directory: ".." run: | + rm -rf themis git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" cd themis ls -la From 01faa2bdd79eaf147e2bb61af3834e3539059e45 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 18 Sep 2025 13:12:39 -0400 Subject: [PATCH 020/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 6c035b35..6272ad40 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -40,7 +40,15 @@ jobs: pip install ansible - name: Install this collection - run: ansible-galaxy collection install . + run: ansible-galaxy collection install . --force + + - name: Download Platform and Gateway binaries + run: | + cd ~/.ansible/collections/ansible_collections/itential/deployer/playbooks + mkdir -p files + cd files + ${{ secrets.PLATFORM_RPM_DOWNLOAD_COMMAND }} + ${{ secrets.GATEWAY_WHL_DOWNLOAD_COMMAND }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 From 8b6450caddd94c9ed7e0b0fe61273155190d3814 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 19 Sep 2025 14:04:05 -0400 Subject: [PATCH 021/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 37 +++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 6272ad40..8337eef8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -85,17 +85,44 @@ jobs: - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws - run: python3 ../scripts/generate_inventory.py --platform-release 6.0 --platform-packages itential-platform-6.0.7-1.noarch.rpm --gateway-release 2023.1 --gateway-whl-file automation_gateway-3.227.0+2023.1.9-py3-none-any.whl -o hosts.json + run: python3 ../scripts/generate_inventory.py -o hosts.json - - name: Wait for EC2 instances to be ready + - name: Create Ansible playbook to wait for EC2 instances to be ready working-directory: ../themis/tofu_aws - run: sleep 60s + run: | + cat > wait-for-instances.yml << 'EOF' + --- + - name: Wait for instances to initialize + hosts: all + gather_facts: no + become: no + serial: "{{ ansible_serial | default(10) }}" + + vars: + ssh_port: 22 + + tasks: + - name: Wait for SSH connection to be ready + ansible.builtin.wait_for_connection: + delay: 10 + timeout: 60 + + - name: Wait for cloud-init to complete (marker file check) + ansible.builtin.wait_for: + path: /var/log/cloud-init-finished.marker + timeout: 60 + become: yes + EOF + + - name: Wait for EC2 instances to be ready + working-directory: ../themis + run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/minimal -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances - working-directory: ../themis/tofu_aws + working-directory: ../themis env: ANSIBLE_HOST_KEY_CHECKING: "False" - run: ansible-playbook itential.deployer.site -i hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/minimal -v --key-file=~/.ssh/pet-east1.open.pem - name: Tear down EC2 instances if: always() From 4c6ef2df3e87622849bfccc7d75902287225b1a7 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 19 Sep 2025 14:05:57 -0400 Subject: [PATCH 022/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 8337eef8..4d074019 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -88,7 +88,7 @@ jobs: run: python3 ../scripts/generate_inventory.py -o hosts.json - name: Create Ansible playbook to wait for EC2 instances to be ready - working-directory: ../themis/tofu_aws + working-directory: ../themis run: | cat > wait-for-instances.yml << 'EOF' --- From f77963b9686c10754bb3ff015e796e94b16d5480 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 19 Sep 2025 14:24:44 -0400 Subject: [PATCH 023/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 4d074019..d05fa9ab 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -5,14 +5,14 @@ on: branches: - pipeline-test -env: - TF_DIRECTORY: ./tofu_aws - ANSIBLE_HOST_KEY_CHECKING: "False" - jobs: test-deployer: runs-on: self-hosted timeout-minutes: 60 + strategy: + matrix: + design: [aio, minimal] + fail-fast: false steps: - name: Checkout repository @@ -79,9 +79,9 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init - - name: Provision EC2 instances for Minimal design + - name: Provision EC2 instances for ${{ matrix.design }} design working-directory: ../themis/tofu_aws - run: tofu apply -var-file=tfvars/minimal.tfvars -auto-approve + run: tofu apply -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws @@ -116,15 +116,15 @@ jobs: - name: Wait for EC2 instances to be ready working-directory: ../themis - run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/minimal -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances working-directory: ../themis env: ANSIBLE_HOST_KEY_CHECKING: "False" - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/minimal -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Tear down EC2 instances if: always() working-directory: ../themis/tofu_aws - run: tofu destroy -var-file=tfvars/minimal.tfvars -auto-approve + run: tofu destroy -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve From da4fa6de78c8d112776e0fdabac727bc83219db2 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 19 Sep 2025 14:34:14 -0400 Subject: [PATCH 024/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index d05fa9ab..e23db5ac 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -116,6 +116,8 @@ jobs: - name: Wait for EC2 instances to be ready working-directory: ../themis + env: + ANSIBLE_HOST_KEY_CHECKING: "False" run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances From 2538f3e6e07c17c0c62ae04da0f09471b87e00c9 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:14:30 -0400 Subject: [PATCH 025/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e23db5ac..345883f4 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -18,16 +18,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - #- name: Setup Python - # uses: actions/setup-python@v4 - # with: - # python-version: '3.11' - - name: Install dnf packages run: | sudo dnf groupinstall 'Development Tools' -y sudo dnf install python3-pip -y sudo dnf install nodejs -y + sudo dnf install jq -y - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 @@ -126,6 +122,23 @@ jobs: ANSIBLE_HOST_KEY_CHECKING: "False" run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + - name: Verify that Platform is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py platform $host + done + for host in $(jq -r '(.all.children.platform_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py platform "http://$host:3000" + done + + - name: Verify that Gateway is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py gateway "http://$host:8083" + done + - name: Tear down EC2 instances if: always() working-directory: ../themis/tofu_aws From 0d2df58699fdfdf97a4060d06aca7e526ca9523f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:18:43 -0400 Subject: [PATCH 026/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 345883f4..0aee46cc 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -51,7 +51,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} + #aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - name: Setup SSH for GitLab and EC2 From 6a9998dd6beba625fa7dc4cb2b880c93da82433f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:19:29 -0400 Subject: [PATCH 027/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 0aee46cc..345883f4 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -51,7 +51,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - #aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} + aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - name: Setup SSH for GitLab and EC2 From ea28a34d7d9432853b284cc9daf140a1fd8aa4fe Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:40:15 -0400 Subject: [PATCH 028/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 345883f4..46903d24 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -71,6 +71,10 @@ jobs: cd themis ls -la + - name: Install Themis Python script dependencies + working-directory: ../themis + run: pip install -r scripts/requirements.txt + - name: Initialize OpenTofu working-directory: ../themis/tofu_aws run: tofu init @@ -123,16 +127,18 @@ jobs: run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly + continue-on-error: true working-directory: ../themis run: | for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do - python3 scripts/validate.py platform $host + python3 scripts/validate.py platform "http://$host:3000" done for host in $(jq -r '(.all.children.platform_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py platform "http://$host:3000" done - name: Verify that Gateway is running correctly + continue-on-error: true working-directory: ../themis run: | for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do From 1a8a315d4bb4213aad92b958bef3ec086647f072 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:47:53 -0400 Subject: [PATCH 029/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 46903d24..fe4fa82d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -21,10 +21,24 @@ jobs: - name: Install dnf packages run: | sudo dnf groupinstall 'Development Tools' -y - sudo dnf install python3-pip -y + sudo dnf install python3.12 python3.12-pip python3.12-devel -y sudo dnf install nodejs -y sudo dnf install jq -y + - name: Set Python 3.12 as default + run: | + alternatives --remove-all python3 2>/dev/null || true + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 + + alternatives --remove-all python 2>/dev/null || true + alternatives --install /usr/bin/python python /usr/bin/python3.12 100 + + alternatives --remove-all pip3 2>/dev/null || true + alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 + + alternatives --remove-all pip 2>/dev/null || true + alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 + - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 with: From 23e5fa82552676f28e0fc73d9ec1bcdf36f40c90 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 11:49:47 -0400 Subject: [PATCH 030/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index fe4fa82d..857039db 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -27,17 +27,17 @@ jobs: - name: Set Python 3.12 as default run: | - alternatives --remove-all python3 2>/dev/null || true - alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 + sudo alternatives --remove-all python3 2>/dev/null || true + sudo alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 - alternatives --remove-all python 2>/dev/null || true - alternatives --install /usr/bin/python python /usr/bin/python3.12 100 + sudo alternatives --remove-all python 2>/dev/null || true + sudo alternatives --install /usr/bin/python python /usr/bin/python3.12 100 - alternatives --remove-all pip3 2>/dev/null || true - alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 + sudo alternatives --remove-all pip3 2>/dev/null || true + sudo alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 - alternatives --remove-all pip 2>/dev/null || true - alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 + sudo alternatives --remove-all pip 2>/dev/null || true + sudo alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From a55a9ce6465e7fd11c5bf039815df2173070c48f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:03:09 -0400 Subject: [PATCH 031/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 857039db..38bc3b0a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -8,6 +8,9 @@ on: jobs: test-deployer: runs-on: self-hosted + container: + image: ubuntu:latest + options: --privileged timeout-minutes: 60 strategy: matrix: @@ -25,19 +28,19 @@ jobs: sudo dnf install nodejs -y sudo dnf install jq -y - - name: Set Python 3.12 as default - run: | - sudo alternatives --remove-all python3 2>/dev/null || true - sudo alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 - - sudo alternatives --remove-all python 2>/dev/null || true - sudo alternatives --install /usr/bin/python python /usr/bin/python3.12 100 - - sudo alternatives --remove-all pip3 2>/dev/null || true - sudo alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 - - sudo alternatives --remove-all pip 2>/dev/null || true - sudo alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 + #- name: Set Python 3.12 as default + # run: | + # sudo alternatives --remove-all python3 2>/dev/null || true + # sudo alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 + # + # sudo alternatives --remove-all python 2>/dev/null || true + # sudo alternatives --install /usr/bin/python python /usr/bin/python3.12 100 + # + # sudo alternatives --remove-all pip3 2>/dev/null || true + # sudo alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 + # + # sudo alternatives --remove-all pip 2>/dev/null || true + # sudo alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From 75ec69c9422886d82dceb63cedf304a45806c5cd Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:17:36 -0400 Subject: [PATCH 032/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 38bc3b0a..7985e5d5 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,7 +9,7 @@ jobs: test-deployer: runs-on: self-hosted container: - image: ubuntu:latest + image: ghcr.io/actions/runner:latest options: --privileged timeout-minutes: 60 strategy: From 2d0cc15f9dcc179b202f16b1fe95b7813dbda084 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:19:05 -0400 Subject: [PATCH 033/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 7985e5d5..5036ce46 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,7 +9,7 @@ jobs: test-deployer: runs-on: self-hosted container: - image: ghcr.io/actions/runner:latest + image: ubuntu:latest options: --privileged timeout-minutes: 60 strategy: @@ -21,12 +21,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install dnf packages - run: | - sudo dnf groupinstall 'Development Tools' -y - sudo dnf install python3.12 python3.12-pip python3.12-devel -y - sudo dnf install nodejs -y - sudo dnf install jq -y + #- name: Install dnf packages + # run: | + # sudo dnf groupinstall 'Development Tools' -y + # sudo dnf install python3.12 python3.12-pip python3.12-devel -y + # sudo dnf install nodejs -y + # sudo dnf install jq -y #- name: Set Python 3.12 as default # run: | From 69c36312c3cd6d21499e6ba52ec20cf9cdb5a195 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:22:49 -0400 Subject: [PATCH 034/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 5036ce46..e5903139 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,7 +9,7 @@ jobs: test-deployer: runs-on: self-hosted container: - image: ubuntu:latest + image: ghcr.io/catthehacker/ubuntu:runner-latest options: --privileged timeout-minutes: 60 strategy: From ace10acb485521598d39c919f8dac5ab6b427040 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:25:08 -0400 Subject: [PATCH 035/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e5903139..1e9923ca 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,7 +9,7 @@ jobs: test-deployer: runs-on: self-hosted container: - image: ghcr.io/catthehacker/ubuntu:runner-latest + image: ubuntu:latest options: --privileged timeout-minutes: 60 strategy: @@ -18,6 +18,9 @@ jobs: fail-fast: false steps: + - name: Install common dependencies + run: apt install -y unzip curl wget git node python3 + - name: Checkout repository uses: actions/checkout@v4 From 8cd92391bf6f0946ac5f76a8e4696f023edab7f9 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:25:43 -0400 Subject: [PATCH 036/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 1e9923ca..84c7c17a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install common dependencies - run: apt install -y unzip curl wget git node python3 + run: apt update && apt install -y unzip curl wget git node python3 - name: Checkout repository uses: actions/checkout@v4 From 89f4553f8236ebdf1cf056326385d000bff1c418 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:26:37 -0400 Subject: [PATCH 037/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 84c7c17a..bf1acec6 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install common dependencies - run: apt update && apt install -y unzip curl wget git node python3 + run: apt update && apt install -y unzip curl wget git nodejs python3 - name: Checkout repository uses: actions/checkout@v4 From fe758b8346861153480732c742cd62d4bc0e8bc3 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:29:30 -0400 Subject: [PATCH 038/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index bf1acec6..a113be21 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -19,7 +19,13 @@ jobs: steps: - name: Install common dependencies - run: apt update && apt install -y unzip curl wget git nodejs python3 + run: apt update && apt install -y unzip curl wget git nodejs + + - name: Install Python + run: | + apt install -y python3 python3-pip + ln -s /usr/bin/python3 /usr/bin/python + ln -s /usr/bin/pip3 /usr/bin/pip - name: Checkout repository uses: actions/checkout@v4 From 7d123d35ee96591fcb9c8a9e3872e3217b0fcd00 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:30:56 -0400 Subject: [PATCH 039/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a113be21..7af7cb42 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -24,8 +24,8 @@ jobs: - name: Install Python run: | apt install -y python3 python3-pip - ln -s /usr/bin/python3 /usr/bin/python - ln -s /usr/bin/pip3 /usr/bin/pip + ln -s /usr/bin/python3 /usr/bin/python || true + ln -s /usr/bin/pip3 /usr/bin/pip || true - name: Checkout repository uses: actions/checkout@v4 From d03764254507f85c35a7643c501a1e971cb09022 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:33:20 -0400 Subject: [PATCH 040/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 7af7cb42..e080e7fe 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -11,6 +11,8 @@ jobs: container: image: ubuntu:latest options: --privileged + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: matrix: From b4acc9123c26aa5b367853c9bfc5496257741997 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:35:12 -0400 Subject: [PATCH 041/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e080e7fe..f6a4a3b6 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -59,9 +59,7 @@ jobs: tofu_version: latest - name: Install Ansible - run: | - python -m pip install --upgrade pip - pip install ansible + run: pip install ansible - name: Install this collection run: ansible-galaxy collection install . --force From 886979563824631c8b257e734669ecbe25bec0c6 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:44:58 -0400 Subject: [PATCH 042/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index f6a4a3b6..96bf4f48 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -80,14 +80,25 @@ jobs: aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - - name: Setup SSH for GitLab and EC2 + #- name: Setup SSH for GitLab and EC2 + # run: | + # mkdir -p ~/.ssh + # echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + # echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + # chmod 600 ~/.ssh/id_rsa + # chmod 600 ~/.ssh/pet-east1.open.pem + # ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts + + - name: Setup SSH for GitLab + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.GITLAB_SSH_KEY }} + + - name: Setup SSH for EC2 run: | mkdir -p ~/.ssh - echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pet-east1.open.pem - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - name: Clone Themis repository working-directory: ".." From bd791fd53991ae231841afe39e47e6a3d3d35242 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:48:16 -0400 Subject: [PATCH 043/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 96bf4f48..1b04f69e 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Install common dependencies - run: apt update && apt install -y unzip curl wget git nodejs + run: apt update && apt install -y unzip curl wget git nodejs openssh-client - name: Install Python run: | @@ -80,25 +80,16 @@ jobs: aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - #- name: Setup SSH for GitLab and EC2 - # run: | - # mkdir -p ~/.ssh - # echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa - # echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - # chmod 600 ~/.ssh/id_rsa - # chmod 600 ~/.ssh/pet-east1.open.pem - # ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Setup SSH for GitLab - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_KEY }} - - - name: Setup SSH for EC2 + - name: Setup SSH for GitLab and EC2 run: | mkdir -p ~/.ssh + echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pet-east1.open.pem + ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts + eval "$(ssh-agent -s)" + ssh-add ~/.ssh/id_rsa - name: Clone Themis repository working-directory: ".." From 1413cc5dcd0e92fb272189d7b20efb92c088d6f4 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:50:38 -0400 Subject: [PATCH 044/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 1b04f69e..59acee69 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,6 +12,7 @@ jobs: image: ubuntu:latest options: --privileged env: + GIT_SSH_COMMAND: ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: From 4a80f4215d569f2d7f2626e23f76157cd9bbb35f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:52:59 -0400 Subject: [PATCH 045/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 59acee69..e7af365e 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,7 +12,7 @@ jobs: image: ubuntu:latest options: --privileged env: - GIT_SSH_COMMAND: ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: From 53cdbcbbcac91c3e2065c699a0e9ae215ecfb49a Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:56:33 -0400 Subject: [PATCH 046/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e7af365e..c37e81c9 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -29,6 +29,7 @@ jobs: apt install -y python3 python3-pip ln -s /usr/bin/python3 /usr/bin/python || true ln -s /usr/bin/pip3 /usr/bin/pip || true + pip install --force-reinstall pip - name: Checkout repository uses: actions/checkout@v4 From 2c3fd8da45e342635831b342dabc166e087d060e Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 13:58:42 -0400 Subject: [PATCH 047/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index c37e81c9..f142b25e 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -26,10 +26,10 @@ jobs: - name: Install Python run: | - apt install -y python3 python3-pip + apt install -y python3 + wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py ln -s /usr/bin/python3 /usr/bin/python || true ln -s /usr/bin/pip3 /usr/bin/pip || true - pip install --force-reinstall pip - name: Checkout repository uses: actions/checkout@v4 From 2aa56a34b924589f0785716ceb6a0255bfbdbf79 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 14:08:47 -0400 Subject: [PATCH 048/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index f142b25e..07956e28 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -154,7 +154,7 @@ jobs: working-directory: ../themis env: ANSIBLE_HOST_KEY_CHECKING: "False" - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem --become - name: Verify that Platform is running correctly continue-on-error: true From 3e6bd8c492ba520066d17639f55148bb65677bf4 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 14:22:47 -0400 Subject: [PATCH 049/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 07956e28..ec312b32 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Install common dependencies - run: apt update && apt install -y unzip curl wget git nodejs openssh-client + run: apt update && apt install -y unzip curl wget git nodejs openssh-client jq - name: Install Python run: | From e34f9f35e3191c4977802e8c0fd0131e27a666fe Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 14:54:07 -0400 Subject: [PATCH 050/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 32 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index ec312b32..d787a951 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,10 +9,12 @@ jobs: test-deployer: runs-on: self-hosted container: - image: ubuntu:latest + #image: ubuntu:latest + #options: --privileged + image: ghcr.io/catthehacker/ubuntu:act-latest options: --privileged env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + #GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -21,8 +23,8 @@ jobs: fail-fast: false steps: - - name: Install common dependencies - run: apt update && apt install -y unzip curl wget git nodejs openssh-client jq + #- name: Install common dependencies + # run: apt update && apt install -y unzip curl wget git nodejs openssh-client jq - name: Install Python run: | @@ -157,7 +159,6 @@ jobs: run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem --become - name: Verify that Platform is running correctly - continue-on-error: true working-directory: ../themis run: | for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do @@ -168,13 +169,32 @@ jobs: done - name: Verify that Gateway is running correctly - continue-on-error: true working-directory: ../themis run: | for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py gateway "http://$host:8083" done + - name: Verify that Redis is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.redis.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py redis "$host" + done + for host in $(jq -r '(.all.children.redis_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py redis "$host" + done + + - name: Verify that MongoDB is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.mongodb.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py mongodb "$host" + done + for host in $(jq -r '(.all.children.mongodb_arbiter.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py mongodb "$host" + done + - name: Tear down EC2 instances if: always() working-directory: ../themis/tofu_aws From 0f7a81d7faec5a55c6be9686655fe19b6e6e5369 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 14:56:48 -0400 Subject: [PATCH 051/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index d787a951..e32854c7 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -15,6 +15,7 @@ jobs: options: --privileged env: #GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -36,27 +37,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - #- name: Install dnf packages - # run: | - # sudo dnf groupinstall 'Development Tools' -y - # sudo dnf install python3.12 python3.12-pip python3.12-devel -y - # sudo dnf install nodejs -y - # sudo dnf install jq -y - - #- name: Set Python 3.12 as default - # run: | - # sudo alternatives --remove-all python3 2>/dev/null || true - # sudo alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 - # - # sudo alternatives --remove-all python 2>/dev/null || true - # sudo alternatives --install /usr/bin/python python /usr/bin/python3.12 100 - # - # sudo alternatives --remove-all pip3 2>/dev/null || true - # sudo alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 - # - # sudo alternatives --remove-all pip 2>/dev/null || true - # sudo alternatives --install /usr/bin/pip pip /usr/bin/pip3.12 100 - - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 with: @@ -92,8 +72,6 @@ jobs: chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pet-east1.open.pem ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - eval "$(ssh-agent -s)" - ssh-add ~/.ssh/id_rsa - name: Clone Themis repository working-directory: ".." From 5c6fc573672be1462b21de35c76443746b591643 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 14:58:32 -0400 Subject: [PATCH 052/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e32854c7..9dc4afe6 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,10 +12,8 @@ jobs: #image: ubuntu:latest #options: --privileged image: ghcr.io/catthehacker/ubuntu:act-latest - options: --privileged env: #GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -134,7 +132,7 @@ jobs: working-directory: ../themis env: ANSIBLE_HOST_KEY_CHECKING: "False" - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem --become + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly working-directory: ../themis From 06e57313426f01fdacf2159b0064ef0bdf2703c0 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 15:03:17 -0400 Subject: [PATCH 053/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 9dc4afe6..84b09e90 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -13,7 +13,8 @@ jobs: #options: --privileged image: ghcr.io/catthehacker/ubuntu:act-latest env: - #GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + ANSIBLE_HOST_KEY_CHECKING: "False" + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -22,8 +23,8 @@ jobs: fail-fast: false steps: - #- name: Install common dependencies - # run: apt update && apt install -y unzip curl wget git nodejs openssh-client jq + - name: Install miscellaneous dependencies + run: apt update && apt install -y jq - name: Install Python run: | @@ -69,7 +70,6 @@ jobs: echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pet-east1.open.pem - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - name: Clone Themis repository working-directory: ".." @@ -124,14 +124,10 @@ jobs: - name: Wait for EC2 instances to be ready working-directory: ../themis - env: - ANSIBLE_HOST_KEY_CHECKING: "False" run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances working-directory: ../themis - env: - ANSIBLE_HOST_KEY_CHECKING: "False" run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly From 20568913d70593affa8a923ba14eeea62640ea49 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 15:21:20 -0400 Subject: [PATCH 054/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 84b09e90..e606ae47 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -9,8 +9,6 @@ jobs: test-deployer: runs-on: self-hosted container: - #image: ubuntu:latest - #options: --privileged image: ghcr.io/catthehacker/ubuntu:act-latest env: ANSIBLE_HOST_KEY_CHECKING: "False" @@ -23,19 +21,24 @@ jobs: fail-fast: false steps: - - name: Install miscellaneous dependencies - run: apt update && apt install -y jq - - - name: Install Python - run: | - apt install -y python3 - wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py - ln -s /usr/bin/python3 /usr/bin/python || true - ln -s /usr/bin/pip3 /usr/bin/pip || true - - name: Checkout repository uses: actions/checkout@v4 + - name: Install jq + run: apt update && apt install -y jq + + #- name: Install Python + # run: | + # apt install -y python3 + # wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py + # ln -s /usr/bin/python3 /usr/bin/python || true + # ln -s /usr/bin/pip3 /usr/bin/pip || true + + - name: Setup Python + uses: actions/setup-python@v6.0.0 + with: + python-version: 3.12 + - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 with: From 9bf1c752f34b7bb0a3c04a8f28a9ea5dcf6a4bad Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 15:36:37 -0400 Subject: [PATCH 055/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e606ae47..adb2ff16 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -17,7 +17,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio, minimal] + design: [aio, minimal, ha2, asa] fail-fast: false steps: From 9efc732dde92c03bea5676199f7657d764c5475f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 15:49:54 -0400 Subject: [PATCH 056/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index adb2ff16..0b2d4166 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -27,17 +27,17 @@ jobs: - name: Install jq run: apt update && apt install -y jq - #- name: Install Python - # run: | - # apt install -y python3 - # wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py - # ln -s /usr/bin/python3 /usr/bin/python || true - # ln -s /usr/bin/pip3 /usr/bin/pip || true - - - name: Setup Python - uses: actions/setup-python@v6.0.0 - with: - python-version: 3.12 + - name: Install Python + run: | + apt install -y python3 + wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py + ln -s /usr/bin/python3 /usr/bin/python || true + ln -s /usr/bin/pip3 /usr/bin/pip || true + + #- name: Setup Python + # uses: actions/setup-python@v6.0.0 + # with: + # python-version: 3.12 - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From c7d279983332afb6ab27d7d784714028812a7bbe Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 22 Sep 2025 16:35:50 -0400 Subject: [PATCH 057/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 0b2d4166..e9130ea1 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -167,7 +167,7 @@ jobs: python3 scripts/validate.py mongodb "$host" done for host in $(jq -r '(.all.children.mongodb_arbiter.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do - python3 scripts/validate.py mongodb "$host" + python3 scripts/validate.py mongodb "$host" --arbiter done - name: Tear down EC2 instances From 12802c80e61881571cd13ecf9abd8d3d97c56596 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 09:29:36 -0400 Subject: [PATCH 058/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index e9130ea1..a8c47ae8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -45,7 +45,9 @@ jobs: tofu_version: latest - name: Install Ansible - run: pip install ansible + run: | + pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" + ansible --version - name: Install this collection run: ansible-galaxy collection install . --force From b68f3d116a4108eadc2015b089c5f531bf91a44e Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 10:33:31 -0400 Subject: [PATCH 059/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a8c47ae8..71a54ab2 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -129,7 +129,9 @@ jobs: - name: Wait for EC2 instances to be ready working-directory: ../themis - run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: | + ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + sleep 60 - name: Run deployer on created instances working-directory: ../themis From 4d0b1c0ad7daeb67348cec0b880b2703cf6afedc Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 10:51:48 -0400 Subject: [PATCH 060/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 71a54ab2..82616ed2 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -100,6 +100,14 @@ jobs: working-directory: ../themis/tofu_aws run: python3 ../scripts/generate_inventory.py -o hosts.json + - name: Configure Ansible to make all errors fatal + run: | + cat > ~/.ansible.cfg << 'EOF' + [defaults] + any_errors_fatal = True + max_fail_percentage = 0 + EOF + - name: Create Ansible playbook to wait for EC2 instances to be ready working-directory: ../themis run: | @@ -131,7 +139,6 @@ jobs: working-directory: ../themis run: | ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - sleep 60 - name: Run deployer on created instances working-directory: ../themis From 5fcab3f64eeae5006bd04f5a2cf5ef23935a10fe Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 12:45:47 -0400 Subject: [PATCH 061/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 82616ed2..c71a589b 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -133,6 +133,14 @@ jobs: path: /var/log/cloud-init-finished.marker timeout: 60 become: yes + + - name: Update packages and refresh cache + shell: | + dnf upgrade --refresh -y + dnf clean all + dnf makecache + become: yes + changed_when: true EOF - name: Wait for EC2 instances to be ready From 42e09fa4db99a333f2be6861a04e5acb5fa84f5c Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 13:01:41 -0400 Subject: [PATCH 062/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index c71a589b..5fa91c23 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -134,13 +134,23 @@ jobs: timeout: 60 become: yes - - name: Update packages and refresh cache - shell: | - dnf upgrade --refresh -y - dnf clean all - dnf makecache + - name: Update all packages + ansible.builtin.dnf: + name: "*" + state: latest + update_cache: yes + become: yes + + - name: Clean DNF cache + ansible.builtin.dnf: + autoremove: yes + clean: all + become: yes + + - name: Rebuild DNF cache + ansible.builtin.dnf: + update_cache: yes become: yes - changed_when: true EOF - name: Wait for EC2 instances to be ready From 6b4fe8de52ea1eb21464447ccbee518a98136cc3 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 13:19:31 -0400 Subject: [PATCH 063/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 5fa91c23..6e42d047 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -142,9 +142,7 @@ jobs: become: yes - name: Clean DNF cache - ansible.builtin.dnf: - autoremove: yes - clean: all + command: dnf clean all become: yes - name: Rebuild DNF cache From 310780e4d2862513ddb0b482ab195911780f6530 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 13:25:28 -0400 Subject: [PATCH 064/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 6e42d047..8e400282 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -153,8 +153,7 @@ jobs: - name: Wait for EC2 instances to be ready working-directory: ../themis - run: | - ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances working-directory: ../themis From 604fb5ebe2185f8299d23c6893247fa641780c88 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 13:37:33 -0400 Subject: [PATCH 065/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 8e400282..6a663e23 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -126,12 +126,12 @@ jobs: - name: Wait for SSH connection to be ready ansible.builtin.wait_for_connection: delay: 10 - timeout: 60 + timeout: 120 - name: Wait for cloud-init to complete (marker file check) ansible.builtin.wait_for: path: /var/log/cloud-init-finished.marker - timeout: 60 + timeout: 120 become: yes - name: Update all packages From c28bff4b9aad4fc6eef98e3eb1e6b4bedfbf7a24 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 14:18:05 -0400 Subject: [PATCH 066/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 60 ++++++++++++++++++----------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 6a663e23..5b6a5e9a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -52,13 +52,13 @@ jobs: - name: Install this collection run: ansible-galaxy collection install . --force - - name: Download Platform and Gateway binaries - run: | - cd ~/.ansible/collections/ansible_collections/itential/deployer/playbooks - mkdir -p files - cd files - ${{ secrets.PLATFORM_RPM_DOWNLOAD_COMMAND }} - ${{ secrets.GATEWAY_WHL_DOWNLOAD_COMMAND }} + #- name: Download Platform and Gateway binaries + # run: | + # cd ~/.ansible/collections/ansible_collections/itential/deployer/playbooks + # mkdir -p files + # cd files + # ${{ secrets.PLATFORM_RPM_DOWNLOAD_COMMAND }} + # ${{ secrets.GATEWAY_WHL_DOWNLOAD_COMMAND }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -88,6 +88,36 @@ jobs: working-directory: ../themis run: pip install -r scripts/requirements.txt + - name: Configure inventory to download Platform from Nexus + working-directory: ../themis + run: | + cat > inventories/common/group_vars/platform.yml << 'EOF' + --- + platform_packages: + - ${{ secrets.PLATFORM_RPM_URL }} + repository_username: ${{ secrets.NEXUS_USERNAME }} + repository_password: ${{ secrets.NEXUS_PASSWORD }} + EOF + + cat > inventories/common/group_vars/platform_secondary.yml << 'EOF' + --- + platform_packages: + - "${{ secrets.PLATFORM_RPM_URL }}" + repository_username: "${{ secrets.NEXUS_USERNAME }}" + repository_password: "${{ secrets.NEXUS_PASSWORD }}" + EOF + + - name: Configure inventory to download Gateway from Nexus + working-directory: ../themis + run: | + cat > inventories/common/group_vars/gateway.yml << 'EOF' + --- + gateway_release: ${{ secrets.GATEWAY_RELEASE }} + gateway_whl_file: "${{ secrets.GATEWAY_WHL_URL }}" + repository_username: "${{ secrets.NEXUS_USERNAME }}" + repository_password: "${{ secrets.NEXUS_PASSWORD }}" + EOF + - name: Initialize OpenTofu working-directory: ../themis/tofu_aws run: tofu init @@ -133,22 +163,6 @@ jobs: path: /var/log/cloud-init-finished.marker timeout: 120 become: yes - - - name: Update all packages - ansible.builtin.dnf: - name: "*" - state: latest - update_cache: yes - become: yes - - - name: Clean DNF cache - command: dnf clean all - become: yes - - - name: Rebuild DNF cache - ansible.builtin.dnf: - update_cache: yes - become: yes EOF - name: Wait for EC2 instances to be ready From a79a6b4617f7afc668b6b9464817c0ac12f22346 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 14:26:35 -0400 Subject: [PATCH 067/144] try fixing weird issue --- roles/redis/tasks/install-from-source.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/redis/tasks/install-from-source.yml b/roles/redis/tasks/install-from-source.yml index 7b3d9d21..81c35e31 100644 --- a/roles/redis/tasks/install-from-source.yml +++ b/roles/redis/tasks/install-from-source.yml @@ -35,10 +35,13 @@ - name: Online install when: not offline_install_enabled block: + - name: Refresh repository metadata + command: dnf clean metadata && dnf makecache + - name: Install Redis build packages ansible.builtin.dnf: name: "{{ item }}" - state: present + state: latest register: redis_install_result with_items: "{{ redis_build_packages }}" when: From 4e4287bcc9fd73a0750a0c5affb561cd842283af Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 14:35:05 -0400 Subject: [PATCH 068/144] nvm try again --- roles/redis/tasks/install-from-source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/redis/tasks/install-from-source.yml b/roles/redis/tasks/install-from-source.yml index 81c35e31..d37e7aca 100644 --- a/roles/redis/tasks/install-from-source.yml +++ b/roles/redis/tasks/install-from-source.yml @@ -36,7 +36,7 @@ when: not offline_install_enabled block: - name: Refresh repository metadata - command: dnf clean metadata && dnf makecache + shell: dnf clean metadata && dnf makecache - name: Install Redis build packages ansible.builtin.dnf: From 5eee598b9d3dc3335f99449ce9040e98cdf45cd5 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 14:48:31 -0400 Subject: [PATCH 069/144] Update install.yml --- roles/redis/defaults/main/install.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/redis/defaults/main/install.yml b/roles/redis/defaults/main/install.yml index f8e1a572..61cce43d 100644 --- a/roles/redis/defaults/main/install.yml +++ b/roles/redis/defaults/main/install.yml @@ -18,7 +18,10 @@ redis_build_packages: - gcc - gcc-c++ - make + - systemd - systemd-devel + - systemd-libs + - systemd-pam # The Remi and EPEL repos are used only when: # - redis_install_from_source is set to 'false' From 025c50c2336a414a1153a82572ff1981972f6da2 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 15:01:39 -0400 Subject: [PATCH 070/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 5b6a5e9a..048b0f94 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -163,6 +163,21 @@ jobs: path: /var/log/cloud-init-finished.marker timeout: 120 become: yes + + - name: Use specific repository instead of mirrors + ini_file: + path: /etc/yum.repos.d/rocky.repo + section: appstream + option: baseurl + value: http://dl.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/ + backup: yes + + - name: Disable mirrorlist for appstream + ini_file: + path: /etc/yum.repos.d/rocky.repo + section: appstream + option: mirrorlist + state: absent EOF - name: Wait for EC2 instances to be ready From a4be9e51bde1bf89e3c616f76750c5ed0341ff72 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Tue, 23 Sep 2025 15:07:41 -0400 Subject: [PATCH 071/144] revert deployer changes --- roles/redis/defaults/main/install.yml | 3 --- roles/redis/tasks/install-from-source.yml | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/roles/redis/defaults/main/install.yml b/roles/redis/defaults/main/install.yml index 61cce43d..f8e1a572 100644 --- a/roles/redis/defaults/main/install.yml +++ b/roles/redis/defaults/main/install.yml @@ -18,10 +18,7 @@ redis_build_packages: - gcc - gcc-c++ - make - - systemd - systemd-devel - - systemd-libs - - systemd-pam # The Remi and EPEL repos are used only when: # - redis_install_from_source is set to 'false' diff --git a/roles/redis/tasks/install-from-source.yml b/roles/redis/tasks/install-from-source.yml index d37e7aca..7b3d9d21 100644 --- a/roles/redis/tasks/install-from-source.yml +++ b/roles/redis/tasks/install-from-source.yml @@ -35,13 +35,10 @@ - name: Online install when: not offline_install_enabled block: - - name: Refresh repository metadata - shell: dnf clean metadata && dnf makecache - - name: Install Redis build packages ansible.builtin.dnf: name: "{{ item }}" - state: latest + state: present register: redis_install_result with_items: "{{ redis_build_packages }}" when: From 76d71fba200965e3b69d051f3bd65c0f35ea8cae Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 15:21:49 -0400 Subject: [PATCH 072/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 048b0f94..7ccd9a4a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -165,6 +165,7 @@ jobs: become: yes - name: Use specific repository instead of mirrors + become: true ini_file: path: /etc/yum.repos.d/rocky.repo section: appstream @@ -173,6 +174,7 @@ jobs: backup: yes - name: Disable mirrorlist for appstream + become: true ini_file: path: /etc/yum.repos.d/rocky.repo section: appstream From 008961ac1172a24f9728742747df29c70642bc2f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 15:38:25 -0400 Subject: [PATCH 073/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 7ccd9a4a..8036cb1d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -113,7 +113,7 @@ jobs: cat > inventories/common/group_vars/gateway.yml << 'EOF' --- gateway_release: ${{ secrets.GATEWAY_RELEASE }} - gateway_whl_file: "${{ secrets.GATEWAY_WHL_URL }}" + gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" repository_username: "${{ secrets.NEXUS_USERNAME }}" repository_password: "${{ secrets.NEXUS_PASSWORD }}" EOF From d122b91367c3800d7696d1ad63976e02fdc83c13 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 16:07:03 -0400 Subject: [PATCH 074/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 8036cb1d..544b0cb0 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -17,7 +17,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio, minimal, ha2, asa] + design: [asa, asa, asa, asa] fail-fast: false steps: From bfa944e6eb31dc2532ff1dfb39955e12a78e7fa6 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Tue, 23 Sep 2025 16:16:26 -0400 Subject: [PATCH 075/144] add debugging tasks before installing build packages --- roles/redis/tasks/install-from-source.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/redis/tasks/install-from-source.yml b/roles/redis/tasks/install-from-source.yml index 7b3d9d21..00e5b5ed 100644 --- a/roles/redis/tasks/install-from-source.yml +++ b/roles/redis/tasks/install-from-source.yml @@ -35,6 +35,14 @@ - name: Online install when: not offline_install_enabled block: + - name: Check current systemd packages + shell: rpm -qa | grep systemd + register: systemd_packages + + - name: Show systemd packages + debug: + var: systemd_packages.stdout_lines + - name: Install Redis build packages ansible.builtin.dnf: name: "{{ item }}" From 4c41b6d6004ee9336033ae6760356368bd6a896f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 23 Sep 2025 16:24:33 -0400 Subject: [PATCH 076/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 544b0cb0..cd18ab68 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -156,12 +156,12 @@ jobs: - name: Wait for SSH connection to be ready ansible.builtin.wait_for_connection: delay: 10 - timeout: 120 + timeout: 300 - name: Wait for cloud-init to complete (marker file check) ansible.builtin.wait_for: path: /var/log/cloud-init-finished.marker - timeout: 120 + timeout: 300 become: yes - name: Use specific repository instead of mirrors From 37cdad55103ae0e41ba73d3df7aa13e68acbbe46 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:39:23 -0400 Subject: [PATCH 077/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 67 ++++++----------------------- 1 file changed, 14 insertions(+), 53 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index cd18ab68..ef3dd930 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,32 +12,27 @@ jobs: image: ghcr.io/catthehacker/ubuntu:act-latest env: ANSIBLE_HOST_KEY_CHECKING: "False" - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + EC2_KEY_PATH: ~/.ssh/pet-east1.open.pem + GIT_KEY_PATH: ~/.ssh/id_rsa PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: matrix: - design: [asa, asa, asa, asa] + design: [aio, minimal, ha2, asa] fail-fast: false steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install jq - run: apt update && apt install -y jq + - name: Update package lists + run: apt update - name: Install Python run: | apt install -y python3 wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py - ln -s /usr/bin/python3 /usr/bin/python || true - ln -s /usr/bin/pip3 /usr/bin/pip || true - - #- name: Setup Python - # uses: actions/setup-python@v6.0.0 - # with: - # python-version: 3.12 + ln -s /usr/bin/python3 /usr/bin/python - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 @@ -52,14 +47,6 @@ jobs: - name: Install this collection run: ansible-galaxy collection install . --force - #- name: Download Platform and Gateway binaries - # run: | - # cd ~/.ansible/collections/ansible_collections/itential/deployer/playbooks - # mkdir -p files - # cd files - # ${{ secrets.PLATFORM_RPM_DOWNLOAD_COMMAND }} - # ${{ secrets.GATEWAY_WHL_DOWNLOAD_COMMAND }} - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -71,15 +58,15 @@ jobs: - name: Setup SSH for GitLab and EC2 run: | mkdir -p ~/.ssh - echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa - echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 ~/.ssh/id_rsa - chmod 600 ~/.ssh/pet-east1.open.pem + echo "${{ secrets.GITLAB_SSH_KEY }}" > "$GIT_KEY_PATH" + echo "${{ secrets.EC2_SSH_KEY }}" > "$EC2_KEY_PATH" + chmod 600 "$GIT_KEY_PATH" + chmod 600 "$EC2_KEY_PATH" - name: Clone Themis repository working-directory: ".." run: | - rm -rf themis + export GIT_SSH_COMMAND="ssh -i \"$GIT_KEY_PATH\" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" cd themis ls -la @@ -95,16 +82,12 @@ jobs: --- platform_packages: - ${{ secrets.PLATFORM_RPM_URL }} - repository_username: ${{ secrets.NEXUS_USERNAME }} - repository_password: ${{ secrets.NEXUS_PASSWORD }} EOF cat > inventories/common/group_vars/platform_secondary.yml << 'EOF' --- platform_packages: - "${{ secrets.PLATFORM_RPM_URL }}" - repository_username: "${{ secrets.NEXUS_USERNAME }}" - repository_password: "${{ secrets.NEXUS_PASSWORD }}" EOF - name: Configure inventory to download Gateway from Nexus @@ -114,8 +97,6 @@ jobs: --- gateway_release: ${{ secrets.GATEWAY_RELEASE }} gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" - repository_username: "${{ secrets.NEXUS_USERNAME }}" - repository_password: "${{ secrets.NEXUS_PASSWORD }}" EOF - name: Initialize OpenTofu @@ -138,7 +119,7 @@ jobs: max_fail_percentage = 0 EOF - - name: Create Ansible playbook to wait for EC2 instances to be ready + - name: Wait for EC2 instances to be ready working-directory: ../themis run: | cat > wait-for-instances.yml << 'EOF' @@ -163,32 +144,12 @@ jobs: path: /var/log/cloud-init-finished.marker timeout: 300 become: yes - - - name: Use specific repository instead of mirrors - become: true - ini_file: - path: /etc/yum.repos.d/rocky.repo - section: appstream - option: baseurl - value: http://dl.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/ - backup: yes - - - name: Disable mirrorlist for appstream - become: true - ini_file: - path: /etc/yum.repos.d/rocky.repo - section: appstream - option: mirrorlist - state: absent EOF + ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file="$EC2_KEY_PATH" - - name: Wait for EC2 instances to be ready - working-directory: ../themis - run: ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file="$EC2_KEY_PATH" --extra-vars "repository_username=${{ secrets.NEXUS_USERNAME }} repository_password=${{ secrets.NEXUS_PASSWORD }}" - name: Verify that Platform is running correctly working-directory: ../themis From 93ef6cd32e19ddc4bd418652a6b23432de4105c1 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:46:06 -0400 Subject: [PATCH 078/144] Update test-deployer.yml From bd828db33ff3f8c3a87fe4a31343ea69fc98390f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:49:10 -0400 Subject: [PATCH 079/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index ef3dd930..9fda3477 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,8 +12,8 @@ jobs: image: ghcr.io/catthehacker/ubuntu:act-latest env: ANSIBLE_HOST_KEY_CHECKING: "False" - EC2_KEY_PATH: ~/.ssh/pet-east1.open.pem - GIT_KEY_PATH: ~/.ssh/id_rsa + EC2_KEY_PATH: $HOME/.ssh/pet-east1.open.pem + GIT_KEY_PATH: $HOME/.ssh/id_rsa PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: From f8b494570d12eecb8a140d720b0bfa07e4ca3d85 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:53:43 -0400 Subject: [PATCH 080/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 9fda3477..dfd0231b 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,8 +12,6 @@ jobs: image: ghcr.io/catthehacker/ubuntu:act-latest env: ANSIBLE_HOST_KEY_CHECKING: "False" - EC2_KEY_PATH: $HOME/.ssh/pet-east1.open.pem - GIT_KEY_PATH: $HOME/.ssh/id_rsa PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -58,15 +56,16 @@ jobs: - name: Setup SSH for GitLab and EC2 run: | mkdir -p ~/.ssh - echo "${{ secrets.GITLAB_SSH_KEY }}" > "$GIT_KEY_PATH" - echo "${{ secrets.EC2_SSH_KEY }}" > "$EC2_KEY_PATH" + echo "${{ secrets.GITLAB_SSH_KEY }}" > /.ssh/id_rsa + echo "${{ secrets.EC2_SSH_KEY }}" > /.ssh/pet-east1.open.pem chmod 600 "$GIT_KEY_PATH" chmod 600 "$EC2_KEY_PATH" - name: Clone Themis repository working-directory: ".." + env: + GIT_SSH_COMMAND: ssh -i /.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no run: | - export GIT_SSH_COMMAND="ssh -i \"$GIT_KEY_PATH\" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" cd themis ls -la @@ -145,11 +144,11 @@ jobs: timeout: 300 become: yes EOF - ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file="$EC2_KEY_PATH" + ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file="$EC2_KEY_PATH" --extra-vars "repository_username=${{ secrets.NEXUS_USERNAME }} repository_password=${{ secrets.NEXUS_PASSWORD }}" + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file=~/.ssh/pet-east1.open.pem --extra-vars "repository_username=${{ secrets.NEXUS_USERNAME }} repository_password=${{ secrets.NEXUS_PASSWORD }}" - name: Verify that Platform is running correctly working-directory: ../themis From ccea485e9b2028f777877d451a50afa0cbf6df05 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:55:02 -0400 Subject: [PATCH 081/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index dfd0231b..40873141 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -56,8 +56,8 @@ jobs: - name: Setup SSH for GitLab and EC2 run: | mkdir -p ~/.ssh - echo "${{ secrets.GITLAB_SSH_KEY }}" > /.ssh/id_rsa - echo "${{ secrets.EC2_SSH_KEY }}" > /.ssh/pet-east1.open.pem + echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem chmod 600 "$GIT_KEY_PATH" chmod 600 "$EC2_KEY_PATH" From b7fa219bf2b1485f175f831054ff9a5434e6ded7 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:56:26 -0400 Subject: [PATCH 082/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 40873141..c0c5848a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -58,8 +58,8 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 "$GIT_KEY_PATH" - chmod 600 "$EC2_KEY_PATH" + chmod 600 ~/.ssh/id_rsa + chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository working-directory: ".." From d0690ff4e04f77001cb2d1bcadebdc0f358ea9f8 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 09:59:14 -0400 Subject: [PATCH 083/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index c0c5848a..c72a3153 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -66,6 +66,7 @@ jobs: env: GIT_SSH_COMMAND: ssh -i /.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no run: | + rm -rf themis git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" cd themis ls -la From fc8fe878dfb1839f1de1097e60d27c07bd03e0bb Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:01:09 -0400 Subject: [PATCH 084/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index c72a3153..fbeb2979 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -64,7 +64,7 @@ jobs: - name: Clone Themis repository working-directory: ".." env: - GIT_SSH_COMMAND: ssh -i /.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no run: | rm -rf themis git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" From 088d989e69470f384fe6986c0b293d99a9e5e523 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:15:31 -0400 Subject: [PATCH 085/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index fbeb2979..bbbb3138 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -99,6 +99,12 @@ jobs: gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" EOF + - Add Nexus credentials to inventory + working-directory: ../themis + run: | + cat 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml + cat 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml + - name: Initialize OpenTofu working-directory: ../themis/tofu_aws run: tofu init @@ -149,7 +155,7 @@ jobs: - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file=~/.ssh/pet-east1.open.pem --extra-vars "repository_username=${{ secrets.NEXUS_USERNAME }} repository_password=${{ secrets.NEXUS_PASSWORD }}" + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly working-directory: ../themis From 3c68ed9502f3f72d2ecf23fcf31cbdf8c5e7c339 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:16:24 -0400 Subject: [PATCH 086/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index bbbb3138..9338ddb5 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -99,7 +99,7 @@ jobs: gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" EOF - - Add Nexus credentials to inventory + - name: Add Nexus credentials to inventory working-directory: ../themis run: | cat 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml From ed46814f986f1fa15cdca12b010052d06daad954 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:37:27 -0400 Subject: [PATCH 087/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 9338ddb5..a238557d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -99,11 +99,18 @@ jobs: gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" EOF + - name: Configure Redis to be installed from binary + run: | + cat > inventories/common/group_vars/redis.yml << 'EOF' + --- + redis_install_from_source: false + EOF + - name: Add Nexus credentials to inventory working-directory: ../themis run: | - cat 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml - cat 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml + echo 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml + echo 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml - name: Initialize OpenTofu working-directory: ../themis/tofu_aws From 30d8a0ae88c21b97e64782f35049455cd5d40f6f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:41:40 -0400 Subject: [PATCH 088/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a238557d..98a3cd8f 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -100,6 +100,7 @@ jobs: EOF - name: Configure Redis to be installed from binary + working-directory: ../themis run: | cat > inventories/common/group_vars/redis.yml << 'EOF' --- From 0ac5eaff6f19548b7d748e63462574e0edd47ad7 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:47:36 -0400 Subject: [PATCH 089/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 98a3cd8f..b4533dd8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -163,7 +163,7 @@ jobs: - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvvv --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly working-directory: ../themis From 9905f69f2fdb86fe9e55a55b07b63898ff1ef695 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 10:56:15 -0400 Subject: [PATCH 090/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index b4533dd8..99648864 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -16,7 +16,8 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio, minimal, ha2, asa] + #design: [aio, minimal, ha2, asa] + design: [minimal] fail-fast: false steps: @@ -163,7 +164,7 @@ jobs: - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvv --key-file=~/.ssh/pet-east1.open.pem - name: Verify that Platform is running correctly working-directory: ../themis From af91bb3311079e14b25bb2b189727613c9ce8bae Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 11:33:52 -0400 Subject: [PATCH 091/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 99648864..cd33a33d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -107,6 +107,10 @@ jobs: --- redis_install_from_source: false EOF + cat > inventories/common/group_vars/redis_secondary.yml << 'EOF' + --- + redis_install_from_source: false + EOF - name: Add Nexus credentials to inventory working-directory: ../themis @@ -164,7 +168,7 @@ jobs: - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -vvv --key-file=~/.ssh/pet-east1.open.pem + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" - name: Verify that Platform is running correctly working-directory: ../themis From f7fa96052a5e07dcf1e9ff13191dd203cc1897bc Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 11:51:18 -0400 Subject: [PATCH 092/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index cd33a33d..42d4b631 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -12,12 +12,12 @@ jobs: image: ghcr.io/catthehacker/ubuntu:act-latest env: ANSIBLE_HOST_KEY_CHECKING: "False" + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: matrix: - #design: [aio, minimal, ha2, asa] - design: [minimal] + design: [aio, minimal, ha2, asa] fail-fast: false steps: @@ -63,9 +63,7 @@ jobs: chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository - working-directory: ".." - env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + working-directory: .. run: | rm -rf themis git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" @@ -100,18 +98,6 @@ jobs: gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" EOF - - name: Configure Redis to be installed from binary - working-directory: ../themis - run: | - cat > inventories/common/group_vars/redis.yml << 'EOF' - --- - redis_install_from_source: false - EOF - cat > inventories/common/group_vars/redis_secondary.yml << 'EOF' - --- - redis_install_from_source: false - EOF - - name: Add Nexus credentials to inventory working-directory: ../themis run: | @@ -168,7 +154,7 @@ jobs: - name: Run deployer on created instances working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" - name: Verify that Platform is running correctly working-directory: ../themis From d2887a190894ce39e449d70c19613c79af561fd4 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 14:11:34 -0400 Subject: [PATCH 093/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 64 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 42d4b631..07cd8dbe 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -124,35 +124,43 @@ jobs: max_fail_percentage = 0 EOF - - name: Wait for EC2 instances to be ready + #- name: Wait for EC2 instances to be ready + # working-directory: ../themis + # run: | + # cat > wait-for-instances.yml << 'EOF' + # --- + # - name: Wait for instances to initialize + # hosts: all + # gather_facts: no + # become: no + # serial: "{{ ansible_serial | default(10) }}" + # + # vars: + # ssh_port: 22 + # + # tasks: + # - name: Wait for SSH connection to be ready + # ansible.builtin.wait_for_connection: + # delay: 10 + # timeout: 300 + # + # - name: Wait for cloud-init to complete (marker file check) + # ansible.builtin.wait_for: + # path: /var/log/cloud-init-finished.marker + # timeout: 300 + # become: yes + # EOF + # ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + + - name: Wait for EC2 instances to be ready to SSH into working-directory: ../themis - run: | - cat > wait-for-instances.yml << 'EOF' - --- - - name: Wait for instances to initialize - hosts: all - gather_facts: no - become: no - serial: "{{ ansible_serial | default(10) }}" - - vars: - ssh_port: 22 - - tasks: - - name: Wait for SSH connection to be ready - ansible.builtin.wait_for_connection: - delay: 10 - timeout: 300 - - - name: Wait for cloud-init to complete (marker file check) - ansible.builtin.wait_for: - path: /var/log/cloud-init-finished.marker - timeout: 300 - become: yes - EOF - ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem - - - name: Run deployer on created instances + run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + + - name: Wait for cloud-init to complete + working-directory: ../themis + run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + + - name: Run the deployer working-directory: ../themis run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" From 0926e2707b892a03c239443f63a60f2c822d8260 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 14:58:30 -0400 Subject: [PATCH 094/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 62 +++++++++-------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 07cd8dbe..764e3340 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -11,7 +11,6 @@ jobs: container: image: ghcr.io/catthehacker/ubuntu:act-latest env: - ANSIBLE_HOST_KEY_CHECKING: "False" GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 @@ -43,6 +42,17 @@ jobs: pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" ansible --version + # Configures Ansible to fail fast, skip host key checking, use correct key file + - name: Write Ansible configuration file + run: | + cat > ~/.ansible.cfg << 'EOF' + [defaults] + any_errors_fatal = True + host_key_checking = False + max_fail_percentage = 0 + private_key_file = ~/.ssh/pet-east1.open.pem + EOF + - name: Install this collection run: ansible-galaxy collection install . --force @@ -108,61 +118,27 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init - - name: Provision EC2 instances for ${{ matrix.design }} design + - name: Provision EC2 instances working-directory: ../themis/tofu_aws run: tofu apply -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws - run: python3 ../scripts/generate_inventory.py -o hosts.json - - - name: Configure Ansible to make all errors fatal - run: | - cat > ~/.ansible.cfg << 'EOF' - [defaults] - any_errors_fatal = True - max_fail_percentage = 0 - EOF - - #- name: Wait for EC2 instances to be ready - # working-directory: ../themis - # run: | - # cat > wait-for-instances.yml << 'EOF' - # --- - # - name: Wait for instances to initialize - # hosts: all - # gather_facts: no - # become: no - # serial: "{{ ansible_serial | default(10) }}" - # - # vars: - # ssh_port: 22 - # - # tasks: - # - name: Wait for SSH connection to be ready - # ansible.builtin.wait_for_connection: - # delay: 10 - # timeout: 300 - # - # - name: Wait for cloud-init to complete (marker file check) - # ansible.builtin.wait_for: - # path: /var/log/cloud-init-finished.marker - # timeout: 300 - # become: yes - # EOF - # ansible-playbook wait-for-instances.yml -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem + run: python3 ../scripts/generate_inventory.py --validate -o hosts.json + # Retries until SSH connection is established or timeout is reached - name: Wait for EC2 instances to be ready to SSH into working-directory: ../themis - run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v - - name: Wait for cloud-init to complete + # Waits for cloud init marker script to be written to disk (see cloud-init.tpl) + - name: Wait for cloud init script to complete working-directory: ../themis - run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v --key-file=~/.ssh/pet-east1.open.pem + run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v - name: Run the deployer working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v --key-file=~/.ssh/pet-east1.open.pem -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" + run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" - name: Verify that Platform is running correctly working-directory: ../themis From ca344258a1653cf1fef2d0bec41023d765f20fb0 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 15:10:42 -0400 Subject: [PATCH 095/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 764e3340..8eedd3f8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -138,7 +138,14 @@ jobs: - name: Run the deployer working-directory: ../themis - run: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ matrix.design }} -v -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" + run: > + ansible-playbook itential.deployer.site + -i tofu_aws/hosts.json + -i inventories/common + -i inventories/${{ matrix.design }} + -e "redis_install_from_source=false" + -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" + -v - name: Verify that Platform is running correctly working-directory: ../themis From c50646decf9ce130641788ee52905199f73b9eae Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 16:45:03 -0400 Subject: [PATCH 096/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 40 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 8eedd3f8..52a9af76 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -5,13 +5,18 @@ on: branches: - pipeline-test +env: + AWS_REGION: us-east-1 + EC2_KEY_PATH: ~/.ssh/pet-east1.open.pem + GITLAB_KEY_PATH: ~/.ssh/id_rsa + jobs: test-deployer: runs-on: self-hosted container: image: ghcr.io/catthehacker/ubuntu:act-latest env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + GIT_SSH_COMMAND: ssh -i ${{ env.GITLAB_KEY_PATH }} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: @@ -42,7 +47,7 @@ jobs: pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" ansible --version - # Configures Ansible to fail fast, skip host key checking, use correct key file + # Configures Ansible to fail immediately on error, skip host key checking, use correct key file - name: Write Ansible configuration file run: | cat > ~/.ansible.cfg << 'EOF' @@ -50,7 +55,7 @@ jobs: any_errors_fatal = True host_key_checking = False max_fail_percentage = 0 - private_key_file = ~/.ssh/pet-east1.open.pem + private_key_file = ${{ env.EC2_KEY_PATH }} EOF - name: Install this collection @@ -62,21 +67,25 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} - aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + aws-region: ${{ env.AWS_REGION }} + + - name: Setup SSH for GitLab + run: | + mkdir -p "$(dirname "${{ env.GITLAB_KEY_PATH }}")" + echo "${{ secrets.GITLAB_SSH_KEY }}" > "${{ env.GITLAB_KEY_PATH }}" + chmod 600 "${{ env.GITLAB_KEY_PATH }}" - - name: Setup SSH for GitLab and EC2 + - name: Setup SSH for EC2 run: | - mkdir -p ~/.ssh - echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa - echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 ~/.ssh/id_rsa - chmod 600 ~/.ssh/pet-east1.open.pem + mkdir -p "$(dirname "${{ env.EC2_KEY_PATH }}")" + echo "${{ secrets.EC2_SSH_KEY }}" > ${{ env.EC2_KEY_PATH }} + chmod 600 - name: Clone Themis repository working-directory: .. run: | rm -rf themis - git clone "${{ secrets.THEMIS_REPO_SSH_STRING }}" + git clone "${{ secrets.THEMIS_REPO_SSH_URI }}" cd themis ls -la @@ -90,7 +99,7 @@ jobs: cat > inventories/common/group_vars/platform.yml << 'EOF' --- platform_packages: - - ${{ secrets.PLATFORM_RPM_URL }} + - "${{ secrets.PLATFORM_RPM_URL }}" EOF cat > inventories/common/group_vars/platform_secondary.yml << 'EOF' @@ -108,6 +117,7 @@ jobs: gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" EOF + # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command - name: Add Nexus credentials to inventory working-directory: ../themis run: | @@ -118,6 +128,10 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init + - name: Generate OpenTofu execution plan + working-directory: ../themis/tofu_aws + run: tofu plan -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve + - name: Provision EC2 instances working-directory: ../themis/tofu_aws run: tofu apply -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve @@ -131,7 +145,7 @@ jobs: working-directory: ../themis run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v - # Waits for cloud init marker script to be written to disk (see cloud-init.tpl) + # Waits for cloud init marker file to be written to disk (see cloud-init.tpl) - name: Wait for cloud init script to complete working-directory: ../themis run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v From 6f0b43d054ecd286090350b9d6279941dd5240dc Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 16:47:15 -0400 Subject: [PATCH 097/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 52a9af76..dcca1713 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -16,7 +16,7 @@ jobs: container: image: ghcr.io/catthehacker/ubuntu:act-latest env: - GIT_SSH_COMMAND: ssh -i ${{ env.GITLAB_KEY_PATH }} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: From 27df46a1ea6de79c0b5cad005d629bb07ca9b2ab Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 16:53:28 -0400 Subject: [PATCH 098/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index dcca1713..6b98d57f 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -78,8 +78,8 @@ jobs: - name: Setup SSH for EC2 run: | mkdir -p "$(dirname "${{ env.EC2_KEY_PATH }}")" - echo "${{ secrets.EC2_SSH_KEY }}" > ${{ env.EC2_KEY_PATH }} - chmod 600 + echo "${{ secrets.EC2_SSH_KEY }}" > "${{ env.EC2_KEY_PATH }}" + chmod 600 "${{ env.EC2_KEY_PATH }}" - name: Clone Themis repository working-directory: .. From cc3e164ad1d0bd702ed5fbc03e364c36c677d7c7 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 17:00:51 -0400 Subject: [PATCH 099/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 6b98d57f..abdf50b9 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -5,11 +5,6 @@ on: branches: - pipeline-test -env: - AWS_REGION: us-east-1 - EC2_KEY_PATH: ~/.ssh/pet-east1.open.pem - GITLAB_KEY_PATH: ~/.ssh/id_rsa - jobs: test-deployer: runs-on: self-hosted @@ -55,7 +50,7 @@ jobs: any_errors_fatal = True host_key_checking = False max_fail_percentage = 0 - private_key_file = ${{ env.EC2_KEY_PATH }} + private_key_file = ~/.ssh/pet-east1.open.pem EOF - name: Install this collection @@ -67,19 +62,18 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} - aws-region: ${{ env.AWS_REGION }} + aws-region: us-east-1 - name: Setup SSH for GitLab run: | - mkdir -p "$(dirname "${{ env.GITLAB_KEY_PATH }}")" - echo "${{ secrets.GITLAB_SSH_KEY }}" > "${{ env.GITLAB_KEY_PATH }}" - chmod 600 "${{ env.GITLAB_KEY_PATH }}" + mkdir -p ~/.ssh + echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa - name: Setup SSH for EC2 run: | - mkdir -p "$(dirname "${{ env.EC2_KEY_PATH }}")" - echo "${{ secrets.EC2_SSH_KEY }}" > "${{ env.EC2_KEY_PATH }}" - chmod 600 "${{ env.EC2_KEY_PATH }}" + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository working-directory: .. From f1da2275b74165e50e152870f088178a0ff6d720 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 24 Sep 2025 17:02:41 -0400 Subject: [PATCH 100/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index abdf50b9..190d5561 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -124,7 +124,7 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve + run: tofu plan -var-file=tfvars/${{ matrix.design }}.tfvars - name: Provision EC2 instances working-directory: ../themis/tofu_aws From 6f23f1542ed2b1070ea86d9f6e6b617fee40fa85 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 09:09:36 -0400 Subject: [PATCH 101/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 190d5561..38c513e7 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -31,6 +31,8 @@ jobs: apt install -y python3 wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py ln -s /usr/bin/python3 /usr/bin/python + python --version + pip --version - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 @@ -143,7 +145,9 @@ jobs: - name: Wait for cloud init script to complete working-directory: ../themis run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v - + + # Overrides inventory variables to install Redis from the Remi repository using a known working URL, as a + # workaround for bugs in the deployer (dependency resolution errors when building Redis, incorrect Remi URL) - name: Run the deployer working-directory: ../themis run: > From 92ad20c16712292b2d4a65bee8726d5af9479266 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 09:15:23 -0400 Subject: [PATCH 102/144] Update test-deployer.yml From 5e5035ee7b7bab00294dbfcaffc8ded43e1fd13c Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 10:39:29 -0400 Subject: [PATCH 103/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 38c513e7..cc669e8c 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio, minimal, ha2, asa] + design: [aio] fail-fast: false steps: From 3a32e7251593373ae5bc47db2ec63502fb5873c9 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 14:13:32 -0400 Subject: [PATCH 104/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index cc669e8c..211c827d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -8,15 +8,15 @@ on: jobs: test-deployer: runs-on: self-hosted - container: - image: ghcr.io/catthehacker/ubuntu:act-latest + #container: + # image: ghcr.io/catthehacker/ubuntu:act-latest env: GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 strategy: matrix: - design: [aio] + design: [aio, minimal, ha2, asa] fail-fast: false steps: From 3d4724d3e58c93be12a0680506bc538bde092fbc Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 14:15:06 -0400 Subject: [PATCH 105/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 211c827d..f96545db 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -24,13 +24,13 @@ jobs: uses: actions/checkout@v4 - name: Update package lists - run: apt update + run: sudo apt update - name: Install Python run: | - apt install -y python3 + sudo apt install -y python3 wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py - ln -s /usr/bin/python3 /usr/bin/python + sudo ln -s /usr/bin/python3 /usr/bin/python python --version pip --version @@ -80,7 +80,6 @@ jobs: - name: Clone Themis repository working-directory: .. run: | - rm -rf themis git clone "${{ secrets.THEMIS_REPO_SSH_URI }}" cd themis ls -la From ee51a15687caea507f440cab87fadc4dcf20e506 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 14:27:36 -0400 Subject: [PATCH 106/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index f96545db..511640b8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -29,8 +29,9 @@ jobs: - name: Install Python run: | sudo apt install -y python3 - wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py - sudo ln -s /usr/bin/python3 /usr/bin/python + wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py + sudo ln -s /usr/bin/python3 /usr/bin/python || true + sudo ln -s /usr/bin/pip3 /usr/bin/pip || true python --version pip --version From 417450a2d52fd99b2fd0895098d68348efae1563 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 14:49:36 -0400 Subject: [PATCH 107/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 511640b8..a655b246 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -34,6 +34,12 @@ jobs: sudo ln -s /usr/bin/pip3 /usr/bin/pip || true python --version pip --version + + # For some reason, Node appears to be a dependency of Themis + - name: Install Node.js + run: | + sudo apt install -y nodejs + node --version - name: Install OpenTofu uses: opentofu/setup-opentofu@v1 From 58e18b7ac264e6087321f7a75fa92ff80fb6277f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 16:12:05 -0400 Subject: [PATCH 108/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a655b246..227e4c68 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -8,8 +8,6 @@ on: jobs: test-deployer: runs-on: self-hosted - #container: - # image: ghcr.io/catthehacker/ubuntu:act-latest env: GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" From ba31202e1e1a9196d408a99f76315bae061a8d77 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 25 Sep 2025 16:50:12 -0400 Subject: [PATCH 109/144] Update test-deployer.yml From eb14af61c726b1f0ab38cd8d3c46a24ae3f6270a Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 26 Sep 2025 09:30:24 -0400 Subject: [PATCH 110/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 227e4c68..d58c9a81 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio, minimal, ha2, asa] + design: [aio] fail-fast: false steps: From 91c3c1b2624037a2b7a2ad2ef2b620736c67c5bc Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 26 Sep 2025 10:49:18 -0400 Subject: [PATCH 111/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index d58c9a81..227e4c68 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - design: [aio] + design: [aio, minimal, ha2, asa] fail-fast: false steps: From ca8b0ae1d397e47a06c8891ed2a84b28f6dda94f Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:31:04 -0400 Subject: [PATCH 112/144] make workflow reusable --- .github/workflows/test-deployer.yml | 45 ++++++++++++++++------ .github/workflows/test-on-pull-request.yml | 24 ++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/test-on-pull-request.yml diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 227e4c68..be2a5001 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -1,9 +1,36 @@ name: Test Deployer + on: - push: - branches: - - pipeline-test + workflow_call: + inputs: + design: + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + AWS_SESSION_TOKEN: + required: true + GITLAB_SSH_KEY: + required: true + EC2_SSH_KEY: + required: true + THEMIS_REPO_SSH_URI: + required: true + NEXUS_USERNAME: + required: true + NEXUS_PASSWORD: + required: true + PLATFORM_RPM_URL: + required: true + GATEWAY_RELEASE: + required: true + GATEWAY_WHL_URL: + required: true + jobs: test-deployer: @@ -12,10 +39,6 @@ jobs: GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 60 - strategy: - matrix: - design: [aio, minimal, ha2, asa] - fail-fast: false steps: - name: Checkout repository @@ -130,11 +153,11 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ matrix.design }}.tfvars + run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars - name: Provision EC2 instances working-directory: ../themis/tofu_aws - run: tofu apply -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve + run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws @@ -158,7 +181,7 @@ jobs: ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common - -i inventories/${{ matrix.design }} + -i inventories/${{ inputs.design }} -e "redis_install_from_source=false" -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" -v @@ -203,4 +226,4 @@ jobs: - name: Tear down EC2 instances if: always() working-directory: ../themis/tofu_aws - run: tofu destroy -var-file=tfvars/${{ matrix.design }}.tfvars -auto-approve + run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml new file mode 100644 index 00000000..3f42c953 --- /dev/null +++ b/.github/workflows/test-on-pull-request.yml @@ -0,0 +1,24 @@ +name: Test Deployer on Pull Request + + +on: + #pull_request: + push: + branches: + - main + + +jobs: + generate-test-cases: + runs-on: ubuntu-latest + strategy: + matrix: + design: [aio, minimal, ha2, asa] + fail-fast: true + + run-tests: + needs: generate-test-cases + uses: ./.github/workflows/test-deployer.yml + with: + design: ${{ needs.generate-test-cases.outputs.design }} + secrets: inherit From 930e386b771fcb4f00291d6f8941417ec89eca78 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:32:15 -0400 Subject: [PATCH 113/144] fix --- .github/workflows/test-on-pull-request.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 3f42c953..d1283718 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -15,6 +15,10 @@ jobs: matrix: design: [aio, minimal, ha2, asa] fail-fast: true + steps: + - name: Set output design + id: set-output + run: echo "design=${{ matrix.design }}" >> $GITHUB_OUTPUT run-tests: needs: generate-test-cases From 0edfa673fb4cf43e5b06cae17da99abe38e844e2 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:33:54 -0400 Subject: [PATCH 114/144] fix --- .github/workflows/test-on-pull-request.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index d1283718..0f96c842 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -9,20 +9,20 @@ on: jobs: - generate-test-cases: - runs-on: ubuntu-latest - strategy: - matrix: - design: [aio, minimal, ha2, asa] - fail-fast: true - steps: - - name: Set output design - id: set-output - run: echo "design=${{ matrix.design }}" >> $GITHUB_OUTPUT + #generate-test-cases: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # design: [aio, minimal, ha2, asa] + # fail-fast: true + # steps: + # - name: Set output design + # id: set-output + # run: echo "design=${{ matrix.design }}" >> $GITHUB_OUTPUT run-tests: needs: generate-test-cases uses: ./.github/workflows/test-deployer.yml with: - design: ${{ needs.generate-test-cases.outputs.design }} + design: minimal secrets: inherit From 5b1f037dcfe900b860a30cfd0f24bc43fe00a9a1 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:34:23 -0400 Subject: [PATCH 115/144] fix --- .github/workflows/test-on-pull-request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 0f96c842..44cab48e 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -21,7 +21,6 @@ jobs: # run: echo "design=${{ matrix.design }}" >> $GITHUB_OUTPUT run-tests: - needs: generate-test-cases uses: ./.github/workflows/test-deployer.yml with: design: minimal From 85003f32340d53a97f82d5bdbed6ef95397d08a8 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:34:57 -0400 Subject: [PATCH 116/144] trigger --- .github/workflows/test-on-pull-request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 44cab48e..0c2a5be9 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -1,6 +1,5 @@ name: Test Deployer on Pull Request - on: #pull_request: push: From 0d287c1650a4b866ef747962e0239aea0e205b2f Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:36:54 -0400 Subject: [PATCH 117/144] fix --- .github/workflows/test-on-pull-request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 0c2a5be9..919dec15 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -1,10 +1,10 @@ name: Test Deployer on Pull Request + on: - #pull_request: push: branches: - - main + - pipeline-test jobs: From cf53067257460e28f79d5392af4292243f6d219c Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:45:17 -0400 Subject: [PATCH 118/144] test all cases --- .github/workflows/test-on-pull-request.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 919dec15..41773d55 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -8,19 +8,11 @@ on: jobs: - #generate-test-cases: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # design: [aio, minimal, ha2, asa] - # fail-fast: true - # steps: - # - name: Set output design - # id: set-output - # run: echo "design=${{ matrix.design }}" >> $GITHUB_OUTPUT - run-tests: + strategy: + matrix: + design: [aio, minimal, ha2, asa] uses: ./.github/workflows/test-deployer.yml with: - design: minimal + design: ${{ matrix.design }} secrets: inherit From f3f63191ba74cabccdbc6ea827f79c547c47612b Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:53:28 -0400 Subject: [PATCH 119/144] add more test cases --- .github/workflows/test-deployer.yml | 4 ++-- .github/workflows/test-on-pull-request.yml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index be2a5001..9e64b46e 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -153,11 +153,11 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars + run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }} -var "os_version=${{ inputs.os-version }}" - name: Provision EC2 instances working-directory: ../themis/tofu_aws - run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve + run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }} -var "os_version=${{ inputs.os-version }}" -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 41773d55..0bbef5f5 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -11,8 +11,13 @@ jobs: run-tests: strategy: matrix: - design: [aio, minimal, ha2, asa] + design: [minimal] + os-type: [rocky, rhel] + os-version: ["8", "9"] + fail-fast: true uses: ./.github/workflows/test-deployer.yml with: design: ${{ matrix.design }} + os-type: ${{ matrix.os-type }} + os-version: ${{ matrix.os-version }} secrets: inherit From 64b1a5d55354f2666bf16c7e67d0ecca0536afb8 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:54:13 -0400 Subject: [PATCH 120/144] fix --- .github/workflows/test-deployer.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 9e64b46e..1e19d718 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -7,6 +7,12 @@ on: design: required: true type: string + os-type: + required: true + type: string + os-version: + required: true + type: string secrets: AWS_ACCESS_KEY_ID: required: true From 02a48353d5835f03695435edde07a25b7b83dcd0 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 11:56:56 -0400 Subject: [PATCH 121/144] fix --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 1e19d718..f7ec2b2c 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -159,11 +159,11 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }} -var "os_version=${{ inputs.os-version }}" + run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" - name: Provision EC2 instances working-directory: ../themis/tofu_aws - run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }} -var "os_version=${{ inputs.os-version }}" -auto-approve + run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws From 42dca7e4ac1b095d2f22aadff0018c1931c1d105 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 12:03:20 -0400 Subject: [PATCH 122/144] improvements --- .github/workflows/test-deployer.yml | 4 ++-- .github/workflows/test-on-pull-request.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index f7ec2b2c..fdaadf4a 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -159,11 +159,11 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" + run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" -out=plan.tfplan - name: Provision EC2 instances working-directory: ../themis/tofu_aws - run: tofu apply -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" -auto-approve + run: tofu apply plan.tfplan -auto-approve - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 0bbef5f5..6254feaa 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -8,7 +8,7 @@ on: jobs: - run-tests: + run-test: strategy: matrix: design: [minimal] From 36b1a0c0227f0a61625cfee8e912e0a7884684ec Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 26 Sep 2025 12:06:43 -0400 Subject: [PATCH 123/144] fixes --- .github/workflows/test-deployer.yml | 2 +- .github/workflows/test-on-pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index fdaadf4a..4b95cbc8 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -163,7 +163,7 @@ jobs: - name: Provision EC2 instances working-directory: ../themis/tofu_aws - run: tofu apply plan.tfplan -auto-approve + run: tofu apply plan.tfplan - name: Generate Ansible inventory hosts file working-directory: ../themis/tofu_aws diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 6254feaa..f3a3125e 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -14,7 +14,7 @@ jobs: design: [minimal] os-type: [rocky, rhel] os-version: ["8", "9"] - fail-fast: true + fail-fast: false uses: ./.github/workflows/test-deployer.yml with: design: ${{ matrix.design }} From cb05d0e10e54bc987b019babf23ef58374a46baf Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 26 Sep 2025 12:46:17 -0400 Subject: [PATCH 124/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 4b95cbc8..cf453141 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -159,7 +159,12 @@ jobs: - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -var "os_type=${{ inputs.os-type }}" -var "os_version=${{ inputs.os-version }}" -out=plan.tfplan + run: > + tofu plan + -var-file=tfvars/${{ inputs.design }}.tfvars + -var "os_type=${{ inputs.os-type }}" + -var "os_version=${{ inputs.os-version }}" + -out=plan.tfplan - name: Provision EC2 instances working-directory: ../themis/tofu_aws @@ -229,7 +234,12 @@ jobs: python3 scripts/validate.py mongodb "$host" --arbiter done - - name: Tear down EC2 instances + - name: Terminate EC2 instances if: always() working-directory: ../themis/tofu_aws - run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve + run: > + tofu destroy + -var-file=tfvars/${{ inputs.design }}.tfvars + -var "os_type=${{ inputs.os-type }}" + -var "os_version=${{ inputs.os-version }}" + -auto-approve From 1081e2b2683b62f951b542364db457730aab49bf Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Fri, 26 Sep 2025 13:33:56 -0400 Subject: [PATCH 125/144] Update test-on-pull-request.yml --- .github/workflows/test-on-pull-request.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index f3a3125e..ee1ff7cb 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -11,13 +11,11 @@ jobs: run-test: strategy: matrix: - design: [minimal] - os-type: [rocky, rhel] - os-version: ["8", "9"] + design: [minimal, aio, ha2, asa] fail-fast: false uses: ./.github/workflows/test-deployer.yml with: design: ${{ matrix.design }} - os-type: ${{ matrix.os-type }} - os-version: ${{ matrix.os-version }} + os-type: rocky + os-version: "9" secrets: inherit From 25a8bb9dd0e6d236cb05ef6ee847f86dfcc5d29f Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 29 Sep 2025 12:40:12 -0400 Subject: [PATCH 126/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index cf453141..d0d83322 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -41,6 +41,8 @@ on: jobs: test-deployer: runs-on: self-hosted + container: + image: ghcr.io/catthehacker/ubuntu:act-latest env: GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" From 322d4d799789027dd17290ac5164ba20daed9ef4 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 30 Sep 2025 15:25:21 -0400 Subject: [PATCH 127/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 48 +++++++++-------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index d0d83322..cb6a206d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -1,3 +1,16 @@ +# Runs the Itential Deployer on EC2 instances conforming to some validated design and other parameters +# (e.g. OS type and version), and validates that the deployed services are all running correctly. +# This allows for testing deployments on various configurations in parallel in a fully automated manner. + +# Main steps performed by this workflow: +# - Cloning Themis from GitLab +# - Provisioning EC2 instances using Themis +# - Converting OpenTofu output to Ansible inventory +# - Running deployer on newly created instances +# - Running validation script to test each service (Redis, MongoDB, Platform, Gateway) +# - Terminating all instances using Themis, irrespective of success + + name: Test Deployer @@ -24,18 +37,10 @@ on: required: true EC2_SSH_KEY: required: true - THEMIS_REPO_SSH_URI: - required: true NEXUS_USERNAME: required: true NEXUS_PASSWORD: required: true - PLATFORM_RPM_URL: - required: true - GATEWAY_RELEASE: - required: true - GATEWAY_WHL_URL: - required: true jobs: @@ -116,38 +121,15 @@ jobs: - name: Clone Themis repository working-directory: .. run: | - git clone "${{ secrets.THEMIS_REPO_SSH_URI }}" + git clone git@gitlab.com:itential/platform-engineering/themis.git cd themis + git switch platform-gateway-urls ls -la - name: Install Themis Python script dependencies working-directory: ../themis run: pip install -r scripts/requirements.txt - - name: Configure inventory to download Platform from Nexus - working-directory: ../themis - run: | - cat > inventories/common/group_vars/platform.yml << 'EOF' - --- - platform_packages: - - "${{ secrets.PLATFORM_RPM_URL }}" - EOF - - cat > inventories/common/group_vars/platform_secondary.yml << 'EOF' - --- - platform_packages: - - "${{ secrets.PLATFORM_RPM_URL }}" - EOF - - - name: Configure inventory to download Gateway from Nexus - working-directory: ../themis - run: | - cat > inventories/common/group_vars/gateway.yml << 'EOF' - --- - gateway_release: ${{ secrets.GATEWAY_RELEASE }} - gateway_archive_download_url: "${{ secrets.GATEWAY_WHL_URL }}" - EOF - # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command - name: Add Nexus credentials to inventory working-directory: ../themis From 7f9905a15d20dde894ad07f4e013fb955e2f8e16 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 30 Sep 2025 15:56:24 -0400 Subject: [PATCH 128/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index cb6a206d..b38bb40f 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -141,6 +141,15 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init + - name: Create .tfvars file with additional variables from workflow inputs + working-directory: ../themis/tofu_aws + run: | + cat > workflow_inputs.tfvars << 'EOF' + prefix = "github" + os_type = "${{ inputs.os-type }}" + os_version = "${{ inputs.os-version }}" + EOF + - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws run: > From d0f8e592d334105466b890612aceceeb33a5c696 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Tue, 30 Sep 2025 16:04:11 -0400 Subject: [PATCH 129/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index b38bb40f..803c7da7 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -141,10 +141,10 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init - - name: Create .tfvars file with additional variables from workflow inputs + - name: Create .tfvars file with additional workflow-specific variables working-directory: ../themis/tofu_aws run: | - cat > workflow_inputs.tfvars << 'EOF' + cat > github_actions.tfvars << 'EOF' prefix = "github" os_type = "${{ inputs.os-type }}" os_version = "${{ inputs.os-version }}" @@ -155,8 +155,7 @@ jobs: run: > tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars - -var "os_type=${{ inputs.os-type }}" - -var "os_version=${{ inputs.os-version }}" + -var-file=github_actions.tfvars -out=plan.tfplan - name: Provision EC2 instances @@ -233,6 +232,5 @@ jobs: run: > tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars - -var "os_type=${{ inputs.os-type }}" - -var "os_version=${{ inputs.os-version }}" + -var-file=github_actions.tfvars -auto-approve From 61cedde8c6b427346c8a706010703bc34738b5fb Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Wed, 1 Oct 2025 14:25:51 -0400 Subject: [PATCH 130/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 803c7da7..0e388d22 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -123,7 +123,7 @@ jobs: run: | git clone git@gitlab.com:itential/platform-engineering/themis.git cd themis - git switch platform-gateway-urls + git switch differentiated-instance-names ls -la - name: Install Themis Python script dependencies @@ -141,22 +141,18 @@ jobs: working-directory: ../themis/tofu_aws run: tofu init - - name: Create .tfvars file with additional workflow-specific variables + - name: Create .tfvars file with additional job-specific variables working-directory: ../themis/tofu_aws run: | - cat > github_actions.tfvars << 'EOF' - prefix = "github" + cat > github.auto.tfvars << 'EOF' + owner = "github" os_type = "${{ inputs.os-type }}" os_version = "${{ inputs.os-version }}" EOF - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws - run: > - tofu plan - -var-file=tfvars/${{ inputs.design }}.tfvars - -var-file=github_actions.tfvars - -out=plan.tfplan + run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -out=plan.tfplan - name: Provision EC2 instances working-directory: ../themis/tofu_aws @@ -229,8 +225,4 @@ jobs: - name: Terminate EC2 instances if: always() working-directory: ../themis/tofu_aws - run: > - tofu destroy - -var-file=tfvars/${{ inputs.design }}.tfvars - -var-file=github_actions.tfvars - -auto-approve + run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve From 3d7f19b8e0d72dd30f0cdd5587f221ac7c6849f4 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 13:39:29 -0400 Subject: [PATCH 131/144] Update test-deployer.yml (3 jobs test) --- .github/workflows/test-deployer.yml | 182 ++++++++++++++++++---------- 1 file changed, 120 insertions(+), 62 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 0e388d22..a0b53f73 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -44,30 +44,18 @@ on: jobs: - test-deployer: - runs-on: self-hosted - container: - image: ghcr.io/catthehacker/ubuntu:act-latest - env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no - PIP_BREAK_SYSTEM_PACKAGES: "1" - timeout-minutes: 60 + provision: + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + themis-artifact-name: ${{ steps.upload.outputs.artifact-name }} steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Update package lists - run: sudo apt update - - name: Install Python run: | - sudo apt install -y python3 - wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py - sudo ln -s /usr/bin/python3 /usr/bin/python || true - sudo ln -s /usr/bin/pip3 /usr/bin/pip || true - python --version - pip --version + sudo apt install -y python3 python3-pip + python3 --version + pip3 --version # For some reason, Node appears to be a dependency of Themis - name: Install Node.js @@ -80,25 +68,6 @@ jobs: with: tofu_version: latest - - name: Install Ansible - run: | - pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" - ansible --version - - # Configures Ansible to fail immediately on error, skip host key checking, use correct key file - - name: Write Ansible configuration file - run: | - cat > ~/.ansible.cfg << 'EOF' - [defaults] - any_errors_fatal = True - host_key_checking = False - max_fail_percentage = 0 - private_key_file = ~/.ssh/pet-east1.open.pem - EOF - - - name: Install this collection - run: ansible-galaxy collection install . --force - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -112,14 +81,10 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - - - name: Setup SSH for EC2 - run: | - echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository - working-directory: .. + env: + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no run: | git clone git@gitlab.com:itential/platform-engineering/themis.git cd themis @@ -127,22 +92,22 @@ jobs: ls -la - name: Install Themis Python script dependencies - working-directory: ../themis - run: pip install -r scripts/requirements.txt + working-directory: themis + run: pip3 install -r scripts/requirements.txt # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command - name: Add Nexus credentials to inventory - working-directory: ../themis + working-directory: themis run: | echo 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml echo 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml - name: Initialize OpenTofu - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: tofu init - name: Create .tfvars file with additional job-specific variables - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: | cat > github.auto.tfvars << 'EOF' owner = "github" @@ -151,31 +116,99 @@ jobs: EOF - name: Generate OpenTofu execution plan - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -out=plan.tfplan - name: Provision EC2 instances - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: tofu apply plan.tfplan - name: Generate Ansible inventory hosts file - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: python3 ../scripts/generate_inventory.py --validate -o hosts.json + - name: Upload Themis workspace as artifact + id: upload + uses: actions/upload-artifact@v4 + with: + name: themis-${{ github.run_id }}-${{ github.run_attempt }} + path: themis/ + retention-days: 1 + + deploy: + needs: provision + runs-on: self-hosted + container: + image: ghcr.io/catthehacker/ubuntu:act-latest + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + timeout-minutes: 40 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Update package lists + run: sudo apt update + + - name: Install Python + run: | + sudo apt install -y python3 + wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py + sudo ln -s /usr/bin/python3 /usr/bin/python || true + sudo ln -s /usr/bin/pip3 /usr/bin/pip || true + python --version + pip --version + + - name: Install Ansible + run: | + pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" + ansible --version + + # Configures Ansible to fail immediately on error, skip host key checking, use correct key file + - name: Write Ansible configuration file + run: | + cat > ~/.ansible.cfg << 'EOF' + [defaults] + any_errors_fatal = True + host_key_checking = False + max_fail_percentage = 0 + private_key_file = ~/.ssh/pet-east1.open.pem + EOF + + - name: Install this collection + run: ansible-galaxy collection install . --force + + - name: Setup SSH for EC2 + run: | + mkdir -p ~/.ssh + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + chmod 600 ~/.ssh/pet-east1.open.pem + + - name: Download Themis workspace artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.provision.outputs.themis-artifact-name }} + path: themis/ + + - name: Install Themis Python script dependencies + working-directory: themis + run: pip install -r scripts/requirements.txt + # Retries until SSH connection is established or timeout is reached - name: Wait for EC2 instances to be ready to SSH into - working-directory: ../themis + working-directory: themis run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v # Waits for cloud init marker file to be written to disk (see cloud-init.tpl) - name: Wait for cloud init script to complete - working-directory: ../themis + working-directory: themis run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v # Overrides inventory variables to install Redis from the Remi repository using a known working URL, as a # workaround for bugs in the deployer (dependency resolution errors when building Redis, incorrect Remi URL) - name: Run the deployer - working-directory: ../themis + working-directory: themis run: > ansible-playbook itential.deployer.site -i tofu_aws/hosts.json @@ -186,7 +219,7 @@ jobs: -v - name: Verify that Platform is running correctly - working-directory: ../themis + working-directory: themis run: | for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py platform "http://$host:3000" @@ -196,14 +229,14 @@ jobs: done - name: Verify that Gateway is running correctly - working-directory: ../themis + working-directory: themis run: | for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py gateway "http://$host:8083" done - name: Verify that Redis is running correctly - working-directory: ../themis + working-directory: themis run: | for host in $(jq -r '.all.children.redis.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py redis "$host" @@ -213,7 +246,7 @@ jobs: done - name: Verify that MongoDB is running correctly - working-directory: ../themis + working-directory: themis run: | for host in $(jq -r '.all.children.mongodb.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py mongodb "$host" @@ -222,7 +255,32 @@ jobs: python3 scripts/validate.py mongodb "$host" --arbiter done + cleanup: + needs: [provision, deploy] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Install OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: latest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} + aws-region: us-east-1 + + - name: Download Themis workspace artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.provision.outputs.themis-artifact-name }} + path: themis/ + - name: Terminate EC2 instances - if: always() - working-directory: ../themis/tofu_aws + working-directory: themis/tofu_aws run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve From f916288d49bd11a8e2dd70e09832d22b37616055 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 13:43:42 -0400 Subject: [PATCH 132/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a0b53f73..9b353a90 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -131,7 +131,7 @@ jobs: id: upload uses: actions/upload-artifact@v4 with: - name: themis-${{ github.run_id }}-${{ github.run_attempt }} + name: themis-${{ inputs.design }}-${{ inputs.os-type }}-${{ inputs.os-version }}-${{ github.run_id }}-${{ github.run_attempt }} path: themis/ retention-days: 1 From 8613bfef65c9df20ba3dd1d524cf31fab7e0d7d3 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 13:53:06 -0400 Subject: [PATCH 133/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 9b353a90..a1fbd678 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -189,7 +189,10 @@ jobs: uses: actions/download-artifact@v4 with: name: ${{ needs.provision.outputs.themis-artifact-name }} - path: themis/ + path: downloaded-artifact + + - name: Move Themis to expected location + run: mv downloaded-artifact/themis ./themis - name: Install Themis Python script dependencies working-directory: themis @@ -279,7 +282,10 @@ jobs: uses: actions/download-artifact@v4 with: name: ${{ needs.provision.outputs.themis-artifact-name }} - path: themis/ + path: downloaded-artifact + + - name: Move Themis to expected location + run: mv downloaded-artifact/themis ./themis - name: Terminate EC2 instances working-directory: themis/tofu_aws From 436e3d6690d1ea53c9436807b3644096ec10a23a Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 13:58:39 -0400 Subject: [PATCH 134/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index a1fbd678..87885f6b 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -192,7 +192,7 @@ jobs: path: downloaded-artifact - name: Move Themis to expected location - run: mv downloaded-artifact/themis ./themis + run: mv downloaded-artifact/${{ needs.provision.outputs.themis-artifact-name }}/themis ./themis - name: Install Themis Python script dependencies working-directory: themis @@ -285,7 +285,7 @@ jobs: path: downloaded-artifact - name: Move Themis to expected location - run: mv downloaded-artifact/themis ./themis + run: mv downloaded-artifact/${{ needs.provision.outputs.themis-artifact-name }}/themis ./themis - name: Terminate EC2 instances working-directory: themis/tofu_aws From 4d2732c79a5a27f118b694bd843e387c77aec985 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 14:05:03 -0400 Subject: [PATCH 135/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 87885f6b..bec29f0d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -48,9 +48,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 outputs: - themis-artifact-name: ${{ steps.upload.outputs.artifact-name }} + themis-artifact-name: ${{ steps.set-output.outputs.artifact-name }} steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Python run: | sudo apt install -y python3 python3-pip @@ -81,6 +84,11 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa + + - name: Setup SSH for EC2 + run: | + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository env: @@ -135,12 +143,17 @@ jobs: path: themis/ retention-days: 1 + - name: Set artifact name output + id: set-output + run: echo "artifact-name=themis-${{ inputs.design }}-${{ inputs.os-type }}-${{ inputs.os-version }}-${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT + deploy: needs: provision runs-on: self-hosted container: image: ghcr.io/catthehacker/ubuntu:act-latest env: + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no PIP_BREAK_SYSTEM_PACKAGES: "1" timeout-minutes: 40 From f3e68e345e76e91e8a0a9ad6e3c29a8dec909dc5 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 14:12:30 -0400 Subject: [PATCH 136/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index bec29f0d..46189733 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -205,7 +205,7 @@ jobs: path: downloaded-artifact - name: Move Themis to expected location - run: mv downloaded-artifact/${{ needs.provision.outputs.themis-artifact-name }}/themis ./themis + run: mv downloaded-artifact themis - name: Install Themis Python script dependencies working-directory: themis @@ -298,7 +298,7 @@ jobs: path: downloaded-artifact - name: Move Themis to expected location - run: mv downloaded-artifact/${{ needs.provision.outputs.themis-artifact-name }}/themis ./themis + run: mv downloaded-artifact themis - name: Terminate EC2 instances working-directory: themis/tofu_aws From c71d70dd306bd03e91acfc2eb4e0b11e569e83f8 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 14:31:00 -0400 Subject: [PATCH 137/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 46189733..c68f39f6 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -300,6 +300,10 @@ jobs: - name: Move Themis to expected location run: mv downloaded-artifact themis + - name: Initialize OpenTofu + working-directory: themis + run: tofu init + - name: Terminate EC2 instances working-directory: themis/tofu_aws run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve From 4512727f2cd2bcd623487d30de36e82426989249 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Thu, 2 Oct 2025 14:41:30 -0400 Subject: [PATCH 138/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index c68f39f6..be623384 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -301,7 +301,7 @@ jobs: run: mv downloaded-artifact themis - name: Initialize OpenTofu - working-directory: themis + working-directory: themis/tofu_aws run: tofu init - name: Terminate EC2 instances From 1a3858e89fe0527505197d1aa4e927fd838a2463 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 6 Oct 2025 11:48:27 -0400 Subject: [PATCH 139/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 209 +++++++++------------------- 1 file changed, 64 insertions(+), 145 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index be623384..bd5f4ce3 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -44,21 +44,30 @@ on: jobs: - provision: - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - themis-artifact-name: ${{ steps.set-output.outputs.artifact-name }} + test-deployer: + runs-on: self-hosted + container: + image: ghcr.io/catthehacker/ubuntu:act-latest + env: + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + PIP_BREAK_SYSTEM_PACKAGES: "1" + timeout-minutes: 60 steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Update package lists + run: sudo apt update + - name: Install Python run: | - sudo apt install -y python3 python3-pip - python3 --version - pip3 --version + sudo apt install -y python3 + wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py + sudo ln -s /usr/bin/python3 /usr/bin/python || true + sudo ln -s /usr/bin/pip3 /usr/bin/pip || true + python --version + pip --version # For some reason, Node appears to be a dependency of Themis - name: Install Node.js @@ -71,6 +80,25 @@ jobs: with: tofu_version: latest + - name: Install Ansible + run: | + pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" + ansible --version + + # Configures Ansible to fail immediately on error, skip host key checking, use correct key file + - name: Write Ansible configuration file + run: | + cat > ~/.ansible.cfg << 'EOF' + [defaults] + any_errors_fatal = True + host_key_checking = False + max_fail_percentage = 0 + private_key_file = ~/.ssh/pet-east1.open.pem + EOF + + - name: Install this collection + run: ansible-galaxy collection install . --force + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -91,151 +119,74 @@ jobs: chmod 600 ~/.ssh/pet-east1.open.pem - name: Clone Themis repository - env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + working-directory: .. run: | git clone git@gitlab.com:itential/platform-engineering/themis.git cd themis - git switch differentiated-instance-names ls -la - name: Install Themis Python script dependencies - working-directory: themis - run: pip3 install -r scripts/requirements.txt + working-directory: ../themis + run: pip install -r scripts/requirements.txt # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command - name: Add Nexus credentials to inventory - working-directory: themis + working-directory: ../themis run: | echo 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml echo 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml - name: Initialize OpenTofu - working-directory: themis/tofu_aws + working-directory: ../themis/tofu_aws run: tofu init + # Copies the vars file for the selected design to the working directory to be imported automatically + - name: Set validated design + working-directory: ../themis/tofu_aws + run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars + - name: Create .tfvars file with additional job-specific variables - working-directory: themis/tofu_aws + working-directory: ../themis/tofu_aws run: | - cat > github.auto.tfvars << 'EOF' - owner = "github" + cat > github_actions.auto.tfvars << 'EOF' + prefix = "github" os_type = "${{ inputs.os-type }}" os_version = "${{ inputs.os-version }}" EOF - name: Generate OpenTofu execution plan - working-directory: themis/tofu_aws - run: tofu plan -var-file=tfvars/${{ inputs.design }}.tfvars -out=plan.tfplan + working-directory: ../themis/tofu_aws + run: tofu plan -out=plan.tfplan - name: Provision EC2 instances - working-directory: themis/tofu_aws + working-directory: ../themis/tofu_aws run: tofu apply plan.tfplan - name: Generate Ansible inventory hosts file - working-directory: themis/tofu_aws + working-directory: ../themis/tofu_aws run: python3 ../scripts/generate_inventory.py --validate -o hosts.json - - name: Upload Themis workspace as artifact - id: upload - uses: actions/upload-artifact@v4 - with: - name: themis-${{ inputs.design }}-${{ inputs.os-type }}-${{ inputs.os-version }}-${{ github.run_id }}-${{ github.run_attempt }} - path: themis/ - retention-days: 1 - - - name: Set artifact name output - id: set-output - run: echo "artifact-name=themis-${{ inputs.design }}-${{ inputs.os-type }}-${{ inputs.os-version }}-${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT - - deploy: - needs: provision - runs-on: self-hosted - container: - image: ghcr.io/catthehacker/ubuntu:act-latest - env: - GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no - PIP_BREAK_SYSTEM_PACKAGES: "1" - timeout-minutes: 40 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Update package lists - run: sudo apt update - - - name: Install Python - run: | - sudo apt install -y python3 - wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py - sudo ln -s /usr/bin/python3 /usr/bin/python || true - sudo ln -s /usr/bin/pip3 /usr/bin/pip || true - python --version - pip --version - - - name: Install Ansible - run: | - pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" - ansible --version - - # Configures Ansible to fail immediately on error, skip host key checking, use correct key file - - name: Write Ansible configuration file - run: | - cat > ~/.ansible.cfg << 'EOF' - [defaults] - any_errors_fatal = True - host_key_checking = False - max_fail_percentage = 0 - private_key_file = ~/.ssh/pet-east1.open.pem - EOF - - - name: Install this collection - run: ansible-galaxy collection install . --force - - - name: Setup SSH for EC2 - run: | - mkdir -p ~/.ssh - echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem - chmod 600 ~/.ssh/pet-east1.open.pem - - - name: Download Themis workspace artifact - uses: actions/download-artifact@v4 - with: - name: ${{ needs.provision.outputs.themis-artifact-name }} - path: downloaded-artifact - - - name: Move Themis to expected location - run: mv downloaded-artifact themis - - - name: Install Themis Python script dependencies - working-directory: themis - run: pip install -r scripts/requirements.txt - # Retries until SSH connection is established or timeout is reached - name: Wait for EC2 instances to be ready to SSH into - working-directory: themis + working-directory: ../themis run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v # Waits for cloud init marker file to be written to disk (see cloud-init.tpl) - name: Wait for cloud init script to complete - working-directory: themis + working-directory: ../themis run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v - - # Overrides inventory variables to install Redis from the Remi repository using a known working URL, as a - # workaround for bugs in the deployer (dependency resolution errors when building Redis, incorrect Remi URL) + - name: Run the deployer - working-directory: themis + working-directory: ../themis run: > ansible-playbook itential.deployer.site -i tofu_aws/hosts.json -i inventories/common -i inventories/${{ inputs.design }} - -e "redis_install_from_source=false" - -e "redis_remi_repo_url=http://rpms.remirepo.net/enterprise/remi-release-9.rpm" -v - name: Verify that Platform is running correctly - working-directory: themis + working-directory: ../themis run: | for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py platform "http://$host:3000" @@ -245,14 +196,14 @@ jobs: done - name: Verify that Gateway is running correctly - working-directory: themis + working-directory: ../themis run: | for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py gateway "http://$host:8083" done - name: Verify that Redis is running correctly - working-directory: themis + working-directory: ../themis run: | for host in $(jq -r '.all.children.redis.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py redis "$host" @@ -262,7 +213,7 @@ jobs: done - name: Verify that MongoDB is running correctly - working-directory: themis + working-directory: ../themis run: | for host in $(jq -r '.all.children.mongodb.hosts[] | .ansible_host' tofu_aws/hosts.json); do python3 scripts/validate.py mongodb "$host" @@ -271,39 +222,7 @@ jobs: python3 scripts/validate.py mongodb "$host" --arbiter done - cleanup: - needs: [provision, deploy] - if: always() - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - name: Install OpenTofu - uses: opentofu/setup-opentofu@v1 - with: - tofu_version: latest - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} - aws-region: us-east-1 - - - name: Download Themis workspace artifact - uses: actions/download-artifact@v4 - with: - name: ${{ needs.provision.outputs.themis-artifact-name }} - path: downloaded-artifact - - - name: Move Themis to expected location - run: mv downloaded-artifact themis - - - name: Initialize OpenTofu - working-directory: themis/tofu_aws - run: tofu init - - name: Terminate EC2 instances - working-directory: themis/tofu_aws - run: tofu destroy -var-file=tfvars/${{ inputs.design }}.tfvars -auto-approve + if: always() + working-directory: ../themis/tofu_aws + run: tofu destroy -auto-approve From 81ecc34f3c125c3957dc5b891af52b468d7a6f59 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 6 Oct 2025 11:58:08 -0400 Subject: [PATCH 140/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index bd5f4ce3..d488c55e 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -143,7 +143,7 @@ jobs: # Copies the vars file for the selected design to the working directory to be imported automatically - name: Set validated design working-directory: ../themis/tofu_aws - run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars + run: cp tfvars/${{ inputs.design }}.tfvars ./design.auto.tfvars - name: Create .tfvars file with additional job-specific variables working-directory: ../themis/tofu_aws @@ -154,6 +154,10 @@ jobs: os_version = "${{ inputs.os-version }}" EOF + - name: Verify directory structure + working-directory: ../themis/tofu_aws + run: ls -la + - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws run: tofu plan -out=plan.tfplan From 8fc758a41aa5fd31a52a45062bb73a57c0db7e67 Mon Sep 17 00:00:00 2001 From: hamza-itential Date: Mon, 6 Oct 2025 12:02:05 -0400 Subject: [PATCH 141/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index d488c55e..42e5e001 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -143,21 +143,17 @@ jobs: # Copies the vars file for the selected design to the working directory to be imported automatically - name: Set validated design working-directory: ../themis/tofu_aws - run: cp tfvars/${{ inputs.design }}.tfvars ./design.auto.tfvars + run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars - name: Create .tfvars file with additional job-specific variables working-directory: ../themis/tofu_aws run: | cat > github_actions.auto.tfvars << 'EOF' - prefix = "github" + owner = "github" os_type = "${{ inputs.os-type }}" os_version = "${{ inputs.os-version }}" EOF - - name: Verify directory structure - working-directory: ../themis/tofu_aws - run: ls -la - - name: Generate OpenTofu execution plan working-directory: ../themis/tofu_aws run: tofu plan -out=plan.tfplan @@ -179,7 +175,7 @@ jobs: - name: Wait for cloud init script to complete working-directory: ../themis run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v - + - name: Run the deployer working-directory: ../themis run: > From ac9e2414f2ef0dc2cf3fd06e7d94b966d6055ecb Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 10 Oct 2025 11:55:03 -0400 Subject: [PATCH 142/144] Use pull_request_target trigger --- .github/workflows/test-deployer.yml | 7 ++++++- .github/workflows/test-on-pull-request.yml | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index 42e5e001..ad0b527d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -17,6 +17,9 @@ name: Test Deployer on: workflow_call: inputs: + ref: + required: true + type: string design: required: true type: string @@ -56,6 +59,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} - name: Update package lists run: sudo apt update @@ -145,7 +150,7 @@ jobs: working-directory: ../themis/tofu_aws run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars - - name: Create .tfvars file with additional job-specific variables + - name: Set additional job-specific variables working-directory: ../themis/tofu_aws run: | cat > github_actions.auto.tfvars << 'EOF' diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index ee1ff7cb..4302ea2d 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -2,9 +2,9 @@ name: Test Deployer on Pull Request on: - push: + pull_request_target: branches: - - pipeline-test + - main jobs: @@ -15,6 +15,7 @@ jobs: fail-fast: false uses: ./.github/workflows/test-deployer.yml with: + ref: ${{ github.event.pull_request.head.sha }} design: ${{ matrix.design }} os-type: rocky os-version: "9" From 40df0e0990d7e6f31fa7c377e19b8586b5d2db68 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 10 Oct 2025 12:00:34 -0400 Subject: [PATCH 143/144] Update test-on-pull-request.yml --- .github/workflows/test-on-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml index 4302ea2d..3b4899d4 100644 --- a/.github/workflows/test-on-pull-request.yml +++ b/.github/workflows/test-on-pull-request.yml @@ -4,7 +4,7 @@ name: Test Deployer on Pull Request on: pull_request_target: branches: - - main + - pipeline-test jobs: From 78a96795123110d4ea85dc83fc404bdca56d9de6 Mon Sep 17 00:00:00 2001 From: Hamza Qadri Date: Fri, 10 Oct 2025 12:42:57 -0400 Subject: [PATCH 144/144] Update test-deployer.yml --- .github/workflows/test-deployer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml index ad0b527d..3392008d 100644 --- a/.github/workflows/test-deployer.yml +++ b/.github/workflows/test-deployer.yml @@ -154,6 +154,7 @@ jobs: working-directory: ../themis/tofu_aws run: | cat > github_actions.auto.tfvars << 'EOF' + profile = "" owner = "github" os_type = "${{ inputs.os-type }}" os_version = "${{ inputs.os-version }}"