From d55711f1500f9db11db3d6df5ad6d027ea98c882 Mon Sep 17 00:00:00 2001 From: anero Date: Wed, 26 Feb 2025 16:53:40 -0300 Subject: [PATCH 1/2] Remove logging to Cloudwatch from EC2 instances --- .buildkite/steps/cleanup.sh | 3 +-- README.md | 7 +------ goss.yaml | 4 ---- packer/linux/buildkite-ami.pkr.hcl | 6 +----- .../linux/conf/bin/bk-install-elastic-stack.sh | 3 +-- .../buildkite-agent/scripts/terminate-instance | 3 --- .../linux/scripts/install-cloudwatch-agent.sh | 15 --------------- packer/windows/buildkite-ami.pkr.hcl | 4 ---- .../scripts/install-cloudwatch-agent.ps1 | 18 ------------------ templates/aws-stack.yml | 1 - 10 files changed, 4 insertions(+), 60 deletions(-) delete mode 100755 packer/linux/scripts/install-cloudwatch-agent.sh delete mode 100644 packer/windows/scripts/install-cloudwatch-agent.ps1 diff --git a/.buildkite/steps/cleanup.sh b/.buildkite/steps/cleanup.sh index 7a14fa4f4..fd76238ed 100755 --- a/.buildkite/steps/cleanup.sh +++ b/.buildkite/steps/cleanup.sh @@ -4,8 +4,7 @@ set -uxo pipefail ######################################################## # We make an attempt to clean up the specific resources created during this pipeline elsewhere. However -# sometimes that fails, are various CloudFormation Stacks, S3 Buckets and CloudWatch Log Groups can be -# left around. +# sometimes that fails, are various CloudFormation Stacks and S3 Buckets left around. # # This is a safety net at the end of the build that attempts to delete any resources created by this # pipeline more than a day ago, regardless of if they were created in the current build or not. diff --git a/README.md b/README.md index b45c43eb7..4873786a9 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ AWS CLI | ✅ | ✅ S3 Secrets Bucket | ✅ | ✅ ECR Login | ✅ | ✅ Docker Login | ✅ | ✅ -CloudWatch Logs Agent | ✅ | ✅ Per-Instance Bootstrap Script | ✅ | ✅ SSM Access | ✅ | ✅ Instance Storage (NVMe) | ✅ | @@ -176,12 +175,8 @@ aws cloudformation describe-stacks --stack-name MY_STACK_NAME \ --query 'Stacks[].Parameters[].[ParameterKey,ParameterValue]' --output table ``` ### Collect logs from CloudWatch -Provide us with logs from CloudWatch Logs: +It's currently not possible to disable logging to Cloudwatch from EC2 instances, so we removed the agent setup on our fork to avoid this. -``` -/buildkite/elastic-stack/{instance-id} -/buildkite/system/{instance-id} -``` ### Collect logs via script An alternative method to collect the logs is to use the `log-collector` script in the `utils` folder. The script will collect CloudWatch logs for the Instance, Lambda function, and AutoScaling activity and package them in a diff --git a/goss.yaml b/goss.yaml index 0d76ed40d..5e5d012a9 100644 --- a/goss.yaml +++ b/goss.yaml @@ -50,10 +50,6 @@ port: - "::" service: - amazon-cloudwatch-agent: - enabled: true - running: true - amazon-ssm-agent: enabled: true running: true diff --git a/packer/linux/buildkite-ami.pkr.hcl b/packer/linux/buildkite-ami.pkr.hcl index fabf10e0d..2a367b624 100644 --- a/packer/linux/buildkite-ami.pkr.hcl +++ b/packer/linux/buildkite-ami.pkr.hcl @@ -65,7 +65,7 @@ source "amazon-ebs" "elastic-ci-stack-ami" { } launch_block_device_mappings { - volume_type = "gp3" + volume_type = "gp3" device_name = "/dev/xvda" volume_size = 10 delete_on_termination = true @@ -108,10 +108,6 @@ build { script = "scripts/install-utils.sh" } - provisioner "shell" { - script = "scripts/install-cloudwatch-agent.sh" - } - provisioner "shell" { script = "scripts/install-docker.sh" } diff --git a/packer/linux/conf/bin/bk-install-elastic-stack.sh b/packer/linux/conf/bin/bk-install-elastic-stack.sh index 08761c98d..7bab23ab4 100755 --- a/packer/linux/conf/bin/bk-install-elastic-stack.sh +++ b/packer/linux/conf/bin/bk-install-elastic-stack.sh @@ -87,8 +87,7 @@ PLUGINS_ENABLED=() echo "The following plugins will be enabled: ${PLUGINS_ENABLED[*]-}" # cfn-env is sourced by the environment hook in builds -# DO NOT PUT SECRETES IN HERE, they will appear in both the cloudwatch and -# build logs, and the agent's log redactor will not be able to redact them. +# DO NOT PUT SECRETES IN HERE, they will appear in build logs. # We will create it in two steps so that we don't need to go crazy with quoting and escaping. The # first sets up a helper function, the second populates the default values for some environment diff --git a/packer/linux/conf/buildkite-agent/scripts/terminate-instance b/packer/linux/conf/buildkite-agent/scripts/terminate-instance index beb1a9814..fc7889f38 100755 --- a/packer/linux/conf/buildkite-agent/scripts/terminate-instance +++ b/packer/linux/conf/buildkite-agent/scripts/terminate-instance @@ -16,9 +16,6 @@ mark_as_unhealthy() { --health-status Unhealthy } -echo "sleeping for 10 seconds before terminating instance to allow agent logs to drain to cloudwatch..." -sleep 10 - token=$( curl \ --fail --silent --show-error \ diff --git a/packer/linux/scripts/install-cloudwatch-agent.sh b/packer/linux/scripts/install-cloudwatch-agent.sh deleted file mode 100755 index db440e703..000000000 --- a/packer/linux/scripts/install-cloudwatch-agent.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "Installing cloudwatch agent..." -sudo dnf install -yq amazon-cloudwatch-agent - -echo "Adding amazon-cloudwatch-agent config..." -sudo cp /tmp/conf/cloudwatch-agent/config.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json - -echo "Configuring amazon-cloudwatch-agent to start at boot" -sudo systemctl enable amazon-cloudwatch-agent - -# These will send some systemctl service logs (like the buildkite agent and docker) to logfiles -echo "Adding rsyslogd configs..." -sudo cp /tmp/conf/cloudwatch-agent/rsyslog.d/* /etc/rsyslog.d/ diff --git a/packer/windows/buildkite-ami.pkr.hcl b/packer/windows/buildkite-ami.pkr.hcl index 609f345b3..a0e1778ee 100644 --- a/packer/windows/buildkite-ami.pkr.hcl +++ b/packer/windows/buildkite-ami.pkr.hcl @@ -88,10 +88,6 @@ build { script = "scripts/install-utils.ps1" } - provisioner "powershell" { - script = "scripts/install-cloudwatch-agent.ps1" - } - provisioner "powershell" { script = "scripts/install-lifecycled.ps1" } diff --git a/packer/windows/scripts/install-cloudwatch-agent.ps1 b/packer/windows/scripts/install-cloudwatch-agent.ps1 deleted file mode 100644 index e04a63696..000000000 --- a/packer/windows/scripts/install-cloudwatch-agent.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Stop script execution when a non-terminating error occurs -$ErrorActionPreference = "Stop" - -Write-Output "Downloading amazon cloudwatch agent..." -Invoke-WebRequest -OutFile C:\packer-temp\amazon-cloudwatch-agent.msi -Uri "https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi" - -Write-Output "Installing amazon cloudwatch agent..." -Start-Process C:\packer-temp\amazon-cloudwatch-agent.msi -Wait - -Write-Output "Setting amazon cloudwatch agent start type to delayed-auto..." -sc.exe config AmazonCloudWatchAgent start= delayed-auto -If ($lastexitcode -ne 0) { Exit $lastexitcode } - -Write-Output "Copying amazon cloudwatch agent config..." -Copy-Item -Path C:\packer-temp\conf\cloudwatch-agent\amazon-cloudwatch-agent.json -Destination C:\ProgramData\Amazon\AmazonCloudWatchAgent - -Write-Output "Starting amazon cloudwatch agent..." -& 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -c file:C:\ProgramData\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent.json -s diff --git a/templates/aws-stack.yml b/templates/aws-stack.yml index e70552d64..5940bb45e 100644 --- a/templates/aws-stack.yml +++ b/templates/aws-stack.yml @@ -1109,7 +1109,6 @@ Resources: - Effect: Allow Action: - autoscaling:DescribeAutoScalingInstances - - cloudwatch:PutMetricData - cloudformation:DescribeStackResource - ec2:DescribeTags Resource: "*" From 4c376744a583c45d6b348bb479292ee5a98b85e7 Mon Sep 17 00:00:00 2001 From: anero Date: Wed, 26 Feb 2025 17:23:03 -0300 Subject: [PATCH 2/2] Fix typo --- .buildkite/steps/cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/cleanup.sh b/.buildkite/steps/cleanup.sh index fd76238ed..6e4ef1d34 100755 --- a/.buildkite/steps/cleanup.sh +++ b/.buildkite/steps/cleanup.sh @@ -4,7 +4,7 @@ set -uxo pipefail ######################################################## # We make an attempt to clean up the specific resources created during this pipeline elsewhere. However -# sometimes that fails, are various CloudFormation Stacks and S3 Buckets left around. +# sometimes that fails, and various CloudFormation Stacks and S3 Buckets left around. # # This is a safety net at the end of the build that attempts to delete any resources created by this # pipeline more than a day ago, regardless of if they were created in the current build or not.