Remove logging to Cloudwatch from EC2 instances

.buildkite/steps/cleanup.sh

@@ -4,8 +4,7 @@ set -uxo pipefail
 
 ########################################################
 # We make an attempt to clean up the specific resources created during this pipeline elsewhere. However
-# sometimes that fails, are various CloudFormation Stacks, S3 Buckets and CloudWatch Log Groups can be
-# left around.
+# sometimes that fails, and various CloudFormation Stacks and S3 Buckets left around.
 #
 # This is a safety net at the end of the build that attempts to delete any resources created by this
 # pipeline more than a day ago, regardless of if they were created in the current build or not.

README.md

@@ -47,7 +47,6 @@ AWS CLI | ✅ | ✅
 S3 Secrets Bucket | ✅ | ✅
 ECR Login | ✅ | ✅
 Docker Login | ✅ | ✅
-CloudWatch Logs Agent | ✅ | ✅
 Per-Instance Bootstrap Script | ✅ | ✅
 SSM Access | ✅ | ✅
 Instance Storage (NVMe) | ✅ |
@@ -176,12 +175,8 @@ aws cloudformation describe-stacks --stack-name MY_STACK_NAME \
   --query 'Stacks[].Parameters[].[ParameterKey,ParameterValue]' --output table
 ```
 ### Collect logs from CloudWatch
-Provide us with logs from CloudWatch Logs:
+It's currently not possible to disable logging to Cloudwatch from EC2 instances, so we removed the agent setup on our fork to avoid this.
 
-```
-/buildkite/elastic-stack/{instance-id}
-/buildkite/system/{instance-id}
-```
 ### Collect logs via script
 An alternative method to collect the logs is to use the `log-collector` script in the `utils` folder.
 The script will collect CloudWatch logs for the Instance, Lambda function, and AutoScaling activity and package them in a

goss.yaml

@@ -50,10 +50,6 @@ port:
     - "::"
 
 service:
-  amazon-cloudwatch-agent:
-    enabled: true
-    running: true
-
   amazon-ssm-agent:
     enabled: true
     running: true

packer/linux/buildkite-ami.pkr.hcl

@@ -65,7 +65,7 @@ source "amazon-ebs" "elastic-ci-stack-ami" {
   }
 
   launch_block_device_mappings {
-    volume_type           = "gp3"    
+    volume_type           = "gp3"
     device_name = "/dev/xvda"
     volume_size = 10
     delete_on_termination = true
@@ -108,10 +108,6 @@ build {
     script = "scripts/install-utils.sh"
   }
 
-  provisioner "shell" {
-    script = "scripts/install-cloudwatch-agent.sh"
-  }
-
   provisioner "shell" {
     script = "scripts/install-docker.sh"
   }

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -87,8 +87,7 @@ PLUGINS_ENABLED=()
 echo "The following plugins will be enabled: ${PLUGINS_ENABLED[*]-}"
 
 # cfn-env is sourced by the environment hook in builds
-# DO NOT PUT SECRETES IN HERE, they will appear in both the cloudwatch and
-# build logs, and the agent's log redactor will not be able to redact them.
+# DO NOT PUT SECRETES IN HERE, they will appear in build logs.
 
 # We will create it in two steps so that we don't need to go crazy with quoting and escaping. The
 # first sets up a helper function, the second populates the default values for some environment

packer/linux/conf/buildkite-agent/scripts/terminate-instance

@@ -16,9 +16,6 @@ mark_as_unhealthy() {
     --health-status Unhealthy
 }
 
-echo "sleeping for 10 seconds before terminating instance to allow agent logs to drain to cloudwatch..."
-sleep 10
-
 token=$(
   curl \
     --fail --silent --show-error \

packer/windows/buildkite-ami.pkr.hcl

@@ -88,10 +88,6 @@ build {
     script = "scripts/install-utils.ps1"
   }
 
-  provisioner "powershell" {
-    script = "scripts/install-cloudwatch-agent.ps1"
-  }
-
   provisioner "powershell" {
     script = "scripts/install-lifecycled.ps1"
   }

templates/aws-stack.yml

@@ -1109,7 +1109,6 @@ Resources:
           - Effect: Allow
             Action:
               - autoscaling:DescribeAutoScalingInstances
-              - cloudwatch:PutMetricData
               - cloudformation:DescribeStackResource
               - ec2:DescribeTags
             Resource: "*"