Merge upstream v6.55.0

go.mod

@@ -1,10 +1,10 @@
 module github.com/buildkite/elastic-ci-stack-for-aws/v6
 
-go 1.23.6
+go 1.24.0
 
-toolchain go1.24.5
+toolchain go1.25.6
 
 require (
 	github.com/google/go-cmp v0.7.0
-	golang.org/x/sys v0.34.0
+	golang.org/x/sys v0.41.0
 )

go.sum

@@ -1,4 +1,4 @@
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=

packer/linux/base/scripts/versions.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Centralized version definitions for Linux packer builds
+# This file is sourced by installation scripts
+
+# Core Tools
+export AWS_CLI_LINUX_VERSION="2.31.32"
+export SESSION_MANAGER_PLUGIN_VERSION="1.2.764.0"
+
+# Development Tools
+export GIT_LFS_VERSION="3.4.0"
+export GOSS_VERSION="v0.4.9"
+
+# Container Tools
+export DOCKER_COMPOSE_V2_VERSION="2.40.3"
+export DOCKER_BUILDX_VERSION="0.29.1"
+
+# Buildkite Tools
+export S3_SECRETS_HELPER_VERSION="2.7.0"
+export LIFECYCLED_VERSION="v3.5.0"

packer/linux/conf/bin/bk-check-disk-space.sh

@@ -4,7 +4,10 @@ set -euo pipefail
 DISK_MIN_AVAILABLE=${DISK_MIN_AVAILABLE:-5242880} # 5GB
 DISK_MIN_INODES=${DISK_MIN_INODES:-250000}        # docker needs lots
 
-DOCKER_DIR="$(jq -r '."data-root" // "/var/lib/docker"' /etc/docker/daemon.json)"
+DOCKER_DIR="$(jq -r '."data-root" // "/var/lib/docker"' /etc/docker/daemon.json || true)"
+if [[ -z "$DOCKER_DIR" ]]; then
+  DOCKER_DIR="/var/lib/docker"
+fi
 
 disk_avail=$(df -k --output=avail "$DOCKER_DIR" | tail -n1)
 
@@ -15,7 +18,32 @@ if [[ $disk_avail -lt $DISK_MIN_AVAILABLE ]]; then
   disk_min_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${DISK_MIN_AVAILABLE}")
   disk_avail_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${disk_avail}")
   echo "Not enough disk space free: ${disk_avail_human} (${disk_avail}KB) available, cutoff is ${disk_min_human} (${DISK_MIN_AVAILABLE}KB) 🚨" >&2
-  exit 1
+
+  # Last resort for clearing space with build directory cleanup (if enabled)
+  if [[ "${BUILDKITE_PURGE_BUILDS_ON_DISK_FULL:-false}" == "true" ]]; then
+    # Extract the agent-specific directory to avoid disrupting other agents on the same instance
+    AGENT_ORG_PIPELINE_DIR="${BUILDKITE_BUILD_CHECKOUT_PATH#"${BUILDKITE_BUILD_PATH}/"}"
+    AGENT_DIR="${AGENT_ORG_PIPELINE_DIR%%/*}"
+    if [[ -z "${AGENT_DIR}" ]]; then
+      echo "Unable to determine agent-specific build directory. Skipping purge to avoid deleting other agents' builds." >&2
+      exit 1
+    fi
+    AGENT_BUILD_DIR="${BUILDKITE_BUILD_PATH}/${AGENT_DIR}"
+
+    echo "Purging builds in ${AGENT_BUILD_DIR}"
+    rm -rf "${AGENT_BUILD_DIR:?}"/*
+
+    disk_avail=$(df -k --output=avail "$DOCKER_DIR" | tail -n1)
+    disk_avail_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${disk_avail}")
+    echo "Disk space free after build purge: ${disk_avail_human} (${disk_avail}KB)"
+    if [[ $disk_avail -lt $DISK_MIN_AVAILABLE ]]; then
+      echo "Insufficient disk space remaining after build purge." >&2
+      exit 1
+    fi
+  else
+    echo "Insufficient disk space. Build purge not enabled." >&2
+    exit 1
+  fi
 fi
 
 inodes_avail=$(df -k --output=iavail "$DOCKER_DIR" | tail -n1)

packer/linux/conf/buildkite-agent/hooks/environment

@@ -41,13 +41,17 @@ fi
 
 echo "Checking disk space"
 if ! /usr/local/bin/bk-check-disk-space.sh; then
-  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL:-4h}"
-  docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}"
+  echo "~~~ :broom: Pruning docker resources to free disk space"
+  docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" || true
+
+  if [[ "${DOCKER_BUILDER_PRUNE_ENABLED:-false}" == "true" ]]; then
+    docker builder prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" || true
+  fi
 
   echo "Checking disk space again"
   # Capture disk space output for potential error logging
   if ! disk_check_output=$(/usr/local/bin/bk-check-disk-space.sh 2>&1); then
-    echo "--- :warning: Disk health checks failed."
+    echo "--- :warning: Insufficient disk space after cleanup."
     echo "${disk_check_output}"
 
     # Check if instance termination is enabled (default: false for backward compatibility)
@@ -101,6 +105,7 @@ done
 if [[ -n "${BUILDKITE_SECRETS_BUCKET:-}" && "${SECRETS_PLUGIN_ENABLED:-}" == "1" ]]; then
   export BUILDKITE_PLUGIN_S3_SECRETS_BUCKET="$BUILDKITE_SECRETS_BUCKET"
   export BUILDKITE_PLUGIN_S3_SECRETS_REGION="$BUILDKITE_SECRETS_BUCKET_REGION"
+  export BUILDKITE_PLUGIN_S3_SECRETS_SKIP_SSH_KEY_NOT_FOUND_WARNING="${BUILDKITE_SECRETS_PLUGIN_SKIP_SSH_KEY_NOT_FOUND_WARNING:-false}"
 
   # shellcheck source=/dev/null
   source /usr/local/buildkite-aws-stack/plugins/secrets/hooks/environment
@@ -110,11 +115,19 @@ if [[ "${BUILDKITE_ECR_POLICY:-}" != "none" && "${ECR_PLUGIN_ENABLED:-}" == "1"
   export BUILDKITE_PLUGIN_ECR_LOGIN=1
   export BUILDKITE_PLUGIN_ECR_RETRIES=3
 
+  # Allow pipeline-level override, otherwise use CloudFormation parameter
+  set_unless_present "BUILDKITE_PLUGIN_ECR_CREDENTIAL_HELPER" "${ECR_CREDENTIAL_HELPER_ENABLED:-false}"
+
   # map AWS_ECR_LOGIN_REGISTRY_IDS into the plugin list format
   if [[ -n "${AWS_ECR_LOGIN_REGISTRY_IDS:-}" ]]; then
     export BUILDKITE_PLUGIN_ECR_ACCOUNT_IDS_0="${AWS_ECR_LOGIN_REGISTRY_IDS}"
   fi
 
+  # map AWS_ECR_LOGIN into the plugin format
+  if [[ -n "${AWS_ECR_LOGIN:-}" ]]; then
+    export BUILDKITE_PLUGIN_ECR_LOGIN="${AWS_ECR_LOGIN}"
+  fi
+
   # shellcheck source=/dev/null
   source /usr/local/buildkite-aws-stack/plugins/ecr/hooks/environment
 fi

packer/linux/conf/docker/scripts/docker-low-disk-gc

@@ -34,12 +34,12 @@ trap mark_instance_unhealthy ERR
 ## if we really need to
 
 if ! /usr/local/bin/bk-check-disk-space.sh; then
-  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL}"
+  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL} to free disk space"
   docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL}"
   docker builder prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL}"
 
   if ! /usr/local/bin/bk-check-disk-space.sh; then
-    echo "Disk health checks failed" >&2 && false
+    echo "Insufficient disk space after cleanup." >&2 && false
     exit 1
   fi
 fi

packer/linux/scripts/install-buildkite-agent.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+AGENT_VERSION="3.117.0"
+
 case $(uname -m) in
 x86_64) ARCH=amd64 ;;
 aarch64) ARCH=arm64 ;;
@@ -15,7 +17,6 @@ sudo mkdir -p /var/lib/buildkite-agent/.aws
 sudo cp /tmp/conf/aws/config /var/lib/buildkite-agent/.aws/config
 sudo chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.aws
 
-AGENT_VERSION=3.103.0
 echo "Downloading buildkite-agent v${AGENT_VERSION} stable..."
 sudo curl -Lsf -o /usr/bin/buildkite-agent-stable \
   "https://download.buildkite.com/agent/stable/${AGENT_VERSION}/buildkite-agent-linux-${ARCH}"
@@ -62,6 +63,8 @@ sudo cp /tmp/conf/buildkite-agent/systemd/buildkite-agent.service /etc/systemd/s
 echo "Adding cloud-init failure safety check..."
 sudo mkdir -p /etc/systemd/system/cloud-final.service.d/
 sudo cp /tmp/conf/buildkite-agent/systemd/cloud-final.service.d/10-power-off-on-failure.conf /etc/systemd/system/cloud-final.service.d/10-power-off-on-failure.conf
+sudo mkdir -p /etc/systemd/system/cloud-init.service.d/
+sudo cp /tmp/conf/buildkite-agent/systemd/cloud-init.service.d/10-power-off-on-failure.conf /etc/systemd/system/cloud-init.service.d/10-power-off-on-failure.conf
 
 echo "Adding termination scripts..."
 sudo cp /tmp/conf/buildkite-agent/scripts/stop-agent-gracefully /usr/local/bin/stop-agent-gracefully

packer/linux/scripts/install-utils.sh

@@ -25,6 +25,12 @@ sudo dnf install -yq \
   python \
   python-pip \
   python-setuptools \
+  python3.11 \
+  python3.11-pip \
+  python3.12 \
+  python3.12-pip \
+  python3.13 \
+  python3.13-pip \
   unzip \
   wget \
   zip

packer/linux/stack/conf/buildkite-agent/hooks/pre-exit

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ "${ENABLE_PRE_EXIT_DISK_CLEANUP:-false}" == "true" ]]; then
+  echo "Checking disk space"
+  if ! /usr/local/bin/bk-check-disk-space.sh; then
+    echo "~~~ :broom: Pruning docker resources to free disk space"
+    docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" || true
+
+    if [[ "${DOCKER_BUILDER_PRUNE_ENABLED:-false}" == "true" ]]; then
+      docker builder prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" || true
+    fi
+
+    echo "Checking disk space again"
+    # Capture disk space output for potential error logging
+    if ! disk_check_output=$(/usr/local/bin/bk-check-disk-space.sh 2>&1); then
+      echo "--- :warning: Insufficient disk space after cleanup."
+      echo "${disk_check_output}"
+
+      # Check if instance termination is enabled (default: false for backward compatibility)
+      if [[ "${BUILDKITE_TERMINATE_INSTANCE_ON_DISK_FULL:-false}" == "true" ]]; then
+        echo "Terminating instance due to disk space issues."
+        export BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB=true
+
+        if [[ -x /usr/local/bin/terminate-instance ]]; then
+          /usr/local/bin/terminate-instance
+        else
+          echo "terminate-instance script not found, falling back to exit 1" >&2
+          exit 1
+        fi
+
+        # Should not reach here if termination succeeds
+        exit 1
+      else
+        echo "Exiting job due to insufficient disk space (set BuildkiteTerminateInstanceOnDiskFull to true in CloudFormation Stack to terminate instance instead)."
+        exit 1
+      fi
+    fi
+  fi
+fi
+
+# clean up our temporary docker config
+if [[ -n "${BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY:-}" && -d "$BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY" ]]; then
+  rm -rf "$BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY"
+fi
+
+if [[ -n "${BUILDKITE_SECRETS_BUCKET:-}" && "${SECRETS_PLUGIN_ENABLED:-}" == "1" ]]; then
+  export BUILDKITE_PLUGIN_S3_SECRETS_BUCKET="$BUILDKITE_SECRETS_BUCKET"
+
+  # shellcheck source=/dev/null
+  source /usr/local/buildkite-aws-stack/plugins/secrets/hooks/pre-exit
+fi

packer/windows/base/scripts/versions.ps1

@@ -0,0 +1,17 @@
+# Centralized version definitions for Windows packer builds
+# This file is sourced by installation scripts
+
+# Core Tools
+$AWS_CLI_WINDOWS_VERSION = "2.31.32"
+$SESSION_MANAGER_PLUGIN_VERSION = "1.2.764.0"
+
+# Development Tools
+$GIT_VERSION = "2.39.1"
+
+# Container Tools
+$docker_compose_version = "2.40.3"
+$ecr_cred_helper_version = "0.10.1"
+
+# Buildkite Tools
+$S3_SECRETS_HELPER_VERSION = "2.7.0"
+$lifecycled_version = "v3.5.0"

packer/windows/scripts/install-buildkite-agent.ps1

@@ -1,10 +1,11 @@
 # Stop script execution when a non-terminating error occurs
 $ErrorActionPreference = "Stop"
 
-$AGENT_VERSION = "3.103.0"
+
+$AGENT_VERSION = "3.117.0"
 
 Write-Output "Creating bin dir..."
-New-Item -ItemType directory -Path C:\buildkite-agent\bin
+if (-not (Test-Path C:\buildkite-agent\bin)) { New-Item -ItemType Directory -Path C:\buildkite-agent\bin -Force }
 
 Write-Output 'Updating PATH'
 $env:PATH = "C:\buildkite-agent\bin;" + $env:PATH
@@ -21,19 +22,19 @@ buildkite-agent-beta.exe --version
 If ($lastexitcode -ne 0) { Exit $lastexitcode }
 
 Write-Output "Creating hooks dir..."
-New-Item -ItemType directory -Path C:\buildkite-agent\hooks
+if (-not (Test-Path C:\buildkite-agent\hooks)) { New-Item -ItemType Directory -Path C:\buildkite-agent\hooks -Force }
 
 Write-Output "Copying custom hooks..."
 Copy-Item -Path C:\packer-temp\conf\buildkite-agent\hooks\* -Destination C:\buildkite-agent\hooks
 
 Write-Output "Creating builds dir..."
-New-Item -ItemType directory -Path C:\buildkite-agent\builds
+if (-not (Test-Path C:\buildkite-agent\builds)) { New-Item -ItemType Directory -Path C:\buildkite-agent\builds -Force }
 
 Write-Output "Creating git-mirrors dir..."
-New-Item -ItemType directory -Path C:\buildkite-agent\git-mirrors
+if (-not (Test-Path C:\buildkite-agent\git-mirrors)) { New-Item -ItemType Directory -Path C:\buildkite-agent\git-mirrors -Force }
 
 Write-Output "Creating plugins dir..."
-New-Item -ItemType directory -Path C:\buildkite-agent\plugins
+if (-not (Test-Path C:\buildkite-agent\plugins)) { New-Item -ItemType Directory -Path C:\buildkite-agent\plugins -Force }
 
 Write-Output "Installing bk elastic stack bin files..."
 Copy-Item -Path C:\packer-temp\conf\bin\bk-* -Destination C:\buildkite-agent\bin
@@ -43,5 +44,5 @@ Copy-Item -Path C:\packer-temp\conf\buildkite-agent\scripts\terminate-instance.p
 Copy-Item -Path C:\packer-temp\conf\buildkite-agent\scripts\stop-agent-gracefully.ps1 -Destination C:\buildkite-agent\bin
 
 Write-Output "Copying built-in plugins..."
-New-Item -ItemType directory -Path "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins"
+if (-not (Test-Path "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins")) { New-Item -ItemType Directory -Path "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins" -Force }
 Copy-Item -Recurse -Path C:\packer-temp\plugins\* -Destination "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins\"