Upgrade Base Image to Amazon Linux 2023

.buildkite/steps/launch.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eu
+set -euo pipefail
 
 os="${1:-linux}"
 arch="${2:-amd64}"
@@ -22,16 +22,16 @@ echo "Using AMI $image_id for $os/$arch"
 service_role="$(buildkite-agent meta-data get service-role-arn)"
 echo "Using service role ${service_role}"
 
-instance_type="t3.nano"
+instance_type="t3.small"
 instance_disk="10"
 
-if [[ "$os" == "windows" ]] ; then
+if [[ "$os" == "windows" ]]; then
   instance_type="m5.large"
   instance_disk="100"
 fi
 
-if [[ "$arch" == "arm64" ]] ; then
-  instance_type="m6g.large"
+if [[ "$arch" == "arm64" ]]; then
+  instance_type="t4g.small"
 fi
 
 cat << EOF > config.json

Makefile

@@ -3,7 +3,7 @@
 VERSION = $(shell git describe --tags --candidates=1)
 SHELL = /bin/bash -o pipefail
 
-PACKER_VERSION ?= 1.6.2
+PACKER_VERSION ?= 1.8.6
 PACKER_LINUX_FILES = $(exec find packer/linux)
 PACKER_WINDOWS_FILES = $(exec find packer/windows)
 

goss.yaml

@@ -40,6 +40,10 @@ service:
     enabled: true
     running: true
 
+  amazon-ssm-agent:
+    enabled: true
+    running: true
+
   docker:
     enabled: true
     running: true
@@ -73,7 +77,7 @@ user:
     gid: 74
     groups:
     - sshd
-    home: /var/empty/sshd
+    home: /usr/share/empty.sshd
     shell: /sbin/nologin
 
 group:
@@ -83,7 +87,7 @@ group:
 
   docker:
     exists: true
-    gid: 1001
+    gid: 993
 
   sshd:
     exists: true
@@ -100,34 +104,66 @@ process:
     running: true
 
 command:
-  "aws --version":
+  aws --version:
     exit-status: 0
 
-  "git --version":
+  git --version:
     exit-status: 0
 
-  "git-lfs --version":
+  git-lfs --version:
     exit-status: 0
 
-  "/etc/cron.hourly/docker-low-disk-gc":
+  systemctl is-enabled docker-gc.timer:
     exit-status: 0
 
-  "/etc/cron.hourly/docker-gc":
+  /usr/local/bin/docker-gc:
     exit-status: 0
 
-  # Checks that docker is running
-  "docker info":
+  systemctl is-enabled docker-low-disk-gc.timer:
     exit-status: 0
-    timeout: 30000 # it can take some time for the daemon to start
 
-  # Checks that docker containers can run
-  "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker:latest version":
+  /usr/local/bin/docker-low-disk-gc:
     exit-status: 0
-    timeout: 30000 # it can take some time to download the image
 
-  # Checks that permissions
-  'sh -c "docker run --rm -v \"$PWD:/pwd\" alpine:latest touch /pwd/test && stat -c %U:%G test"':
+  # Check docker userns is enabled
+  # Note that goss will evaluate the outer layer of templating, and docker will evaluate the second
+  # Running `goss validate --format documentation` will print this with the first layer of templating evaluated
+  '{{ `docker info --format=",{{range .SecurityOptions}}{{.}},{{end}}"` }}':
     exit-status: 0
-    timeout: 30000 # it can take some time to download the image
+    timeout: 30000
+    stdout:
+    - /,name=userns,/
+
+  # Check docker plugins are installed
+  # Note that goss will evaluate the first layer of templating, and docker will evaluate the second
+  # Running `goss validate --format documentation` will print this with the first layer of templating evaluated
+  '{{ `docker info --format=",{{range .ClientInfo.Plugins}}{{.Name}},{{end}}"` }}':
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - /,buildx,/
+    - /,compose,/
+
+  # Check that docker containers can run
+  docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker:latest version:
+    exit-status: 0
+    timeout: 30000
+
+  # Check that userns allows writing as buildkite-agent
+  sh -c 'docker run --rm -v "$PWD:/pwd" alpine:latest touch /pwd/test && stat -c %U:%G test' && rm test:
+    exit-status: 0
+    timeout: 30000
     stdout:
     - buildkite-agent:docker
+
+  docker run --rm -t arm64v8/ubuntu uname -m:
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - aarch64
+
+  docker run --rm -t amd64/ubuntu uname -m:
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - x86_64

packer/linux/buildkite-ami.json

@@ -11,7 +11,7 @@
       "region": "{{user `region`}}",
       "source_ami_filter": {
         "filters": {
-          "name": "amzn2-ami-kernel-5.10-hvm-2.0.*-gp2",
+          "name": "al2023-ami-minimal-2023.0.*.*-kernel-*",
           "architecture": "{{user `arch`}}",
           "virtualization-type": "hvm"
         },
@@ -21,7 +21,7 @@
       "instance_type": "{{user `instance_type`}}",
       "ssh_username": "ec2-user",
       "ami_name": "buildkite-stack-linux-{{user `arch`}}-{{isotime | clean_resource_name}}",
-      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2 LTS w/ docker)",
+      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2023 w/ docker)",
       "ami_groups": ["all"]
     }
   ],
@@ -44,10 +44,6 @@
       "type": "shell",
       "script": "scripts/install-cloudwatch-agent.sh"
     },
-    {
-      "type": "shell",
-      "script": "scripts/install-lifecycled.sh"
-    },
     {
       "type": "shell",
       "script": "scripts/install-docker.sh"
@@ -58,29 +54,11 @@
     },
     {
       "type": "shell",
-      "script": "scripts/install-s3secrets-helper.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-git-lfs.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-session-manager-plugin.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-nvme-cli.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/upgrade-kernel.sh"
+      "script": "scripts/install-buildkite-utils.sh"
     },
     {
       "type": "shell",
-      "inline": [
-        "rm /home/ec2-user/.ssh/authorized_keys"
-      ]
+      "inline": ["rm /home/ec2-user/.ssh/authorized_keys"]
     }
   ]
 }

packer/linux/conf/bin/bk-configure-docker.sh

@@ -1,27 +1,29 @@
 #!/bin/bash
 # shellcheck disable=SC2094
-set -euo pipefail
+set -euxo pipefail
 
 ## Configures docker before system starts
 
 # Write to system console and to our log file
 # See https://alestic.com/2010/12/ec2-user-data-output/
-exec > >(tee -a /var/log/elastic-stack.log|logger -t user-data -s 2>/dev/console) 2>&1
+exec > >(tee -a /var/log/elastic-stack.log | logger -t user-data -s 2>/dev/console) 2>&1
 
 # Set user namespace remapping in config
-if [[ "${DOCKER_USERNS_REMAP:-false}" == "true" ]] ; then
+if [[ "${DOCKER_USERNS_REMAP:-false}" == "true" ]]; then
   cat <<< "$(jq '."userns-remap"="buildkite-agent"' /etc/docker/daemon.json)" > /etc/docker/daemon.json
 fi
 
 # Set experimental in config
-if [[ "${DOCKER_EXPERIMENTAL:-false}" == "true" ]] ; then
+if [[ "${DOCKER_EXPERIMENTAL:-false}" == "true" ]]; then
   cat <<< "$(jq '.experimental=true' /etc/docker/daemon.json)" > /etc/docker/daemon.json
 fi
 
 # Move docker root to the ephemeral device
-if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]] ; then
+if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]]; then
   cat <<< "$(jq '."data-root"="/mnt/ephemeral/docker"' /etc/docker/daemon.json)" > /etc/docker/daemon.json
 fi
 
 # Customise address pools
 cat <<<"$(jq '."default-address-pools"=[{"base":"172.17.0.0/12","size":20},{"base":"192.168.0.0/16","size":24}]' /etc/docker/daemon.json)" >/etc/docker/daemon.json
+
+systemctl restart docker