0lut · 0lut · Jun 8, 2026 · Jun 5, 2026 · Jun 5, 2026
diff --git a/docs/deployment.md b/docs/deployment.md
@@ -22,7 +22,7 @@ script creates or updates the runtime infrastructure:
 - ECS cluster, service, task definition, and CloudWatch log group
 - hourly EventBridge rule that runs `git-cache compact --all` as a one-off ECS task
 - internet-facing HTTP ALB and target group
-- ECS-optimized EC2 instance
+- Amazon Linux 2023 ECS-optimized EC2 instance
 - non-root gp3 EBS volume mounted on the host at `/cache`
 - Docker image built with `git-cache-api` and `git-cache` CLI S3 support
 
@@ -35,6 +35,7 @@ NAME_PREFIX=gitmirrorcache-arm
 AWS_REGION=us-west-2
 ECS_EC2_INSTANCE_TYPE=m8g.2xlarge
 ECS_CPU_ARCHITECTURE=ARM64
+ECS_EC2_AMI_ID=ami-0ac01d3c8b7a34f9d
 DOCKER_PLATFORM=linux/arm64
 ECS_EBS_SIZE_GIB=128
 ECS_EBS_IOPS=8000
@@ -63,6 +64,83 @@ If a disposable environment can tolerate interrupting in-flight requests during
 deploy, set `ECS_ALB_DEREGISTRATION_DELAY_SECONDS` or
 `ECS_CONTAINER_STOP_TIMEOUT_SECONDS` explicitly.
 
+## Amazon Linux 2023 Host Migration
+
+The maintained ECS host default is pinned to the latest Amazon Linux 2023
+ECS-optimized ARM64 AMI verified in `us-west-2` on June 5, 2026:
+
+```sh
+ECS_EC2_AMI_ID=ami-0ac01d3c8b7a34f9d
+```
+
+AWS metadata at pin time:
+
+- image name: `al2023-ami-ecs-hvm-2023.0.20260527-kernel-6.1-arm64`
+- image version: `2023.0.20260527`
+- ECS agent: `1.103.2`
+- ECS runtime: `Docker version 25.0.14`
+
+To intentionally advance the host image later, query AWS's current recommendation
+and update the pinned `ECS_EC2_AMI_ID` in the deploy script and this document:
+
+```sh
+aws ssm get-parameter \
+  --region us-west-2 \
+  --name /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended \
+  --query Parameter.Value \
+  --output text
+```
+
+Changing the AMI ID only affects newly launched EC2 instances. The deploy
+script reuses the existing ECS host when it finds one with
+`ECS_INSTANCE_NAME=$NAME_PREFIX-ecs-cache`, so an existing Amazon Linux 2 host
+must be replaced during a maintenance window.
+
+For the current single-host deployment, use the checked-in downtime migration
+script. It scales the ECS service to zero, optionally snapshots the old hot-cache
+EBS volume, terminates the existing host, and runs the normal deploy/smoke path
+with the pinned AL2023 AMI ID exported:
+
+```sh
+AWS_REGION=us-west-2 \
+ENVIRONMENT=dev-arm \
+NAME_PREFIX=gitmirrorcache-arm \
+CONFIRM_DOWNTIME=true \
+scripts/aws/migrate-ecs-host-al2023.sh
+```
+
+Set `CREATE_CACHE_SNAPSHOT=true` to snapshot the old `/cache` EBS volume before
+termination. The old cache volume is otherwise left unattached for manual
+cleanup or rollback inspection. S3 remains the durable cache source of truth, so
+the replacement host may start with a cold local `/cache`.
+
+Rollback is to repeat the same downtime flow with the prior AL2 AMI parameter,
+then restore the AL2023 default after the incident:
+
+```sh
+AWS_REGION=us-west-2 \
+ENVIRONMENT=dev-arm \
+NAME_PREFIX=gitmirrorcache-arm \
+CONFIRM_DOWNTIME=true \
+CONFIRM_AL2_ROLLBACK=true \
+ECS_EC2_AMI_PARAMETER=/aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id \
+scripts/aws/migrate-ecs-host-al2023.sh
+```
+
+After the deploy smoke test passes, verify the deployed Git remote path:
+
+```sh
+base="${PUBLIC_BASE_URL:-http://$(aws elbv2 describe-load-balancers \
+  --names gitmirrorcache-arm-ec2-alb \
+  --query 'LoadBalancers[0].DNSName' \
+  --output text)}"
+git ls-remote "$base/git/github.com/astral-sh/uv.git" refs/heads/main
+tmp="$(mktemp -d)"
+git clone --depth 1 --branch main --no-tags \
+  "$base/git/github.com/astral-sh/uv.git" "$tmp/uv"
+rm -rf "$tmp"
+```
+
 ## Why EC2/EBS Instead Of App Runner
 
 App Runner was removed from the maintained deployment path. Large repositories

diff --git a/scripts/aws/deploy-ecs-ec2-ebs.sh b/scripts/aws/deploy-ecs-ec2-ebs.sh
@@ -79,6 +79,7 @@ ECS_CONTAINER_STOP_TIMEOUT_SECONDS="${ECS_CONTAINER_STOP_TIMEOUT_SECONDS:-30}"
 IMAGE_TAG="${IMAGE_TAG:-$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || date -u +%Y%m%d%H%M%S)}"
 IMAGE_URI="${IMAGE_URI:-${ECR_REPOSITORY_URI}:${IMAGE_TAG}}"
 LATEST_URI="${ECR_REPOSITORY_URI}:latest"
+DEFAULT_AL2023_ARM64_ECS_AMI_ID="ami-0ac01d3c8b7a34f9d"
 
 case "$ECS_EBS_DELETE_ON_TERMINATION" in
   true | false) ;;
@@ -767,11 +768,22 @@ build_and_push_image() {
 }
 
 ecs_optimized_ami_id() {
-  local ami_parameter
+  local ami_id ami_parameter
+  ami_id="${ECS_EC2_AMI_ID:-}"
   ami_parameter="${ECS_EC2_AMI_PARAMETER:-}"
+
+  [[ -z "$ami_id" || -z "$ami_parameter" ]] || die "set only one of ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER"
+  if [[ -n "$ami_id" ]]; then
+    [[ "$ami_id" =~ ^ami-[a-f0-9]+$ ]] || die "invalid ECS_EC2_AMI_ID: $ami_id"
+    printf '%s\n' "$ami_id"
+    return
+  fi
+
   if [[ -z "$ami_parameter" ]]; then
-    [[ "$ECS_CPU_ARCHITECTURE" == "ARM64" ]] || die "ECS_EC2_AMI_PARAMETER must be set for $ECS_CPU_ARCHITECTURE"
-    ami_parameter="/aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id"
+    [[ "$ECS_CPU_ARCHITECTURE" == "ARM64" ]] || die "ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER must be set for $ECS_CPU_ARCHITECTURE"
+    [[ "$AWS_REGION" == "us-west-2" ]] || die "default pinned ECS_EC2_AMI_ID is for us-west-2; set ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER for $AWS_REGION"
+    printf '%s\n' "$DEFAULT_AL2023_ARM64_ECS_AMI_ID"
+    return
   fi
 
   aws_cli ssm get-parameter \

diff --git a/scripts/aws/migrate-ecs-host-al2023.sh b/scripts/aws/migrate-ecs-host-al2023.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/common.sh"
+
+[[ "${CONFIRM_DOWNTIME:-false}" == "true" ]] || die "set CONFIRM_DOWNTIME=true to scale the service to zero and replace the ECS host"
+
+init_aws_context
+
+PINNED_AL2023_ARM64_ECS_AMI_ID="ami-0ac01d3c8b7a34f9d"
+
+ECS_CLUSTER_NAME="${ECS_CLUSTER_NAME:-$NAME_PREFIX-ec2}"
+ECS_SERVICE_NAME="${ECS_SERVICE_NAME:-$NAME_PREFIX-ec2-api}"
+ECS_INSTANCE_NAME="${ECS_INSTANCE_NAME:-$NAME_PREFIX-ecs-cache}"
+ECS_EBS_DEVICE_NAME="${ECS_EBS_DEVICE_NAME:-/dev/xvdf}"
+CREATE_CACHE_SNAPSHOT="${CREATE_CACHE_SNAPSHOT:-false}"
+
+if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then
+  if [[ "$ECS_EC2_AMI_PARAMETER" == *"amazon-linux-2"* ]]; then
+    [[ "${CONFIRM_AL2_ROLLBACK:-false}" == "true" ]] \
+      || die "set CONFIRM_AL2_ROLLBACK=true to use an AL2 AMI parameter"
+  else
+    [[ "$ECS_EC2_AMI_PARAMETER" == *"amazon-linux-2023"* ]] \
+      || die "ECS_EC2_AMI_PARAMETER must point at AL2023 unless CONFIRM_AL2_ROLLBACK=true is set for rollback"
+  fi
+  unset ECS_EC2_AMI_ID
+else
+  ECS_EC2_AMI_ID="${ECS_EC2_AMI_ID:-$PINNED_AL2023_ARM64_ECS_AMI_ID}"
+  [[ "$ECS_EC2_AMI_ID" =~ ^ami-[a-f0-9]+$ ]] || die "invalid ECS_EC2_AMI_ID: $ECS_EC2_AMI_ID"
+fi
+
+current_desired_count="$(aws_cli ecs describe-services \
+  --cluster "$ECS_CLUSTER_NAME" \
+  --services "$ECS_SERVICE_NAME" \
+  --query 'services[0].desiredCount' \
+  --output text)"
+[[ "$current_desired_count" =~ ^[0-9]+$ ]] || die "could not read desired count for $ECS_CLUSTER_NAME/$ECS_SERVICE_NAME"
+
+ECS_DESIRED_COUNT="${ECS_DESIRED_COUNT:-$current_desired_count}"
+[[ "$ECS_DESIRED_COUNT" =~ ^[1-9][0-9]*$ ]] || die "ECS_DESIRED_COUNT must be greater than zero for redeploy; got $ECS_DESIRED_COUNT"
+export ECS_DESIRED_COUNT
+if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then
+  export ECS_EC2_AMI_PARAMETER
+else
+  export ECS_EC2_AMI_ID
+fi
+
+instance_id="$(aws_cli ec2 describe-instances \
+  --filters "Name=tag:Name,Values=$ECS_INSTANCE_NAME" Name=instance-state-name,Values=pending,running,stopping,stopped \
+  --query 'Reservations[].Instances[].InstanceId | [0]' \
+  --output text)"
+
+printf 'Scaling ECS service to zero: %s/%s\n' "$ECS_CLUSTER_NAME" "$ECS_SERVICE_NAME"
+aws_cli ecs update-service \
+  --cluster "$ECS_CLUSTER_NAME" \
+  --service "$ECS_SERVICE_NAME" \
+  --desired-count 0 >/dev/null
+aws_cli ecs wait services-stable --cluster "$ECS_CLUSTER_NAME" --services "$ECS_SERVICE_NAME"
+
+if [[ "$instance_id" == "None" || -z "$instance_id" ]]; then
+  printf 'No existing ECS container instance named %s was found; continuing to deploy AL2023 host\n' "$ECS_INSTANCE_NAME"
+else
+  [[ "$instance_id" =~ ^i-[a-f0-9]+$ ]] || die "invalid EC2 instance id for $ECS_INSTANCE_NAME: $instance_id"
+
+  cache_volume_id="$(aws_cli ec2 describe-instances \
+    --instance-ids "$instance_id" \
+    --query "Reservations[0].Instances[0].BlockDeviceMappings[?DeviceName=='$ECS_EBS_DEVICE_NAME'].Ebs.VolumeId | [0]" \
+    --output text)"
+
+  if [[ "$CREATE_CACHE_SNAPSHOT" == "true" && "$cache_volume_id" != "None" && -n "$cache_volume_id" ]]; then
+    snapshot_id="$(aws_cli ec2 create-snapshot \
+      --volume-id "$cache_volume_id" \
+      --description "$NAME_PREFIX cache before AL2023 ECS host migration" \
+      --query SnapshotId \
+      --output text)"
+    printf 'Created cache volume snapshot: %s from %s\n' "$snapshot_id" "$cache_volume_id"
+  elif [[ "$cache_volume_id" != "None" && -n "$cache_volume_id" ]]; then
+    printf 'Leaving old cache volume for manual cleanup/rollback: %s\n' "$cache_volume_id"
+  fi
+
+  printf 'Terminating existing ECS host: %s\n' "$instance_id"
+  aws_cli ec2 terminate-instances --instance-ids "$instance_id" >/dev/null
+  aws_cli ec2 wait instance-terminated --instance-ids "$instance_id"
+fi
+
+if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then
+  printf 'Deploying replacement ECS host with AMI parameter: %s\n' "$ECS_EC2_AMI_PARAMETER"
+else
+  printf 'Deploying replacement ECS host with pinned AMI ID: %s\n' "$ECS_EC2_AMI_ID"
+fi
+"$SCRIPT_DIR/deploy-and-smoke.sh"