diff --git a/docs/deployment.md b/docs/deployment.md index f22afd0..fc06712 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -22,7 +22,7 @@ script creates or updates the runtime infrastructure: - ECS cluster, service, task definition, and CloudWatch log group - hourly EventBridge rule that runs `git-cache compact --all` as a one-off ECS task - internet-facing HTTP ALB and target group -- ECS-optimized EC2 instance +- Amazon Linux 2023 ECS-optimized EC2 instance - non-root gp3 EBS volume mounted on the host at `/cache` - Docker image built with `git-cache-api` and `git-cache` CLI S3 support @@ -35,6 +35,7 @@ NAME_PREFIX=gitmirrorcache-arm AWS_REGION=us-west-2 ECS_EC2_INSTANCE_TYPE=m8g.2xlarge ECS_CPU_ARCHITECTURE=ARM64 +ECS_EC2_AMI_ID=ami-0ac01d3c8b7a34f9d DOCKER_PLATFORM=linux/arm64 ECS_EBS_SIZE_GIB=128 ECS_EBS_IOPS=8000 @@ -63,6 +64,83 @@ If a disposable environment can tolerate interrupting in-flight requests during deploy, set `ECS_ALB_DEREGISTRATION_DELAY_SECONDS` or `ECS_CONTAINER_STOP_TIMEOUT_SECONDS` explicitly. +## Amazon Linux 2023 Host Migration + +The maintained ECS host default is pinned to the latest Amazon Linux 2023 +ECS-optimized ARM64 AMI verified in `us-west-2` on June 5, 2026: + +```sh +ECS_EC2_AMI_ID=ami-0ac01d3c8b7a34f9d +``` + +AWS metadata at pin time: + +- image name: `al2023-ami-ecs-hvm-2023.0.20260527-kernel-6.1-arm64` +- image version: `2023.0.20260527` +- ECS agent: `1.103.2` +- ECS runtime: `Docker version 25.0.14` + +To intentionally advance the host image later, query AWS's current recommendation +and update the pinned `ECS_EC2_AMI_ID` in the deploy script and this document: + +```sh +aws ssm get-parameter \ + --region us-west-2 \ + --name /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended \ + --query Parameter.Value \ + --output text +``` + +Changing the AMI ID only affects newly launched EC2 instances. The deploy +script reuses the existing ECS host when it finds one with +`ECS_INSTANCE_NAME=$NAME_PREFIX-ecs-cache`, so an existing Amazon Linux 2 host +must be replaced during a maintenance window. + +For the current single-host deployment, use the checked-in downtime migration +script. It scales the ECS service to zero, optionally snapshots the old hot-cache +EBS volume, terminates the existing host, and runs the normal deploy/smoke path +with the pinned AL2023 AMI ID exported: + +```sh +AWS_REGION=us-west-2 \ +ENVIRONMENT=dev-arm \ +NAME_PREFIX=gitmirrorcache-arm \ +CONFIRM_DOWNTIME=true \ +scripts/aws/migrate-ecs-host-al2023.sh +``` + +Set `CREATE_CACHE_SNAPSHOT=true` to snapshot the old `/cache` EBS volume before +termination. The old cache volume is otherwise left unattached for manual +cleanup or rollback inspection. S3 remains the durable cache source of truth, so +the replacement host may start with a cold local `/cache`. + +Rollback is to repeat the same downtime flow with the prior AL2 AMI parameter, +then restore the AL2023 default after the incident: + +```sh +AWS_REGION=us-west-2 \ +ENVIRONMENT=dev-arm \ +NAME_PREFIX=gitmirrorcache-arm \ +CONFIRM_DOWNTIME=true \ +CONFIRM_AL2_ROLLBACK=true \ +ECS_EC2_AMI_PARAMETER=/aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id \ +scripts/aws/migrate-ecs-host-al2023.sh +``` + +After the deploy smoke test passes, verify the deployed Git remote path: + +```sh +base="${PUBLIC_BASE_URL:-http://$(aws elbv2 describe-load-balancers \ + --names gitmirrorcache-arm-ec2-alb \ + --query 'LoadBalancers[0].DNSName' \ + --output text)}" +git ls-remote "$base/git/github.com/astral-sh/uv.git" refs/heads/main +tmp="$(mktemp -d)" +git clone --depth 1 --branch main --no-tags \ + "$base/git/github.com/astral-sh/uv.git" "$tmp/uv" +rm -rf "$tmp" +``` + ## Why EC2/EBS Instead Of App Runner App Runner was removed from the maintained deployment path. Large repositories diff --git a/scripts/aws/deploy-ecs-ec2-ebs.sh b/scripts/aws/deploy-ecs-ec2-ebs.sh index 26c111d..ff8d530 100755 --- a/scripts/aws/deploy-ecs-ec2-ebs.sh +++ b/scripts/aws/deploy-ecs-ec2-ebs.sh @@ -79,6 +79,7 @@ ECS_CONTAINER_STOP_TIMEOUT_SECONDS="${ECS_CONTAINER_STOP_TIMEOUT_SECONDS:-30}" IMAGE_TAG="${IMAGE_TAG:-$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || date -u +%Y%m%d%H%M%S)}" IMAGE_URI="${IMAGE_URI:-${ECR_REPOSITORY_URI}:${IMAGE_TAG}}" LATEST_URI="${ECR_REPOSITORY_URI}:latest" +DEFAULT_AL2023_ARM64_ECS_AMI_ID="ami-0ac01d3c8b7a34f9d" case "$ECS_EBS_DELETE_ON_TERMINATION" in true | false) ;; @@ -767,11 +768,22 @@ build_and_push_image() { } ecs_optimized_ami_id() { - local ami_parameter + local ami_id ami_parameter + ami_id="${ECS_EC2_AMI_ID:-}" ami_parameter="${ECS_EC2_AMI_PARAMETER:-}" + + [[ -z "$ami_id" || -z "$ami_parameter" ]] || die "set only one of ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER" + if [[ -n "$ami_id" ]]; then + [[ "$ami_id" =~ ^ami-[a-f0-9]+$ ]] || die "invalid ECS_EC2_AMI_ID: $ami_id" + printf '%s\n' "$ami_id" + return + fi + if [[ -z "$ami_parameter" ]]; then - [[ "$ECS_CPU_ARCHITECTURE" == "ARM64" ]] || die "ECS_EC2_AMI_PARAMETER must be set for $ECS_CPU_ARCHITECTURE" - ami_parameter="/aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id" + [[ "$ECS_CPU_ARCHITECTURE" == "ARM64" ]] || die "ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER must be set for $ECS_CPU_ARCHITECTURE" + [[ "$AWS_REGION" == "us-west-2" ]] || die "default pinned ECS_EC2_AMI_ID is for us-west-2; set ECS_EC2_AMI_ID or ECS_EC2_AMI_PARAMETER for $AWS_REGION" + printf '%s\n' "$DEFAULT_AL2023_ARM64_ECS_AMI_ID" + return fi aws_cli ssm get-parameter \ diff --git a/scripts/aws/migrate-ecs-host-al2023.sh b/scripts/aws/migrate-ecs-host-al2023.sh new file mode 100755 index 0000000..235e710 --- /dev/null +++ b/scripts/aws/migrate-ecs-host-al2023.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +[[ "${CONFIRM_DOWNTIME:-false}" == "true" ]] || die "set CONFIRM_DOWNTIME=true to scale the service to zero and replace the ECS host" + +init_aws_context + +PINNED_AL2023_ARM64_ECS_AMI_ID="ami-0ac01d3c8b7a34f9d" + +ECS_CLUSTER_NAME="${ECS_CLUSTER_NAME:-$NAME_PREFIX-ec2}" +ECS_SERVICE_NAME="${ECS_SERVICE_NAME:-$NAME_PREFIX-ec2-api}" +ECS_INSTANCE_NAME="${ECS_INSTANCE_NAME:-$NAME_PREFIX-ecs-cache}" +ECS_EBS_DEVICE_NAME="${ECS_EBS_DEVICE_NAME:-/dev/xvdf}" +CREATE_CACHE_SNAPSHOT="${CREATE_CACHE_SNAPSHOT:-false}" + +if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then + if [[ "$ECS_EC2_AMI_PARAMETER" == *"amazon-linux-2"* ]]; then + [[ "${CONFIRM_AL2_ROLLBACK:-false}" == "true" ]] \ + || die "set CONFIRM_AL2_ROLLBACK=true to use an AL2 AMI parameter" + else + [[ "$ECS_EC2_AMI_PARAMETER" == *"amazon-linux-2023"* ]] \ + || die "ECS_EC2_AMI_PARAMETER must point at AL2023 unless CONFIRM_AL2_ROLLBACK=true is set for rollback" + fi + unset ECS_EC2_AMI_ID +else + ECS_EC2_AMI_ID="${ECS_EC2_AMI_ID:-$PINNED_AL2023_ARM64_ECS_AMI_ID}" + [[ "$ECS_EC2_AMI_ID" =~ ^ami-[a-f0-9]+$ ]] || die "invalid ECS_EC2_AMI_ID: $ECS_EC2_AMI_ID" +fi + +current_desired_count="$(aws_cli ecs describe-services \ + --cluster "$ECS_CLUSTER_NAME" \ + --services "$ECS_SERVICE_NAME" \ + --query 'services[0].desiredCount' \ + --output text)" +[[ "$current_desired_count" =~ ^[0-9]+$ ]] || die "could not read desired count for $ECS_CLUSTER_NAME/$ECS_SERVICE_NAME" + +ECS_DESIRED_COUNT="${ECS_DESIRED_COUNT:-$current_desired_count}" +[[ "$ECS_DESIRED_COUNT" =~ ^[1-9][0-9]*$ ]] || die "ECS_DESIRED_COUNT must be greater than zero for redeploy; got $ECS_DESIRED_COUNT" +export ECS_DESIRED_COUNT +if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then + export ECS_EC2_AMI_PARAMETER +else + export ECS_EC2_AMI_ID +fi + +instance_id="$(aws_cli ec2 describe-instances \ + --filters "Name=tag:Name,Values=$ECS_INSTANCE_NAME" Name=instance-state-name,Values=pending,running,stopping,stopped \ + --query 'Reservations[].Instances[].InstanceId | [0]' \ + --output text)" + +printf 'Scaling ECS service to zero: %s/%s\n' "$ECS_CLUSTER_NAME" "$ECS_SERVICE_NAME" +aws_cli ecs update-service \ + --cluster "$ECS_CLUSTER_NAME" \ + --service "$ECS_SERVICE_NAME" \ + --desired-count 0 >/dev/null +aws_cli ecs wait services-stable --cluster "$ECS_CLUSTER_NAME" --services "$ECS_SERVICE_NAME" + +if [[ "$instance_id" == "None" || -z "$instance_id" ]]; then + printf 'No existing ECS container instance named %s was found; continuing to deploy AL2023 host\n' "$ECS_INSTANCE_NAME" +else + [[ "$instance_id" =~ ^i-[a-f0-9]+$ ]] || die "invalid EC2 instance id for $ECS_INSTANCE_NAME: $instance_id" + + cache_volume_id="$(aws_cli ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query "Reservations[0].Instances[0].BlockDeviceMappings[?DeviceName=='$ECS_EBS_DEVICE_NAME'].Ebs.VolumeId | [0]" \ + --output text)" + + if [[ "$CREATE_CACHE_SNAPSHOT" == "true" && "$cache_volume_id" != "None" && -n "$cache_volume_id" ]]; then + snapshot_id="$(aws_cli ec2 create-snapshot \ + --volume-id "$cache_volume_id" \ + --description "$NAME_PREFIX cache before AL2023 ECS host migration" \ + --query SnapshotId \ + --output text)" + printf 'Created cache volume snapshot: %s from %s\n' "$snapshot_id" "$cache_volume_id" + elif [[ "$cache_volume_id" != "None" && -n "$cache_volume_id" ]]; then + printf 'Leaving old cache volume for manual cleanup/rollback: %s\n' "$cache_volume_id" + fi + + printf 'Terminating existing ECS host: %s\n' "$instance_id" + aws_cli ec2 terminate-instances --instance-ids "$instance_id" >/dev/null + aws_cli ec2 wait instance-terminated --instance-ids "$instance_id" +fi + +if [[ -n "${ECS_EC2_AMI_PARAMETER:-}" ]]; then + printf 'Deploying replacement ECS host with AMI parameter: %s\n' "$ECS_EC2_AMI_PARAMETER" +else + printf 'Deploying replacement ECS host with pinned AMI ID: %s\n' "$ECS_EC2_AMI_ID" +fi +"$SCRIPT_DIR/deploy-and-smoke.sh"