From da9ae7dce7697e424c0bea37e812c1e95b5eb87a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 12:47:32 -0500 Subject: [PATCH 1/8] fix: relocate all build ops to us-east-1 --- .github/workflows/publish-migrations-prod.yml | 2 +- .github/workflows/publish-migrations-staging.yml | 2 +- .github/workflows/testinfra-ami-build.yml | 12 ++++++------ amazon-arm64-nix.pkr.hcl | 2 +- nix/docs/development-workflow.md | 2 +- nix/packages/build-test-ami.nix | 2 +- nix/packages/cleanup-ami.nix | 2 +- nix/packages/run-testinfra.nix | 8 ++++---- testinfra/README.md | 4 ++-- testinfra/test_ami_nix.py | 10 +++++----- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish-migrations-prod.yml b/.github/workflows/publish-migrations-prod.yml index 81c686db7..e59bac9ba 100644 --- a/.github/workflows/publish-migrations-prod.yml +++ b/.github/workflows/publish-migrations-prod.yml @@ -32,7 +32,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Deploy to S3 prod shell: bash diff --git a/.github/workflows/publish-migrations-staging.yml b/.github/workflows/publish-migrations-staging.yml index 82f01bcf4..0c862990d 100644 --- a/.github/workflows/publish-migrations-staging.yml +++ b/.github/workflows/publish-migrations-staging.yml @@ -25,7 +25,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Deploy to S3 staging shell: bash diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index c337e6e47..65e8f7013 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -112,13 +112,13 @@ jobs: run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 @@ -132,12 +132,12 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids + aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true + aws ec2 --region us-east-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids || true - name: Cleanup AMIs if: always() @@ -149,10 +149,10 @@ jobs: # Function to deregister AMIs by name pattern deregister_ami_by_name() { local ami_name_pattern=$1 - local ami_ids=$(aws ec2 describe-images --region ap-southeast-1 --owners self --filters "Name=name,Values=${ami_name_pattern}" --query 'Images[*].ImageId' --output text) + local ami_ids=$(aws ec2 describe-images --region us-east-1 --owners self --filters "Name=name,Values=${ami_name_pattern}" --query 'Images[*].ImageId' --output text) for ami_id in $ami_ids; do echo "Deregistering AMI: $ami_id" - aws ec2 deregister-image --region ap-southeast-1 --image-id $ami_id + aws ec2 deregister-image --region us-east-1 --image-id $ami_id done } diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index f625ac4af..c2b73cde8 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -15,7 +15,7 @@ variable "ami_name" { variable "ami_regions" { type = list(string) - default = ["ap-southeast-2"] + default = ["us-east-1"] } variable "ansible_arguments" { diff --git a/nix/docs/development-workflow.md b/nix/docs/development-workflow.md index 695427abc..c3b66ba50 100644 --- a/nix/docs/development-workflow.md +++ b/nix/docs/development-workflow.md @@ -110,7 +110,7 @@ This will: The following environment variables are used: - `AWS_VAULT`: AWS Vault profile name (default: staging) -- `AWS_REGION`: AWS region (default: ap-southeast-1) +- `AWS_REGION`: AWS region (default: us-east-1) - `AMI_NAME`: Name of the AMI to test ## Best Practices diff --git a/nix/packages/build-test-ami.nix b/nix/packages/build-test-ami.nix index 115ea793d..f63968e27 100644 --- a/nix/packages/build-test-ami.nix +++ b/nix/packages/build-test-ami.nix @@ -84,7 +84,7 @@ runCommand "build-test-ami" fi # Set values - REGION="ap-southeast-1" + REGION="us-east-1" POSTGRES_VERSION="$1" RANDOM_STRING=$(openssl rand -hex 8) GIT_SHA=$(git rev-parse HEAD) diff --git a/nix/packages/cleanup-ami.nix b/nix/packages/cleanup-ami.nix index f1f9cde27..6c2e3cbc3 100644 --- a/nix/packages/cleanup-ami.nix +++ b/nix/packages/cleanup-ami.nix @@ -45,7 +45,7 @@ runCommand "cleanup-ami" fi AMI_NAME="$1" - REGION="ap-southeast-1" + REGION="us-east-1" # Deregister AMIs for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do diff --git a/nix/packages/run-testinfra.nix b/nix/packages/run-testinfra.nix index 92f9d0a4a..c17c66eee 100644 --- a/nix/packages/run-testinfra.nix +++ b/nix/packages/run-testinfra.nix @@ -93,19 +93,19 @@ runCommand "run-testinfra" fi # Set environment variables - export AWS_REGION="ap-southeast-1" - export AWS_DEFAULT_REGION="ap-southeast-1" + export AWS_REGION="us-east-1" + export AWS_DEFAULT_REGION="us-east-1" export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID # Function to terminate EC2 instances terminate_instances() { echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." - aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \ + aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region us-east-1 describe-instances \ --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ --query "Reservations[].Instances[].InstanceId" \ --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \ - --region ap-southeast-1 --instance-ids || true + --region us-east-1 --instance-ids || true } # Set up traps for various signals to ensure cleanup diff --git a/testinfra/README.md b/testinfra/README.md index 6ecdeadb5..9960fc6df 100644 --- a/testinfra/README.md +++ b/testinfra/README.md @@ -56,8 +56,8 @@ AWS_PROFILE=supabase-dev packer build \ -var-file=common.vars.pkr.hcl \ -var "ansible_arguments=" \ -var "postgres-version=ci-ami-test" \ - -var "region=ap-southeast-1" \ - -var 'ami_regions=["ap-southeast-1"]' \ + -var "region=us-east-1" \ + -var 'ami_regions=["us-east-1"]' \ -var "force-deregister=true" \ amazon-arm64.pkr.hcl diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 123ef1baf..476ebf3ad 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -132,7 +132,7 @@ """ walg_config_json_content = """ { - "AWS_REGION": "ap-southeast-1", + "AWS_REGION": "us-east-1", "WALG_S3_PREFIX": "", "PGDATABASE": "postgres", "PGUSER": "supabase_admin", @@ -158,7 +158,7 @@ "service_key": "{service_role_key}", "supabase_admin_key": "{supabase_admin_key}", "common_name": "db.aaaaaaaaaaaaaaaaaaaa.supabase.red", - "region": "ap-southeast-1", + "region": "us-east-1", "init_database_only": false }} """ @@ -223,7 +223,7 @@ def run_ssh_command(ssh, command, timeout=None): # scope='function' uses a new container per test function. @pytest.fixture(scope="session") def host(): - ec2 = boto3.resource("ec2", region_name="ap-southeast-1") + ec2 = boto3.resource("ec2", region_name="us-east-1") images = list( ec2.images.filter( Filters=[{"Name": "name", "Values": [AMI_NAME]}], @@ -256,7 +256,7 @@ def gzip_then_base64_encode(s: str) -> str: "HttpTokens": "required", "HttpEndpoint": "enabled", }, - IamInstanceProfile={"Name": "pg-ap-southeast-1"}, + IamInstanceProfile={"Name": "pg-us-east-1"}, InstanceType="t4g.micro", MinCount=1, MaxCount=1, @@ -281,7 +281,7 @@ def gzip_then_base64_encode(s: str) -> str: - {{path: /tmp/init.json, content: {gzip_then_base64_encode(init_json_content)}, permissions: '0600', encoding: gz+b64}} runcmd: - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' - - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' + - 'cd /tmp && aws s3 cp --region us-east-1 s3://init-scripts-staging/project/init.sh .' - 'bash init.sh "staging"' - 'touch /var/lib/init-complete' - 'rm -rf /tmp/*' From 6f821212bed250310871dd44927e448805bb7e51 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 12:52:48 -0500 Subject: [PATCH 2/8] fix: try instead to consolidate all on ap-southeast-1 --- .github/workflows/ami-release-nix-single.yml | 6 +++--- .github/workflows/ami-release-nix.yml | 6 +++--- .github/workflows/mirror.yml | 2 +- .github/workflows/publish-migrations-prod.yml | 2 +- .github/workflows/publish-migrations-staging.yml | 2 +- .../publish-nix-pgupgrade-bin-flake-version.yml | 4 ++-- .../workflows/publish-nix-pgupgrade-scripts.yml | 4 ++-- .github/workflows/qemu-image-build.yml | 8 ++++---- .github/workflows/testinfra-ami-build.yml | 14 +++++++------- amazon-arm64-nix.pkr.hcl | 2 +- development-arm.vars.pkr.hcl | 4 ++-- nix/docs/development-workflow.md | 2 +- nix/packages/build-test-ami.nix | 2 +- nix/packages/cleanup-ami.nix | 2 +- nix/packages/run-testinfra.nix | 8 ++++---- testinfra/README.md | 4 ++-- testinfra/test_ami_nix.py | 10 +++++----- 17 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index bb6641ff0..ef9c353a8 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -31,7 +31,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" output-credentials: true role-duration-seconds: 7200 @@ -97,7 +97,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload software manifest to s3 staging run: | @@ -118,7 +118,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload software manifest to s3 prod run: | diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index af003c0e2..619a5abbe 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -55,7 +55,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" output-credentials: true role-duration-seconds: 7200 @@ -126,7 +126,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload software manifest to s3 staging run: | @@ -147,7 +147,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload software manifest to s3 prod run: | diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index 28a068277..920643f5c 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -28,7 +28,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: us-east-1 + aws-region: ap-southeast-1 - uses: docker/login-action@v2 with: registry: public.ecr.aws diff --git a/.github/workflows/publish-migrations-prod.yml b/.github/workflows/publish-migrations-prod.yml index e59bac9ba..81c686db7 100644 --- a/.github/workflows/publish-migrations-prod.yml +++ b/.github/workflows/publish-migrations-prod.yml @@ -32,7 +32,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Deploy to S3 prod shell: bash diff --git a/.github/workflows/publish-migrations-staging.yml b/.github/workflows/publish-migrations-staging.yml index 0c862990d..82f01bcf4 100644 --- a/.github/workflows/publish-migrations-staging.yml +++ b/.github/workflows/publish-migrations-staging.yml @@ -25,7 +25,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Deploy to S3 staging shell: bash diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 1e1b69b1d..59f60c8a4 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -62,7 +62,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -111,7 +111,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 029d7deaf..c2e3c45e7 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -67,7 +67,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -118,7 +118,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index b66f16128..6ecde9f4b 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -103,7 +103,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Login to Amazon ECR id: login-ecr-private-dev @@ -117,7 +117,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 812073016711.dkr.ecr.us-east-1.amazonaws.com + REGISTRY: 812073016711.dkr.ecr.ap-southeast-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | @@ -129,7 +129,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" - name: Login to Amazon ECR id: login-ecr-private-prod @@ -137,7 +137,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 156470330064.dkr.ecr.us-east-1.amazonaws.com + REGISTRY: 156470330064.dkr.ecr.ap-southeast-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 65e8f7013..0363eff7e 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -67,7 +67,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" output-credentials: true role-duration-seconds: 7200 @@ -112,13 +112,13 @@ jobs: run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 @@ -132,12 +132,12 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids || true + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true - name: Cleanup AMIs if: always() @@ -149,10 +149,10 @@ jobs: # Function to deregister AMIs by name pattern deregister_ami_by_name() { local ami_name_pattern=$1 - local ami_ids=$(aws ec2 describe-images --region us-east-1 --owners self --filters "Name=name,Values=${ami_name_pattern}" --query 'Images[*].ImageId' --output text) + local ami_ids=$(aws ec2 describe-images --region ap-southeast-1 --owners self --filters "Name=name,Values=${ami_name_pattern}" --query 'Images[*].ImageId' --output text) for ami_id in $ami_ids; do echo "Deregistering AMI: $ami_id" - aws ec2 deregister-image --region us-east-1 --image-id $ami_id + aws ec2 deregister-image --region ap-southeast-1 --image-id $ami_id done } diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index c2b73cde8..86944a323 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -15,7 +15,7 @@ variable "ami_name" { variable "ami_regions" { type = list(string) - default = ["us-east-1"] + default = ["ap-southeast-1"] } variable "ansible_arguments" { diff --git a/development-arm.vars.pkr.hcl b/development-arm.vars.pkr.hcl index 6772bf6d0..3a07e0b53 100644 --- a/development-arm.vars.pkr.hcl +++ b/development-arm.vars.pkr.hcl @@ -1,7 +1,7 @@ arch = "arm64" -ami_regions = ["us-east-1"] +ami_regions = ["ap-southeast-1"] environment = "dev" instance-type = "c6g.4xlarge" -region= "us-east-1" +region= "ap-southeast-1" ubuntu-2004 = "ami-0b49a4a6e8e22fa16" diff --git a/nix/docs/development-workflow.md b/nix/docs/development-workflow.md index c3b66ba50..695427abc 100644 --- a/nix/docs/development-workflow.md +++ b/nix/docs/development-workflow.md @@ -110,7 +110,7 @@ This will: The following environment variables are used: - `AWS_VAULT`: AWS Vault profile name (default: staging) -- `AWS_REGION`: AWS region (default: us-east-1) +- `AWS_REGION`: AWS region (default: ap-southeast-1) - `AMI_NAME`: Name of the AMI to test ## Best Practices diff --git a/nix/packages/build-test-ami.nix b/nix/packages/build-test-ami.nix index f63968e27..115ea793d 100644 --- a/nix/packages/build-test-ami.nix +++ b/nix/packages/build-test-ami.nix @@ -84,7 +84,7 @@ runCommand "build-test-ami" fi # Set values - REGION="us-east-1" + REGION="ap-southeast-1" POSTGRES_VERSION="$1" RANDOM_STRING=$(openssl rand -hex 8) GIT_SHA=$(git rev-parse HEAD) diff --git a/nix/packages/cleanup-ami.nix b/nix/packages/cleanup-ami.nix index 6c2e3cbc3..f1f9cde27 100644 --- a/nix/packages/cleanup-ami.nix +++ b/nix/packages/cleanup-ami.nix @@ -45,7 +45,7 @@ runCommand "cleanup-ami" fi AMI_NAME="$1" - REGION="us-east-1" + REGION="ap-southeast-1" # Deregister AMIs for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do diff --git a/nix/packages/run-testinfra.nix b/nix/packages/run-testinfra.nix index c17c66eee..92f9d0a4a 100644 --- a/nix/packages/run-testinfra.nix +++ b/nix/packages/run-testinfra.nix @@ -93,19 +93,19 @@ runCommand "run-testinfra" fi # Set environment variables - export AWS_REGION="us-east-1" - export AWS_DEFAULT_REGION="us-east-1" + export AWS_REGION="ap-southeast-1" + export AWS_DEFAULT_REGION="ap-southeast-1" export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID # Function to terminate EC2 instances terminate_instances() { echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." - aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region us-east-1 describe-instances \ + aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \ --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ --query "Reservations[].Instances[].InstanceId" \ --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \ - --region us-east-1 --instance-ids || true + --region ap-southeast-1 --instance-ids || true } # Set up traps for various signals to ensure cleanup diff --git a/testinfra/README.md b/testinfra/README.md index 9960fc6df..6ecdeadb5 100644 --- a/testinfra/README.md +++ b/testinfra/README.md @@ -56,8 +56,8 @@ AWS_PROFILE=supabase-dev packer build \ -var-file=common.vars.pkr.hcl \ -var "ansible_arguments=" \ -var "postgres-version=ci-ami-test" \ - -var "region=us-east-1" \ - -var 'ami_regions=["us-east-1"]' \ + -var "region=ap-southeast-1" \ + -var 'ami_regions=["ap-southeast-1"]' \ -var "force-deregister=true" \ amazon-arm64.pkr.hcl diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 476ebf3ad..123ef1baf 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -132,7 +132,7 @@ """ walg_config_json_content = """ { - "AWS_REGION": "us-east-1", + "AWS_REGION": "ap-southeast-1", "WALG_S3_PREFIX": "", "PGDATABASE": "postgres", "PGUSER": "supabase_admin", @@ -158,7 +158,7 @@ "service_key": "{service_role_key}", "supabase_admin_key": "{supabase_admin_key}", "common_name": "db.aaaaaaaaaaaaaaaaaaaa.supabase.red", - "region": "us-east-1", + "region": "ap-southeast-1", "init_database_only": false }} """ @@ -223,7 +223,7 @@ def run_ssh_command(ssh, command, timeout=None): # scope='function' uses a new container per test function. @pytest.fixture(scope="session") def host(): - ec2 = boto3.resource("ec2", region_name="us-east-1") + ec2 = boto3.resource("ec2", region_name="ap-southeast-1") images = list( ec2.images.filter( Filters=[{"Name": "name", "Values": [AMI_NAME]}], @@ -256,7 +256,7 @@ def gzip_then_base64_encode(s: str) -> str: "HttpTokens": "required", "HttpEndpoint": "enabled", }, - IamInstanceProfile={"Name": "pg-us-east-1"}, + IamInstanceProfile={"Name": "pg-ap-southeast-1"}, InstanceType="t4g.micro", MinCount=1, MaxCount=1, @@ -281,7 +281,7 @@ def gzip_then_base64_encode(s: str) -> str: - {{path: /tmp/init.json, content: {gzip_then_base64_encode(init_json_content)}, permissions: '0600', encoding: gz+b64}} runcmd: - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' - - 'cd /tmp && aws s3 cp --region us-east-1 s3://init-scripts-staging/project/init.sh .' + - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' - 'bash init.sh "staging"' - 'touch /var/lib/init-complete' - 'rm -rf /tmp/*' From 717b3ca5e531c3cf63e0a25ab52359a8dc881e66 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 13:41:49 -0500 Subject: [PATCH 3/8] fix: 2 more regions to consolidate --- .github/actions/nix-install-ephemeral/action.yml | 2 +- stage2-nix-psql.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index caa9a051d..f95bf64f2 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -13,7 +13,7 @@ runs: if: ${{ inputs.push-to-cache == 'true' }} with: role-to-assume: ${{ env.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: "ap-southeast-1" output-credentials: true role-duration-seconds: 7200 - name: Setup AWS credentials for Nix diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 243e5e2e4..344ced288 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -5,7 +5,7 @@ variable "profile" { variable "ami_regions" { type = list(string) - default = ["ap-southeast-2"] + default = ["ap-southeast-1"] } variable "environment" { From 654d4ba00d8abb741140d502cafe4aec08cbbf59 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 14:20:54 -0500 Subject: [PATCH 4/8] fix: mirror docker in us-east-1 --- .github/workflows/mirror.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index 920643f5c..28a068277 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -28,7 +28,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: ap-southeast-1 + aws-region: us-east-1 - uses: docker/login-action@v2 with: registry: public.ecr.aws From 6b6d60416d485ff67bb379a5f25278e3cdf408e3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 14:24:50 -0500 Subject: [PATCH 5/8] fix: restore some builds back to us-east-1 --- .../workflows/publish-nix-pgupgrade-bin-flake-version.yml | 4 ++-- .github/workflows/publish-nix-pgupgrade-scripts.yml | 4 ++-- .github/workflows/qemu-image-build.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 59f60c8a4..c3207f972 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -62,7 +62,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -111,7 +111,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index c2e3c45e7..3cbf47324 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -67,7 +67,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -118,7 +118,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 6ecde9f4b..14882fc5b 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -103,7 +103,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Login to Amazon ECR id: login-ecr-private-dev @@ -117,7 +117,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 812073016711.dkr.ecr.ap-southeast-1.amazonaws.com + REGISTRY: 812073016711.dkr.ecr.us-east-1-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | @@ -129,7 +129,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1-1" - name: Login to Amazon ECR id: login-ecr-private-prod @@ -137,7 +137,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 156470330064.dkr.ecr.ap-southeast-1.amazonaws.com + REGISTRY: 156470330064.dkr.ecr.us-east-1-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | From 5d925b31c5d946c2f895a292d3098cca36a4c15e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 14:31:58 -0500 Subject: [PATCH 6/8] fix: revert some builds to us-east-1 to make sure they still function while investigating if they can change --- .../workflows/publish-nix-pgupgrade-bin-flake-version.yml | 4 ++-- .github/workflows/publish-nix-pgupgrade-scripts.yml | 4 ++-- .github/workflows/qemu-image-build.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index c3207f972..1e1b69b1d 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -62,7 +62,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -111,7 +111,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 3cbf47324..029d7deaf 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -67,7 +67,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Upload pg_upgrade scripts to s3 staging run: | @@ -118,7 +118,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Upload pg_upgrade scripts to s3 prod run: | diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 14882fc5b..b66f16128 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -103,7 +103,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Login to Amazon ECR id: login-ecr-private-dev @@ -117,7 +117,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 812073016711.dkr.ecr.us-east-1-1.amazonaws.com + REGISTRY: 812073016711.dkr.ecr.us-east-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | @@ -129,7 +129,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} - aws-region: "us-east-1-1" + aws-region: "us-east-1" - name: Login to Amazon ECR id: login-ecr-private-prod @@ -137,7 +137,7 @@ jobs: - name: Push docker image to Amazon ECR env: - REGISTRY: 156470330064.dkr.ecr.us-east-1-1.amazonaws.com + REGISTRY: 156470330064.dkr.ecr.us-east-1.amazonaws.com REPOSITORY: postgres-vm-image IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | From 20907f12d477284538dac1f76c7b12a4cbfcc2de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 14:42:27 -0500 Subject: [PATCH 7/8] fix: restore these IAM config actions --- .github/workflows/ami-release-nix-single.yml | 6 +++--- .github/workflows/ami-release-nix.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index ef9c353a8..bb6641ff0 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -31,7 +31,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" output-credentials: true role-duration-seconds: 7200 @@ -97,7 +97,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Upload software manifest to s3 staging run: | @@ -118,7 +118,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Upload software manifest to s3 prod run: | diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 619a5abbe..af003c0e2 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -55,7 +55,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" output-credentials: true role-duration-seconds: 7200 @@ -126,7 +126,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Upload software manifest to s3 staging run: | @@ -147,7 +147,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" - name: Upload software manifest to s3 prod run: | From a4fc46e842fc52057f3874f8179638976750fe93 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Nov 2025 14:43:58 -0500 Subject: [PATCH 8/8] fix: restore this role to assume action region --- .github/actions/nix-install-ephemeral/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index f95bf64f2..caa9a051d 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -13,7 +13,7 @@ runs: if: ${{ inputs.push-to-cache == 'true' }} with: role-to-assume: ${{ env.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: "us-east-1" output-credentials: true role-duration-seconds: 7200 - name: Setup AWS credentials for Nix