feat: automate pg_upgrade testing (#695)

.github/workflows/publish-pgupgrade-scripts.yml

@@ -13,7 +13,7 @@ permissions:
   id-token: write
 
 jobs:
-  build:
+  publish-staging:
     runs-on: ubuntu-latest
 
     steps:
@@ -42,6 +42,36 @@ jobs:
         run: |
           aws s3 cp /tmp/pg_upgrade_scripts.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz
 
+      - name: Slack Notification on Failure
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          SLACK_USERNAME: 'gha-failures-notifier'
+          SLACK_COLOR: 'danger'
+          SLACK_MESSAGE: 'Publishing pg_upgrade scripts failed'
+          SLACK_FOOTER: ''
+
+  publish-prod:
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'develop'
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(grep 'postgres-version' common.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Create a tarball containing pg_upgrade scripts
+        run: |
+          mkdir -p /tmp/pg_upgrade_scripts
+          cp -r ansible/files/admin_api_scripts/pg_upgrade_scripts/* /tmp/pg_upgrade_scripts
+          tar -czvf /tmp/pg_upgrade_scripts.tar.gz -C /tmp/ pg_upgrade_scripts
+
       - name: configure aws credentials - prod
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/test-pg-upgrade.yml

@@ -0,0 +1,129 @@
+name: Test pg_upgrade
+
+on:
+  push:
+    branches:
+      - develop
+      - pcnc/auto-pg_upgrade-testing
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        base_pg_version:
+          - 15.1.1.60
+    runs-on: arm-runner
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: ./tests/pg_upgrade
+    env:
+      PGPORT: 5478
+      PGPASSWORD: postgres
+      PGDATABASE: postgres
+      PGUSER: supabase_admin
+      PGHOST: localhost
+      PG_MAJOR_VERSION: 15
+      IS_CI: true
+      container: pg_upgrade_test
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Grab release version
+        id: process_release_version
+        working-directory: ./
+        run: |
+          VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl)
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Download pg_upgrade_scripts and binaries
+        run: |
+          aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz scripts/pg_upgrade_scripts.tar.gz
+          aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz scripts/pg_upgrade_bin.tar.gz
+      
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+          driver-opts: image=moby/buildkit:v0.11.6
+          buildkitd-flags: --debug
+
+      - name: Start Postgres
+        run: |
+          docker rm -f "$container" || true
+          docker run --name "$container" --env-file .env \
+          -v "$(pwd)/scripts:/tmp/upgrade" \
+          --entrypoint "/tmp/upgrade/entrypoint.sh" -d \
+          -p ${PGPORT}:5432 \
+          "supabase/postgres:${{ matrix.base_pg_version }}"
+
+      - name: Install psql
+        run: |
+          sudo apt update
+          sudo apt install -y --no-install-recommends postgresql-client
+
+      - name: Install pg_prove
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends perl cpanminus
+          sudo cpanm -n App::cpanminus
+          sudo cpanm -n TAP::Parser::SourceHandler::pgTAP
+        env:
+          SHELL: /bin/bash
+          PERL_MM_USE_DEFAULT: 1
+          PERL_MM_NONINTERACTIVE: 1
+
+      - name: Wait for healthy database
+        run: |
+          count=0
+          while ! docker exec "$container" bash -c "pg_isready"; do
+            count=$((count + 1))
+            if [ $count -ge "$retries" ]; then
+                echo "Retry $count/$retries exited $exit, no more retries left."
+                docker logs "$container"
+                docker rm -f "$container"
+                exit 1
+            fi
+          done
+        env:
+          retries: 20
+
+      - name: Run migrations
+        run: docker exec "$container" bash -c "/docker-entrypoint-initdb.d/migrate.sh > /tmp/migrate.log 2>&1"
+
+      - name: Run initial tests
+        run: pg_prove "../../migrations/tests/test.sql"
+        env:
+          PERL5LIB: /usr/local/lib/perl5
+
+      - name: Apply pre-upgrade fixtures
+        run: |
+          psql -f "./tests/98-data-fixtures.sql"
+          psql -f "./tests/99-fixtures.sql"
+          
+      - name: Initiate upgrade
+        run: docker exec "$container" bash -c '/tmp/upgrade/pg_upgrade_scripts/initiate.sh "$PG_MAJOR_VERSION"; exit $?'
+
+      - name: Complete pg_upgrade
+        run: docker exec pg_upgrade_test bash -c '/tmp/upgrade/pg_upgrade_scripts/complete.sh; exit $?'
+
+      - name: Run post-upgrade tests
+        run: |
+          pg_prove tests/01-schema.sql
+          pg_prove tests/02-data.sql
+          pg_prove tests/03-settings.sql
+
+      - name: Clean up container
+        if: ${{ always() }}
+        continue-on-error: true
+        run: docker rm -f "$container" || true

ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh

@@ -61,3 +61,25 @@ function retry {
   done
   return 0
 }
+
+CI_stop_postgres() {
+    BINDIR=$(pg_config --bindir)
+    ARG=${1:-""}
+
+    if [ "$ARG" = "--new-bin" ]; then
+        BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
+    fi
+
+    su postgres -c "$BINDIR/pg_ctl stop -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
+}
+
+CI_start_postgres() {
+    BINDIR=$(pg_config --bindir)
+    ARG=${1:-""}
+
+    if [ "$ARG" = "--new-bin" ]; then
+        BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
+    fi
+
+    su postgres -c "$BINDIR/pg_ctl start -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
+}

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -11,6 +11,7 @@ SCRIPT_DIR=$(dirname -- "$0";)
 # shellcheck disable=SC1091
 source "$SCRIPT_DIR/common.sh"
 
+IS_CI=${IS_CI:-}
 LOG_FILE="/var/log/pg-upgrade-complete.log"
 
 function cleanup {
@@ -81,14 +82,22 @@ function complete_pg_upgrade {
     echo "running" > /tmp/pg-upgrade-status
 
     echo "1. Mounting data disk"
-    retry 8 mount -a -v
+    if [ -z "$IS_CI" ]; then
+        retry 8 mount -a -v
+    else
+        echo "Skipping mount -a -v"
+    fi
 
     # copying custom configurations
     echo "2. Copying custom configurations"
     retry 3 copy_configs
 
     echo "3. Starting postgresql"
-    retry 3 service postgresql start
+    if [ -z "$IS_CI" ]; then
+        retry 3 service postgresql start
+    else
+        CI_start_postgres --new-bin
+    fi
 
     echo "4. Running generated SQL files"
     retry 3 run_generated_sql
@@ -104,11 +113,16 @@ function complete_pg_upgrade {
     sleep 5
 
     echo "5. Restarting postgresql"
-    retry 3 service postgresql restart
-
-    echo "5.1. Restarting gotrue and postgrest"
-    retry 3 service gotrue restart
-    retry 3 service postgrest restart
+    if [ -z "$IS_CI" ]; then
+        retry 3 service postgresql restart
+
+        echo "5.1. Restarting gotrue and postgrest"
+        retry 3 service gotrue restart
+        retry 3 service postgrest restart
+    else
+        retry 3 CI_stop_postgres || true
+        retry 3 CI_start_postgres
+    fi
 
     echo "6. Starting vacuum analyze"
     retry 3 start_vacuum_analyze
@@ -137,7 +151,10 @@ function apply_auth_scheme_updates {
     if [ "$PASSWORD_ENCRYPTION_SETTING" = "md5" ]; then
         run_sql -c "ALTER SYSTEM SET password_encryption TO 'scram-sha-256';"
         run_sql -c "SELECT pg_reload_conf();"
-        run_sql -f /etc/postgresql.schema.sql
+
+        if [ -z "$IS_CI" ]; then
+            run_sql -f /etc/postgresql.schema.sql
+        fi
     fi
 }
 
@@ -149,5 +166,16 @@ function start_vacuum_analyze {
 
 trap cleanup ERR
 
+if [ -z "$IS_CI" ]; then
+    complete_pg_upgrade >> $LOG_FILE 2>&1 &
+else 
+    CI_stop_postgres || true
+
+    rm -f /tmp/pg-upgrade-status
+    mv /data_migration /data
+
+    rm -rf /var/lib/postgresql/data
+    ln -s /data/pgdata /var/lib/postgresql/data
 
-complete_pg_upgrade >> $LOG_FILE 2>&1 &
+    complete_pg_upgrade
+fi