feat: automate pg_upgrade testing

.github/workflows/test-pg-upgrade.yml

@@ -0,0 +1,130 @@
+name: Test pg_upgrade
+
+on:
+  push:
+    branches:
+      - develop
+      - pcnc/auto-pg_upgrade-testing
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        base_pg_version:
+          - 15.1.0.100
+    runs-on: arm-runner
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: ./tests/pg_upgrade
+    env:
+      PGPORT: 5478
+      PGPASSWORD: postgres
+      PGDATABASE: postgres
+      PGUSER: supabase_admin
+      PGHOST: localhost
+      PG_MAJOR_VERSION: 15
+      IS_CI: true
+      container: pg_upgrade_test
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Grab release version
+        id: process_release_version
+        working-directory: ./
+        run: |
+          VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl)
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Download pg_upgrade_scripts and binaries
+        run: |
+          aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz scripts/pg_upgrade_scripts.tar.gz
+          aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz scripts/pg_upgrade_bin.tar.gz
+
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+          driver-opts: image=moby/buildkit:v0.11.6
+          buildkitd-flags: --debug
+
+      - name: Start Postgres
+        run: |
+          docker rm -f "$container" || true
+          docker run --name "$container" --env-file .env \
+          -v "$(pwd)/scripts:/tmp/upgrade" \
+          --entrypoint "/tmp/upgrade/entrypoint.sh" -d \
+          -p ${PGPORT}:5432 \
+          "supabase/postgres:${{ matrix.base_pg_version }}"
+
+      - name: Install psql
+        run: |
+          sudo apt update
+          sudo apt install -y --no-install-recommends postgresql-client
+
+      - name: Install pg_prove
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends perl cpanminus
+          sudo cpanm -n App::cpanminus
+          sudo cpanm -n TAP::Parser::SourceHandler::pgTAP
+        env:
+          SHELL: /bin/bash
+          PERL_MM_USE_DEFAULT: 1
+          PERL_MM_NONINTERACTIVE: 1
+
+      - name: Wait for healthy database
+        run: |
+          count=0
+          while ! docker exec "$container" bash -c "pg_isready"; do
+            count=$((count + 1))
+            if [ $count -ge "$retries" ]; then
+                echo "Retry $count/$retries exited $exit, no more retries left."
+                docker logs "$container"
+                docker rm -f "$container"
+                exit 1
+            fi
+          done
+        env:
+          retries: 20
+
+      - name: Run migrations
+        run: docker exec "$container" bash -c "/docker-entrypoint-initdb.d/migrate.sh > /tmp/migrate.log 2>&1"
+
+      - name: Run initial tests
+        run: pg_prove "../../migrations/tests/test.sql"
+        env:
+          PERL5LIB: /usr/local/lib/perl5
+
+      - name: Apply pre-upgrade fixtures
+        run: |
+          psql -f "./tests/98-data-fixtures.sql"
+          psql -f "./tests/99-fixtures.sql"
+
+      - name: Initiate upgrade
+        run: docker exec "$container" bash -c '/tmp/upgrade/pg_upgrade_scripts/initiate.sh "$PG_MAJOR_VERSION"; exit $?'
+
+      - name: Complete pg_upgrade
+        run: docker exec pg_upgrade_test bash -c '/tmp/upgrade/pg_upgrade_scripts/complete.sh; exit $?'
+
+      - name: Run post-upgrade tests
+        run: |
+          pg_prove tests/01-schema.sql
+          pg_prove tests/02-data.sql
+          pg_prove tests/03-settings.sql
+
+      - name: Clean up container
+        if: ${{ always() }}
+        continue-on-error: true
+        run: docker rm -f "$container" || true

ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh

@@ -61,3 +61,33 @@ function retry {
   done
   return 0
 }
+
+stop_postgres() {
+    BINDIR=$(pg_config --bindir)
+    ARG=${1:-""}
+
+    if [ "$ARG" = "--new-bin" ]; then
+        BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
+    fi
+
+    if [ -z "$IS_CI" ]; then
+        systemctl stop postgresql
+    else
+        su postgres -c "$BINDIR/pg_ctl stop -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
+    fi
+}
+
+start_postgres() {
+    BINDIR=$(pg_config --bindir)
+    ARG=${1:-""}
+
+    if [ "$ARG" = "--new-bin" ]; then
+        BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
+    fi
+
+    if [ -z "$IS_CI" ]; then
+        systemctl start postgresql
+    else
+        su postgres -c "$BINDIR/pg_ctl start -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
+    fi
+}

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -33,22 +33,31 @@ function complete_pg_upgrade {
     echo "running" > /tmp/pg-upgrade-status
 
     echo "1. Mounting data disk"
-    retry 3 mount -a -v
+    if [ -z "$IS_CI" ]; then
+        retry 3 mount -a -v
+    else
+        echo "Skipping mount -a -v"
+    fi
 
     # copying custom configurations
     echo "2. Copying custom configurations"
     retry 3 copy_configs
 
     echo "3. Starting postgresql"
-    retry 3 service postgresql start
+    if [ -z "$IS_CI" ]; then
+        retry 3 start_postgres
+    else
+        start_postgres --new-bin
+    fi
 
     echo "4. Running generated SQL files"
     retry 3 run_generated_sql
 
     sleep 5
 
     echo "5. Restarting postgresql"
-    retry 3 service postgresql restart
+    retry 3 stop_postgres || true
+    retry 3 start_postgres
 
     echo "6. Starting vacuum analyze"
     retry 3 start_vacuum_analyze
@@ -78,5 +87,16 @@ function start_vacuum_analyze {
 
 trap cleanup ERR
 
+if [ -z "$IS_CI" ]; then
+    complete_pg_upgrade >> $LOG_FILE 2>&1 &
+else 
+    stop_postgres || true
+
+    rm -f /tmp/pg-upgrade-status
+    mv /data_migration /data
+
+    rm -rf /var/lib/postgresql/data
+    ln -s /data/pgdata /var/lib/postgresql/data
 
-complete_pg_upgrade >> $LOG_FILE 2>&1 &
+    complete_pg_upgrade
+fi

ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh

@@ -34,11 +34,6 @@ source "$SCRIPT_DIR/common.sh"
 LOG_FILE="/var/log/pg-upgrade-initiate.log"
 
 PGVERSION=$1
-IS_DRY_RUN=${2:-false}
-if [ "$IS_DRY_RUN" != false ]; then
-    IS_DRY_RUN=true
-fi
-
 MOUNT_POINT="/data_migration"
 
 POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql"
@@ -52,11 +47,16 @@ if [[ "$OLD_PGVERSION" =~ 13* ]]; then
     EXTENSIONS_TO_DISABLE+=("${PG13_EXTENSIONS_TO_DISABLE[@]}")
 fi
 
+PGBINOLD="$(pg_config --bindir)"
 
 cleanup() {
     UPGRADE_STATUS=${1:-"failed"}
     EXIT_CODE=${?:-0}
 
+    if [ "$UPGRADE_STATUS" = "failed" ]; then
+        EXIT_CODE=1
+    fi
+
     if [ "$UPGRADE_STATUS" = "failed" ]; then
         echo "Upgrade job failed. Cleaning up and exiting."
     fi
@@ -71,21 +71,21 @@ cleanup() {
 
     if [ -L "/usr/share/postgresql/${PGVERSION}" ]; then
         rm "/usr/share/postgresql/${PGVERSION}"
+    fi
 
-        if [ -f "/usr/share/postgresql/${PGVERSION}.bak" ]; then
-            mv "/usr/share/postgresql/${PGVERSION}.bak" "/usr/share/postgresql/${PGVERSION}"
-        fi
+    if [ -d "/usr/share/postgresql/${PGVERSION}.bak" ]; then
+        rm -f "/usr/share/postgresql/${PGVERSION}"
+        mv "/usr/share/postgresql/${PGVERSION}.bak" "/usr/share/postgresql/${PGVERSION}"
     fi
 
     if [ -f "/usr/lib/postgresql/lib/aarch64/libpq.so.5.bak" ]; then
         rm /usr/lib/postgresql/lib/aarch64/libpq.so.5
         mv /usr/lib/postgresql/lib/aarch64/libpq.so.5.bak /usr/lib/postgresql/lib/aarch64/libpq.so.5
     fi
 
-    if [ "$IS_DRY_RUN" = false ]; then
-        echo "Restarting postgresql"
-        systemctl restart postgresql
-    fi
+    echo "Restarting postgresql"
+    stop_postgres || true
+    start_postgres
 
     echo "Re-enabling extensions"
     if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then
@@ -95,13 +95,18 @@ cleanup() {
     echo "Removing SUPERUSER grant from postgres"
     run_sql -c "ALTER USER postgres WITH NOSUPERUSER;"
 
-    if [ "$IS_DRY_RUN" = false ]; then
+    if [ "$IS_CI" = false ]; then
         echo "Unmounting data disk from ${MOUNT_POINT}"
         umount $MOUNT_POINT
     fi
     echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status
 
-    exit "$EXIT_CODE"
+    if [ "$IS_CI" = false ]; then
+        exit "$EXIT_CODE"
+    else 
+        echo "CI run complete with code ${EXIT_CODE}. Exiting."
+        exit "$EXIT_CODE"
+    fi
 }
 
 function handle_extensions {
@@ -165,6 +170,7 @@ function initiate_upgrade {
 
     # copy upgrade-specific pgsodium_getkey script into the share dir
     chmod +x "$SCRIPT_DIR/pgsodium_getkey.sh"
+    mkdir -p "$PGSHARENEW/extension"
     cp  "$SCRIPT_DIR/pgsodium_getkey.sh" "$PGSHARENEW/extension/pgsodium_getkey"
     if [ -d "/var/lib/postgresql/extension/" ]; then
         cp  "$SCRIPT_DIR/pgsodium_getkey.sh" "/var/lib/postgresql/extension/pgsodium_getkey"
@@ -190,12 +196,13 @@ function initiate_upgrade {
     DEBIAN_FRONTEND=noninteractive dpkg --configure -a --force-confold || true # handle errors generated by dpkg
 
     # Needed for PostGIS, since it's compiled with Protobuf-C support now
-    echo "3. Installing libprotobuf-c1 if missing"
+    echo "3. Installing libprotobuf-c1 and libicu66 if missing"
     if [[ ! "$(apt list --installed libprotobuf-c1 | grep "installed")" ]]; then
-        apt-get update && apt --fix-broken install -y libprotobuf-c1
+        apt-get update -y
+        apt --fix-broken install -y libprotobuf-c1 libicu66 || true
     fi
 
-    if [ "$IS_DRY_RUN" = false ]; then
+    if [ "$IS_CI" = false ]; then
         # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere
         # excluding nvme0 since it is the root disk
         echo "4. Determining block device to mount"
@@ -213,6 +220,8 @@ function initiate_upgrade {
 
         sleep 1
         resize2fs "$BLOCK_DEVICE"
+    else 
+        mkdir -p "$MOUNT_POINT"
     fi
 
     if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then
@@ -242,7 +251,7 @@ function initiate_upgrade {
 
     UPGRADE_COMMAND=$(cat <<EOF
     time ${PGBINNEW}/pg_upgrade \
-    --old-bindir="/usr/lib/postgresql/bin" \
+    --old-bindir="${PGBINOLD}" \
     --new-bindir=${PGBINNEW} \
     --old-datadir=${PGDATAOLD} \
     --new-datadir=${PGDATANEW} \
@@ -254,12 +263,10 @@ function initiate_upgrade {
 EOF
     )
 
-    if [ "$IS_DRY_RUN" = true ]; then
-        UPGRADE_COMMAND="$UPGRADE_COMMAND --check"
-    else 
-        echo "9. Stopping postgres; running pg_upgrade"
-        systemctl stop postgresql
-    fi
+    su -c "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
+
+    echo "9. Stopping postgres; running pg_upgrade"
+    stop_postgres
 
     su -c "$UPGRADE_COMMAND" -s "$SHELL" postgres
 
@@ -281,7 +288,8 @@ EOF
 trap cleanup ERR
 
 echo "running" > /tmp/pg-upgrade-status
-if [ "$IS_DRY_RUN" = true ]; then
+if [ "$IS_CI" = true ]; then
+    rm -f /tmp/pg-upgrade-status
     initiate_upgrade
 else 
     initiate_upgrade >> "$LOG_FILE" 2>&1 &

tests/pg_upgrade/.env

@@ -0,0 +1,6 @@
+POSTGRES_PASSWORD=postgres
+POSTGRES_HOST=/var/run/postgresql
+POSTGRES_INITDB_ARGS=--lc-ctype=C.UTF-8
+PG_MAJOR_VERSION=15
+IS_CI=true
+SCRIPT_DIR=/tmp/upgrade

tests/pg_upgrade/.gitignore

@@ -0,0 +1,4 @@
+# excluding these since running debug.sh will download the files locally
+pg_upgrade_bin.tar.gz
+pg_upgrade_scripts.tar.gz
+pg_upgrade_scripts/