dbmate migration running and testing tool (#1316)

* feat: basic running of dbmate migrations from tool

* feat: a nix flake app tool to run dbmate migrations and produce
schema.slq files

* feat: test against schema.sql per mjr vrs + tool to generate

* chore: pin at development branch version of tool

* docs: update docs with usage info

* fix: tmp use this working branch for flakeurl

* chore: cleanup of test and dbmage migration tool

* chore: build this version

* chore: extra-conf

* chore: reinstall true

* chore: test no longer needs self-hosted runner, can use ephemeral

* chore: correct format of runner

* chore: not longer need sudo

* chore: trying to get the action to use our cache

* chore: conf 2nd invocation

* chore: rm var

* chore: nl

* chore: cleanup and remove unnecessary changes

* fix: make sure to run sql for pg-stat-extensions and pg-bouncer

Author: samrose

.github/workflows/test.yml

@@ -17,6 +17,10 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          extra-conf: |
+            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
+            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
 
       - name: Set PostgreSQL versions
         id: set-versions
@@ -30,19 +34,24 @@ jobs:
       matrix:
         postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
         include:
-          - runner: [self-hosted, X64]
+          - runner: ubuntu-22.04
             arch: amd64
-          - runner: arm-runner
+          - runner: ubuntu-22.04
             arch: arm64
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 180
     env:
       POSTGRES_PORT: 5478
       POSTGRES_PASSWORD: password
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          extra-conf: |
+            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
+            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+
 
       - name: Set PostgreSQL version environment variable
         run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
@@ -54,7 +63,7 @@ jobs:
 
       - name: Generate common-nix.vars.pkr.hcl
         run: |
-          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
           PG_VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
           # Ensure there's a newline at the end of the file
@@ -67,96 +76,16 @@ jobs:
       - name: Generate args
         id: args
         run: |
-          ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml)
+          ARGS=$(nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml)
           echo "result<<EOF" >> $GITHUB_OUTPUT
           echo "$ARGS" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
-      - run: docker context create builders
-      - uses: docker/setup-buildx-action@v3
-        with:
-          endpoint: builders
-      - uses: docker/build-push-action@v5
-        with:
-          load: true
-          context: .
-          file: Dockerfile-${{ env.PGMAJOR }}
-          target: production
-          build-args: |
-            ${{ steps.args.outputs.result }}
-          tags: supabase/postgres:${{ steps.settings.outputs.postgres-version }},supabase_postgres
-          cache-from: |
-            type=gha,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-            type=gha,scope=${{ github.base_ref }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-
-      - name: Start Postgres
-        run: |
-          docker run --rm --pull=never \
-          -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \
-          -p ${{ env.POSTGRES_PORT }}:5432 \
-          --name supabase_postgres \
-          -d supabase/postgres:${{ steps.settings.outputs.postgres-version }}
-
-      - name: Install psql
-        run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-          sudo apt update
-          sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }}
-
-      - name: Install pg_prove
-        run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP
-        env:
-          SHELL: /bin/bash
-
-      - name: Wait for healthy database
-        run: |
-          count=0
-          until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
-              exit=$?
-              count=$((count + 1))
-              if [ $count -ge "$retries" ]; then
-                  echo "Retry $count/$retries exited $exit, no more retries left."
-                  docker stop -t 2 "$container"
-                  return $exit
-              fi
-              sleep 1;
-          done;
-          echo "$container container is healthy"
-        env:
-          retries: 20
-          container: supabase_postgres
-
-      - name: Run tests
-        run: pg_prove migrations/tests/test.sql
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Check migrations are idempotent
-        run: |
-          for sql in ./migrations/db/migrations/*.sql; do
-            echo "$0: running $sql"
-            psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql"
-          done
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Update Dockerfile.dbmate version
-        run: |
-          sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate
-
+      #TODO PR Convert to develop branch flakeurl
       - name: verify schema.sql is committed
         run: |
-          docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit
+          GIT_SHA=${{github.sha}}
+          nix run github:supabase/postgres/${GIT_SHA}#dbmate-tool -- --version ${{ env.PGMAJOR }}
           if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
             echo "Detected changes in schema.sql:"
             git diff migrations/schema-${{ env.PGMAJOR }}.sql

.gitignore

@@ -22,3 +22,5 @@ result*
 #IDE
 .idea/
 .vscode/
+
+db

flake.nix

@@ -530,7 +530,7 @@
               chmod +x $out/bin/pg-restore
             '';
           sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } ''
-            mkdir -p $out/bin
+            mkdir -p $out/bin 
             substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \
               --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \
               --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \
@@ -539,8 +539,42 @@
               --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix'
             chmod +x $out/bin/sync-exts-versions
           '';
+          dbmate-tool = 
+          let
+            migrationsDir = ./migrations/db;
+            ansibleVars = ./ansible/vars.yml;
+            pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
+            statExtensionSql = ./ansible/files/stat_extension.sql;
+          in
+          pkgs.runCommand "dbmate-tool" {
+            buildInputs = with pkgs; [
+              overmind
+              dbmate
+              nix
+              jq
+              yq
+            ];
+            nativeBuildInputs = with pkgs; [
+              makeWrapper
+            ];
+          } ''
+            mkdir -p $out/bin $out/migrations 
+            cp -r ${migrationsDir}/* $out
+            substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \
+              --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
+              --subst-var-by 'MIGRATIONS_DIR' $out \
+              --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
+              --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \
+              --subst-var-by 'CURRENT_SYSTEM' '${system}' \
+              --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
+              --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
+            chmod +x $out/bin/dbmate-tool
+            wrapProgram $out/bin/dbmate-tool \
+              --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]}
+          '';       
         };
 
+
         # Create a testing harness for a PostgreSQL package. This is used for
         # 'nix flake check', and works with any PostgreSQL package you hand it.
         makeCheckHarness = pgpkg:
@@ -654,9 +688,11 @@
             start-server = mkApp "start-server" "start-postgres-server";
             start-client = mkApp "start-client" "start-postgres-client";
             start-replica = mkApp "start-replica" "start-postgres-replica";
-            migration-test = mkApp "migrate-tool" "migrate-postgres";
+            migrate-postgres = mkApp "migrate-tool" "migrate-postgres";
             sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions";
             pg-restore = mkApp "pg-restore" "pg-restore";
+            dbmate-tool = mkApp "dbmate-tool" "dbmate-tool";
+            migration-unit-tests = mkApp "migration-unit-tests" "migration-unit-tests";
           };
 
         # 'devShells.default' lists the set of packages that are included in the
@@ -695,6 +731,7 @@
             basePackages.start-replica
             basePackages.migrate-tool
             basePackages.sync-exts-versions
+            dbmate
           ];
           shellHook = ''
             export HISTFILE=.history

migrations/README.md

@@ -1,3 +1,36 @@
+# Usage
+
+from the root of the `supabase/postgres` project, you can run the following commands:
+
+
+```shell
+Usage: nix run .#dbmate-tool -- [options]
+
+Options:
+  -v, --version [15|16|orioledb-17|all]  Specify the PostgreSQL version to use (required defaults to --version all)
+  -p, --port PORT                    Specify the port number to use (default: 5435)
+  -h, --help                         Show this help message
+
+Description:
+  Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions.
+  NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new <migration_name>' to create a new migration file.
+
+Examples:
+  nix run .#dbmate-tool
+  nix run .#dbmate-tool -- --version 15
+  nix run .#dbmate-tool -- --version 16 --port 5433
+
+```
+
+This can also be run from a github "flake url" for example:
+
+```shell
+nix run github:supabase/postgres#dbmate-tool -- --version 15
+
+or
+
+nix run github:supabase/postgres/mybranch#dbmate-tool -- --version 15
+```
 # supabase/migrations
 
 `supabase/migrations` is a consolidation of SQL migrations from:
@@ -9,6 +42,8 @@
 
 aiming to provide a single source of truth for migrations on the platform that can be depended upon by those components. For more information on goals see [the RFC](https://www.notion.so/supabase/Centralize-SQL-Migrations-cd3847ae027d4f2bba9defb2cc82f69a)
 
+
+
 ## How it was Created
 
 Migrations were pulled (in order) from:
@@ -20,10 +55,12 @@ For compatibility with hosted projects, we include [migrate.sh](migrate.sh) that
 
 1. Run all `db/init-scripts` with `postgres` superuser role.
 2. Run all `db/migrations` with `supabase_admin` superuser role.
-3. Finalize role passwords with `/etc/postgres.schema.sql` if present.
+3. Finalize role passwords with `/etc/postgresql.schema.sql` if present.
 
 Additionally, [supabase/postgres](https://github.com/supabase/postgres/blob/develop/ansible/playbook-docker.yml#L9) image contains several migration scripts to configure default extensions. These are run first by docker entrypoint and included in ami by ansible.
 
+
+
 ## Guidelines
 
 - Migrations are append only. Never edit existing migrations once they are on master.