refactor: improve Atlas migration management with better baseline handling

nx.json

@@ -103,10 +103,6 @@
       "local": true,
       "cache": true
     },
-    "dump-realtime-schema": {
-      "local": true,
-      "cache": true
-    },
     "gen-types": {
       "local": true,
       "cache": true

pkgs/core/.gitignore

@@ -1,2 +1 @@
 atlas/schema.sql
-atlas/.supabase-baseline-schema.sql

pkgs/core/README.md

@@ -5,10 +5,6 @@ PostgreSQL-native workflow engine for defining, managing, and tracking DAG-based
 > [!NOTE]
 > This project and all its components are licensed under [Apache 2.0](./LICENSE) license.
 
-> [!WARNING]
-> This project uses [Atlas](https://atlasgo.io/docs) to manage the schemas and migrations.
-> See [ATLAS.md](ATLAS.md) for more details.
-
 ## Table of Contents
 
 - [Overview](#overview)
@@ -49,8 +45,7 @@ The actual execution of workflow tasks is handled by the [Edge Worker](../edge-w
 
 ## Requirements
 
-> [!IMPORTANT]
-> **pgmq Version Requirement** (since v0.8.0)
+> [!IMPORTANT] > **pgmq Version Requirement** (since v0.8.0)
 >
 > pgflow v0.8.0 and later requires **pgmq 1.5.0 or higher**. This version of pgflow will NOT work with pgmq 1.4.x or earlier.
 >

pkgs/core/atlas/.gitignore

@@ -0,0 +1,2 @@
+# Temp Supabase project for baseline dumping
+supabase/

pkgs/core/atlas/Dockerfile

@@ -1,4 +1,8 @@
-# Use official PostgreSQL 17
+# Atlas Postgres image with extensions for pgflow
+#
+# Extension versions aligned with Supabase Postgres 17.6.1.054
+# Reference: https://github.com/supabase/postgres/blob/17.6.1.054/Dockerfile-17
+
 FROM postgres:17
 
 # Set environment variables for postgres
@@ -10,22 +14,65 @@ RUN apt-get update && \
     apt-get install -y \
     build-essential \
     git \
-    postgresql-server-dev-17
+    postgresql-server-dev-17 \
+    libcurl4-openssl-dev \
+    libsodium-dev \
+    libicu-dev \
+    pkg-config
 
-# Clone and install pgmq
+# Clone and install pgmq v1.5.1 (not in Supabase image, pgflow dependency)
 RUN mkdir -p /usr/share/postgresql/17/extension && \
     git clone https://github.com/tembo-io/pgmq.git /tmp/pgmq && \
     cd /tmp/pgmq/pgmq-extension && \
     git checkout v1.5.1 && \
-    # Copy extension files manually to PostgreSQL 17 extensions directory
     make && \
     cp pgmq.control /usr/share/postgresql/17/extension/ && \
-    cp sql/pgmq--*.sql /usr/share/postgresql/17/extension/
+    cp sql/pgmq--*.sql /usr/share/postgresql/17/extension/ && \
+    rm -rf /tmp/pgmq
+
+# Install pg_cron v1.6.4 (v1.6.3+ required for PG17 compatibility)
+RUN git clone https://github.com/citusdata/pg_cron.git /tmp/pg_cron && \
+    cd /tmp/pg_cron && \
+    git checkout v1.6.4 && \
+    make && \
+    make install && \
+    rm -rf /tmp/pg_cron
+
+# Install pg_net v0.20.2 (PG17 compatible, requires libcurl)
+RUN git clone https://github.com/supabase/pg_net.git /tmp/pg_net && \
+    cd /tmp/pg_net && \
+    git checkout v0.20.2 && \
+    make && \
+    make install && \
+    rm -rf /tmp/pg_net
 
-# Clean up
-RUN apt-get remove -y build-essential git postgresql-server-dev-17 && \
+# Install pgsodium v3.1.6 (requires libsodium)
+RUN git clone https://github.com/michelp/pgsodium.git /tmp/pgsodium && \
+    cd /tmp/pgsodium && \
+    git checkout v3.1.6 && \
+    make && \
+    make install && \
+    rm -rf /tmp/pgsodium
+
+# Install supabase_vault v0.2.8 (depends on pgsodium)
+RUN git clone https://github.com/supabase/vault.git /tmp/vault && \
+    cd /tmp/vault && \
+    git checkout v0.2.8 && \
+    make && \
+    make install && \
+    rm -rf /tmp/vault
+
+# Clean up build dependencies (keep runtime libs: libcurl4t64, libsodium23, libicu)
+RUN apt-get remove -y build-essential git postgresql-server-dev-17 \
+    libcurl4-openssl-dev libsodium-dev libicu-dev pkg-config && \
     apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/* /tmp/pgmq
+    apt-get update && \
+    apt-get install -y --no-install-recommends libcurl4t64 libsodium23 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Configure shared_preload_libraries for extensions that require it
+# pg_cron and pg_net need to be preloaded (pgsodium NOT needed for Atlas dev)
+RUN echo "shared_preload_libraries = 'pg_cron,pg_net'" >> /usr/share/postgresql/postgresql.conf.sample
 
 # We need to ensure we don't create any init scripts that will run on startup
 # We must also remove any existing init scripts to ensure a clean database for Atlas

pkgs/core/atlas/README.md

@@ -0,0 +1,82 @@
+# Atlas Migration Management
+
+This directory contains configuration for generating pgflow database migrations using [Atlas](https://atlasgo.io/).
+
+## Files
+
+- `atlas.hcl` - Atlas configuration
+- `Dockerfile` - Custom Postgres image with required extensions
+- `supabase-baseline-schema.sql` - Schema baseline representing a fresh hosted Supabase project
+- `fresh-extensions.txt` - Record of extensions in fresh Supabase (for reference)
+- `dump-fresh-baseline.sh` - Script to regenerate baseline
+
+## Docker Image
+
+The `Dockerfile` builds a custom Postgres 17 image with extensions matching hosted Supabase.
+
+**Reference:** [Supabase Postgres 17.6.1.054 Dockerfile](https://github.com/supabase/postgres/blob/17.6.1.054/Dockerfile-17)
+
+### Installed Extensions
+
+| Extension | Git Tag | Ext Version | Notes |
+|-----------|---------|-------------|-------|
+| pgmq | v1.5.1 | 1.5.1 | pgflow dependency (not in Supabase image) |
+| pg_cron | v1.6.4 | 1.6 | Scheduled jobs (v1.6.3+ for PG17) |
+| pg_net | v0.7.1 | 0.20.2 | HTTP requests from SQL |
+| pgsodium | v3.1.6 | 3.1.9 | Cryptography primitives |
+| supabase_vault | v0.2.8 | 0.3.1 | Secrets management (depends on pgsodium) |
+
+### Building the Image
+
+```bash
+cd pkgs/core
+./scripts/build-atlas-postgres-image
+./scripts/push-atlas-postgres-image
+```
+
+## Baseline Schema
+
+The baseline represents what a **fresh hosted Supabase project** looks like before pgflow migrations.
+
+### Pre-installed Extensions (hosted Supabase)
+
+| Extension | Status |
+|-----------|--------|
+| `supabase_vault` | Pre-installed |
+| `pgmq` | Available, not installed |
+| `pg_cron` | Available, not installed |
+| `pg_net` | Available, not installed |
+
+Our migrations must `CREATE EXTENSION` for pgmq, pg_cron, and pg_net.
+
+### Local CLI Difference
+
+The local Supabase CLI pre-installs `pg_net`, but hosted Supabase does not. The `dump-fresh-baseline.sh` script accounts for this by dropping `pg_net` before dumping to match hosted behavior.
+
+## Regenerating the Baseline
+
+Run this when:
+- Supabase CLI version changes significantly
+- Adding new Supabase-provided schemas to baseline
+
+```bash
+cd pkgs/core/atlas
+./dump-fresh-baseline.sh
+```
+
+The script will:
+1. Create a fresh Supabase project in `supabase/` (temp, gitignored)
+2. Drop `pg_net` to match hosted Supabase defaults
+3. Verify extension assumptions match hosted Supabase
+4. Dump schema and extension list
+5. Clean up temp project
+
+## Generating Migrations
+
+After updating SQL in `supabase/migrations/`, run from `pkgs/core/`:
+
+```bash
+pnpm nx atlas:migrate:diff core --name="description_of_change"
+```
+
+This compares the current schema against the baseline + existing migrations.

pkgs/core/atlas/atlas.hcl

@@ -15,8 +15,10 @@ docker "postgres" "pgflow" {
   # image = "postgres:17"
   # custom image is built and pushed to speed up schema verification,
   # otherwise it takes around 30s
-  image = "jumski/postgres-17-pgmq:latest"
-  baseline = file(".supabase-baseline-schema.sql")
+  # contains: pgmq, pg_cron, pg_net, pgsodium, supabase_vault
+  # tag matches Supabase Postgres version we align extensions with
+  image = "jumski/atlas-postgres-pgflow:17.6.1.054"
+  baseline = file("supabase-baseline-schema.sql")
   build {
     dockerfile = "atlas/Dockerfile"
     context = "."

pkgs/core/atlas/dump-fresh-baseline.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+# Dump baseline schema from a fresh Supabase instance
+#
+# This script:
+# 1. Creates a fresh Supabase project (no pgflow)
+# 2. Verifies extension availability matches our assumptions
+# 3. Dumps the schema as our baseline
+# 4. Cleans up
+#
+# Run this when:
+# - Supabase CLI version changes
+# - Adding new Supabase-provided schemas to baseline
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# Use npx to run supabase directly in this directory (not via pnpm which runs from pkgs/core)
+SUPA="npx supabase"
+
+echo "=== Cleaning up any previous temp project ==="
+rm -rf supabase/ || true
+
+echo "=== Initializing fresh Supabase project ==="
+$SUPA init --force
+
+echo "=== Stopping any running Supabase instances ==="
+$SUPA stop --all || true
+
+echo "=== Starting fresh Supabase ==="
+$SUPA start
+
+echo "=== Getting database URL ==="
+DB_URL=$($SUPA status --output json | jq -r '.DB_URL')
+echo "DB_URL: $DB_URL"
+
+echo "=== Matching hosted Supabase defaults ==="
+# Local CLI pre-installs pg_net, but hosted Supabase doesn't
+# Drop it to match hosted behavior (our migrations will create it)
+echo "Dropping pg_net to match hosted Supabase defaults..."
+psql "$DB_URL" -c "DROP EXTENSION IF EXISTS pg_net CASCADE;" 2>/dev/null || true
+
+echo "=== Verifying extension assumptions ==="
+# Query extension status (using --no-psqlrc to avoid extra output)
+EXTENSION_STATUS=$(PAGER='' psql "$DB_URL" --no-psqlrc -t -A -F'|' -c "
+SELECT
+  name,
+  CASE WHEN installed_version IS NOT NULL THEN 'installed' ELSE 'available' END as status
+FROM pg_available_extensions
+WHERE name IN ('pgmq', 'pg_cron', 'pg_net', 'supabase_vault')
+ORDER BY name;
+")
+
+echo "Extension status:"
+echo "$EXTENSION_STATUS"
+
+# Expected state matching HOSTED Supabase (not local CLI)
+# - supabase_vault: pre-installed
+# - pgmq, pg_cron, pg_net: available but NOT installed
+EXPECTED="pg_cron|available
+pg_net|available
+pgmq|available
+supabase_vault|installed"
+
+if [ "$EXTENSION_STATUS" != "$EXPECTED" ]; then
+  echo ""
+  echo "ERROR: Extension status does not match hosted Supabase assumptions!"
+  echo ""
+  echo "Expected (matching hosted Supabase):"
+  echo "$EXPECTED"
+  echo ""
+  echo "Got:"
+  echo "$EXTENSION_STATUS"
+  echo ""
+  echo "If hosted Supabase changed defaults, update this script's expectations."
+  $SUPA stop
+  rm -rf supabase/
+  exit 1
+fi
+
+echo ""
+echo "Extension assumptions verified (matches hosted Supabase)!"
+
+echo "=== Dumping full \\dx output ==="
+psql "$DB_URL" -c '\dx' > fresh-extensions.txt
+echo "Saved to fresh-extensions.txt"
+
+echo "=== Dumping baseline schema ==="
+# Only dump schemas we need:
+# - realtime: we reference realtime.send()
+# - vault: supabase_vault is pre-installed here
+# Skip extensions schema - it contains Supabase internal event triggers
+# that reference roles not available in Atlas dev container
+atlas schema inspect \
+  --schema realtime,vault \
+  -u "$DB_URL?sslmode=disable" \
+  --format "{{ sql . }}" > supabase-baseline-schema.sql
+
+# Strip VERSION strings (they change between Supabase versions)
+sed -i 's/ VERSION "[^"]*"//g' supabase-baseline-schema.sql
+
+# Strip date-specific partitions (they change daily, we don't reference them)
+# e.g., messages_2025_12_04, messages_2025_12_05, etc.
+sed -i '/messages_20[0-9]\{2\}_[0-9]\{2\}_[0-9]\{2\}/d' supabase-baseline-schema.sql
+
+echo "Saved to supabase-baseline-schema.sql"
+
+echo "=== Stopping Supabase ==="
+$SUPA stop
+
+echo "=== Cleaning up temp project ==="
+rm -rf supabase/
+
+echo ""
+echo "=== Done! ==="
+echo "Baseline regenerated from fresh Supabase."
+echo "Commit the updated supabase-baseline-schema.sql and fresh-extensions.txt"

pkgs/core/atlas/fresh-extensions.txt

@@ -0,0 +1,13 @@
+Null display is "[NULL]".
+Expanded display is used automatically.
+                                            List of installed extensions
+        Name        | Version |   Schema   |                              Description                               
+--------------------+---------+------------+------------------------------------------------------------------------
+ pg_graphql         | 1.5.11  | graphql    | pg_graphql: GraphQL support
+ pg_stat_statements | 1.11    | extensions | track planning and execution statistics of all SQL statements executed
+ pgcrypto           | 1.3     | extensions | cryptographic functions
+ plpgsql            | 1.0     | pg_catalog | PL/pgSQL procedural language
+ supabase_vault     | 0.3.1   | vault      | Supabase Vault Extension
+ uuid-ossp          | 1.1     | extensions | generate universally unique identifiers (UUIDs)
+(6 rows)
+