From 211fd8456975dbb1401828ce3364a4a8bb90c06c Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Sun, 2 Nov 2025 18:45:36 +0100 Subject: [PATCH] chore: update PostgreSQL version references to 17 across configs, Dockerfiles, and scripts - Changed major_version from 15 to 17 in multiple config files for various environments - Updated Dockerfile to use postgres:17 and related build scripts for new image - Modified Atlas setup to rely on custom postgres:17-pgmq image - Adjusted database type definitions for compatibility with PostgreSQL 17 - Updated compose files to reference the new PostgreSQL 17 image - Added note in documentation about PostgreSQL 17 support and migration guidance --- .../upgrade-postgres-seventeen-supabase.md | 8 + PLAN.md | 533 ++++++++++++++++++ pkgs/client/supabase/config.toml | 2 +- pkgs/core/ATLAS.md | 2 +- pkgs/core/atlas/Dockerfile | 18 +- pkgs/core/atlas/atlas.hcl | 4 +- pkgs/core/package.json | 2 +- pkgs/core/scripts/build-atlas-postgres-image | 2 +- pkgs/core/scripts/push-atlas-postgres-image | 2 +- pkgs/core/src/database-types.ts | 48 +- pkgs/core/supabase/.temp/pgmeta-version | 1 - pkgs/core/supabase/.temp/postgres-version | 1 - pkgs/core/supabase/config.toml | 2 +- pkgs/edge-worker/supabase/config.toml | 2 +- pkgs/edge-worker/tests/db/compose.yaml | 6 +- .../src/content/docs/deploy/update-pgflow.mdx | 12 + pnpm-lock.yaml | 4 +- 17 files changed, 600 insertions(+), 49 deletions(-) create mode 100644 .changeset/upgrade-postgres-seventeen-supabase.md create mode 100644 PLAN.md delete mode 100644 pkgs/core/supabase/.temp/pgmeta-version delete mode 100644 pkgs/core/supabase/.temp/postgres-version diff --git a/.changeset/upgrade-postgres-seventeen-supabase.md b/.changeset/upgrade-postgres-seventeen-supabase.md new file mode 100644 index 000000000..6e48db95a --- /dev/null +++ b/.changeset/upgrade-postgres-seventeen-supabase.md @@ -0,0 +1,8 @@ +--- +'@pgflow/core': minor +'@pgflow/client': minor +'@pgflow/edge-worker': minor +'@pgflow/website': minor +--- + +Upgrade PostgreSQL 17 and Supabase CLI 2.54.11. Removes strict version pins for Docker images and PGMQ extension to use latest compatible versions. diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 000000000..359a0ea66 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,533 @@ +# PGMQ 1.5.1 Upgrade Plan + +This document tracks the work needed to fully upgrade to PGMQ 1.5.1 and remove compatibility workarounds that are no longer needed. + +## Context + +### Current State +- **PGMQ Version**: 1.5.1 (confirmed via psql) +- **Supabase CLI**: v2.54.11 +- **Postgres**: 17 +- **Version Pin Removed**: Commit a71b371 removed PGMQ version constraint + +### Why This Upgrade? + +**PGMQ 1.5.0 Changes:** +- Added `headers` column to `message_record` type (now 6 columns: msg_id, read_ct, enqueued_at, vt, message, headers) +- Improved `read_with_poll` function with bug fixes + +**Supabase Realtime v2.35.4+ Changes:** +- Fixed issue #1369: Janitor now creates partitions immediately on startup +- Auto-creates 7 partitions (3 days back, today, 3 days forward) +- **Verified**: After `db reset`, partitions are created automatically + +### Breaking Changes +- Functions returning `SETOF pgmq.message_record` must return 6 columns (was 5) +- The `pgflow.read_with_poll` backport is obsolete +- The `pgflow_tests.create_realtime_partition` helper is obsolete + +--- + +## [ ] Task 1: Fix set_vt_batch Function + +**Priority**: HIGH (blocking 3 test files) + +### Why +The function returns `SETOF pgmq.message_record` but only returns 5 columns. PGMQ 1.5.0+ expects 6 columns (added `headers`). + +### Current Error +``` +ERROR: structure of query does not match function result type +DETAIL: Number of returned columns (5) does not match expected column count (6). +``` + +### What to Change + +**File**: `pkgs/core/schemas/0110_function_set_vt_batch.sql` + +**Line 49-53**, change: +```sql +RETURNING q.msg_id, + q.read_ct, + q.enqueued_at, + q.vt, + q.message +``` + +To: +```sql +RETURNING q.msg_id, + q.read_ct, + q.enqueued_at, + q.vt, + q.message, + q.headers +``` + +### Affected Tests +- `supabase/tests/set_vt_batch/basic_batch_update.test.sql` +- `supabase/tests/set_vt_batch/queue_security.test.sql` +- `supabase/tests/set_vt_batch/return_format.test.sql` + +### Verification +```bash +./scripts/run-test-with-colors supabase/tests/set_vt_batch/*.test.sql +``` + +--- + +## [ ] Task 2: Remove read_with_poll Backport + +**Priority**: MEDIUM (cleanup, not blocking) + +### Why +The `pgflow.read_with_poll` function was a backport from PGMQ 1.5.0 with headers removed for 1.4.4 compatibility. The comment in the file says: + +> "This is a backport of the pgmq.read_with_poll function from version 1.5.0. It is required because it fixes a bug with high CPU usage and Supabase is still using version 1.4.4. It is slightly modified (removed headers which are not available in 1.4.4). **This will be removed once Supabase upgrades to 1.5.0 or higher.**" + +We're now on 1.5.1, so this should be removed. + +### What to Delete + +**File**: `pkgs/core/schemas/0080_function_read_with_poll.sql` (entire file) + +### Migration Impact +**IMPORTANT**: Do NOT edit migration files manually. Existing migrations that reference this function will remain unchanged (they're historical). + +The deletion of `schemas/0080_function_read_with_poll.sql` will cause Atlas to generate a `DROP FUNCTION pgflow.read_with_poll` statement in the new migration (Task 5). + +Historical migrations that created this function: +- `supabase/migrations/20250429164909_pgflow_initial.sql` (CREATE FUNCTION statement) +- `supabase/migrations/20250517072017_pgflow_fix_poll_for_tasks_to_use_separate_statement_for_polling.sql` (DROP + CREATE) +- `supabase/migrations/20250627090700_pgflow_fix_function_search_paths.sql` (DROP + CREATE) + +These remain untouched - migrations are append-only. + +--- + +## [ ] Task 3: Update Code to Use pgmq.read_with_poll + +**Priority**: MEDIUM (required after Task 2) + +### Why +After removing `pgflow.read_with_poll`, code needs to call PGMQ's native `pgmq.read_with_poll()` instead. + +### Affected Packages +- `pkgs/core` - SQL client + tests +- `pkgs/edge-worker` - Queue implementation + +### What to Change + +#### 3.1 Core Package - TypeScript Client + +**File**: `pkgs/core/src/PgflowSqlClient.ts` (line 29) + +Change: +```typescript +FROM pgflow.read_with_poll( +``` + +To: +```typescript +FROM pgmq.read_with_poll( +``` + +#### 3.2 Core Package - Test Files + +The following test files call `pgflow.read_with_poll` and need updating: + +**Location**: `pkgs/core/supabase/tests/start_tasks/` +``` +builds_proper_input_from_deps_outputs.test.sql +multiple_task_processing.test.sql +returns_task_index.test.sql +started_at_timestamps.test.sql +status_transitions.test.sql +task_index_returned_correctly.test.sql +worker_tracking.test.sql +basic_start_tasks.test.sql +``` + +Bulk update command: +```bash +cd pkgs/core +rg -l "pgflow\.read_with_poll" supabase/tests/ | xargs sed -i 's/pgflow\.read_with_poll/pgmq.read_with_poll/g' +``` + +#### 3.3 Edge Worker Package - Queue Class + +**File**: `pkgs/edge-worker/src/queue/Queue.ts` (line 82) + +Change: +```typescript +return await this.sql[]>` + SELECT * + FROM pgflow.read_with_poll( + queue_name => ${this.queueName}, + vt => ${visibilityTimeout}, + qty => ${batchSize}, + max_poll_seconds => ${maxPollSeconds}, + poll_interval_ms => ${pollIntervalMs} + ); +`; +``` + +To: +```typescript +return await this.sql[]>` + SELECT * + FROM pgmq.read_with_poll( + queue_name => ${this.queueName}, + vt => ${visibilityTimeout}, + qty => ${batchSize}, + max_poll_seconds => ${maxPollSeconds}, + poll_interval_ms => ${pollIntervalMs} + ); +`; +``` + +### Verification + +#### Verify Core Package +```bash +cd pkgs/core +./scripts/run-test-with-colors supabase/tests/start_tasks/*.test.sql +``` + +#### Verify Edge Worker Package +```bash +cd pkgs/edge-worker +pnpm nx test:unit edge-worker +pnpm nx test:integration edge-worker +``` + +--- + +## [ ] Task 4: Remove Realtime Partition Helpers + +**Priority**: LOW (cleanup, not blocking) + +### Why +Supabase Realtime v2.35.4+ fixed issue #1369. The janitor now creates partitions immediately on startup. **Verified**: After `pnpm supabase db reset`, 7 partitions are created automatically (3 days back, today, 3 days forward). + +The helper function was a workaround that is no longer needed. + +### What to Delete + +#### 4.1 Helper Function + +**File**: `supabase/seed.sql` (lines ~540-580) + +Delete the entire `pgflow_tests.create_realtime_partition()` function definition. + +#### 4.2 Helper Test File + +**File**: `supabase/tests/pgflow_tests/create_realtime_partition.test.sql` (entire file) + +This test fails with: +``` +ERROR: must be owner of table messages_2025_11_02 +``` + +The test tries to DROP partitions owned by `supabase_admin`, which causes permission errors. Since the helper is obsolete, delete the test. + +#### 4.3 Remove Helper Calls from Tests + +The following test files call `pgflow_tests.create_realtime_partition()` at the start: + +``` +supabase/tests/regressions/step_failed_event_bug.test.sql (line 5) +supabase/tests/realtime/start_ready_steps_events.test.sql (line 7) +supabase/tests/realtime/complete_task_events.test.sql (line 5) +supabase/tests/realtime/start_flow_events.test.sql (line 5) +supabase/tests/realtime/maybe_complete_run_events.test.sql (line 5) +supabase/tests/realtime/full_flow_events.test.sql (line 5) +supabase/tests/realtime/fail_task_events.test.sql (line 5) +supabase/tests/map_output_aggregation/broadcast_event_fixed.test.sql (line 11) +``` + +Remove the `select pgflow_tests.create_realtime_partition();` lines from all these files. + +### Verification + +#### Verify partitions auto-create: +```bash +pnpm supabase db reset +PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -d postgres -c \ + "SELECT COUNT(*) as partition_count FROM pg_tables WHERE schemaname = 'realtime' AND tablename LIKE 'messages_%'" +``` + +Expected: ~7 partitions + +#### Verify tests still pass: +```bash +./scripts/run-test-with-colors supabase/tests/realtime/*.test.sql +./scripts/run-test-with-colors supabase/tests/map_output_aggregation/broadcast_event_fixed.test.sql +./scripts/run-test-with-colors supabase/tests/regressions/step_failed_event_bug.test.sql +``` + +--- + +## [ ] Task 5: Generate Migration from Schema Changes + +**Priority**: HIGH (required after schema changes) + +### Why +Tasks 1 and 2 modify schema files in `pkgs/core/schemas/`. We need to generate a new migration from these schema changes. **NEVER manually edit migrations** - always generate them from schema files. + +### Schema Files Modified +- `schemas/0110_function_set_vt_batch.sql` - Added `headers` column to RETURNING +- `schemas/0080_function_read_with_poll.sql` - Deleted (entire file) + +### Migration Generation Process + +Follow the workflow from `.claude/reference/schema_development.md`: + +```bash +cd pkgs/core + +# Generate migration from schema changes +./scripts/atlas-migrate-diff upgrade_pgmq_1_5_1 # NO pgflow_ prefix (auto-added) + +# This creates: supabase/migrations/TIMESTAMP_pgflow_upgrade_pgmq_1_5_1.sql + +# Verify migration is valid +pnpm nx verify-migrations core + +# Test that migration applies cleanly +pnpm nx supabase:reset core +pnpm nx test:pgtap core +``` + +### Expected Migration Contents +The generated migration should contain: +- `DROP FUNCTION pgflow.read_with_poll(...)` - Removing backport +- Updated `CREATE FUNCTION pgflow.set_vt_batch(...)` - With `headers` in RETURNING clause +- Updated `atlas.sum` checksums + +### Verification +```bash +# Clean slate test +pnpm nx supabase:reset core + +# All tests should pass +pnpm nx test:pgtap core + +# Specifically verify the previously failing tests +./scripts/run-test-with-colors supabase/tests/set_vt_batch/*.test.sql +``` + +All tests should pass after migrations are applied. + +### Migration Sync to Other Packages + +After generating the migration in core, it needs to be synced to other packages: + +#### Auto-sync (via Nx tasks) +These packages automatically copy migrations from core when their Supabase tasks run: + +**pkgs/client** +- Syncs via: `nx run client:supabase:prepare` +- Copies: `../core/supabase/migrations/*.sql` → `supabase/migrations/` +- Runs before: Supabase database operations + +**pkgs/edge-worker** +- Syncs via: `nx run edge-worker:supabase:reset` +- Copies: `../core/supabase/migrations/*.sql` → `supabase/migrations/` +- Runs during: `pnpm nx supabase:reset edge-worker` + +**No action needed** - these will pick up the new migration automatically on next run. + +#### Manual sync required + +**examples/playground** +- Has diverged migrations (not identical to core) +- No auto-sync mechanism +- Must manually copy the new migration file + +```bash +# After generating migration in core, copy to playground +cp pkgs/core/supabase/migrations/*_pgflow_upgrade_pgmq_1_5_1.sql \ + examples/playground/supabase/migrations/ +``` + +### Important Notes +- Migration name: `upgrade_pgmq_1_5_1` (Atlas auto-adds `pgflow_` prefix) +- Do NOT use `TEMP_` prefix (this is final migration for merging to main) +- Commit both schema changes AND generated migration together +- Remember to copy migration to playground manually + +--- + +## [ ] Task 6: Update Documentation + +**Priority**: LOW (good practice) + +### What to Update + +#### 6.1 CHANGELOG.md +Add entry under `[Unreleased]`: +```markdown +### Changed +- Upgraded to PGMQ 1.5.1, adding support for message headers +- Removed `pgflow.read_with_poll` backport (use `pgmq.read_with_poll` instead) +- Removed `pgflow_tests.create_realtime_partition` helper (partitions auto-created by Realtime v2.35.4+) + +### Breaking Changes +- Requires PGMQ 1.5.0 or higher (Supabase ships 1.5.1 by default) +- Code calling `pgflow.read_with_poll` must update to `pgmq.read_with_poll` +``` + +#### 6.2 Documentation Files + +Update references from `pgflow.read_with_poll` to `pgmq.read_with_poll` in: + +**Website docs**: +- `pkgs/website/src/content/docs/reference/queue-worker/configuration.mdx` + +**Architecture diagrams**: +- `pkgs/website/src/assets/architecture-diagrams/task-execution.mermaid` +- `pkgs/core/assets/flow-lifecycle.mermaid` +- `pkgs/core/assets/flow-lifecycle.svg` + +**Guides**: +- `ARCHITECTURE_GUIDE.md` + +**Package READMEs**: +- `pkgs/core/README.md` +- `pkgs/core/CHANGELOG.md` +- `pkgs/edge-worker/CHANGELOG.md` +- `pkgs/website/CHANGELOG.md` + +Use global search to find all references: +```bash +rg "pgflow\.read_with_poll" --type md --type mermaid +``` + +#### 6.3 Migration Guide (if needed) +If this affects external users, add a migration guide to the website docs explaining: +- How to update from PGMQ 1.4.4 to 1.5.1 +- What code changes are needed +- Why the breaking changes were made + +--- + +## [ ] Task 7: Final Verification + +**Priority**: CRITICAL (before merging) + +### Full Test Suite +```bash +cd pkgs/core +pnpm nx test core +``` + +This runs: +- All pgTAP tests +- All Vitest tests +- Type checks + +### Specific Regression Tests + +#### 7.1 set_vt_batch +```bash +./scripts/run-test-with-colors supabase/tests/set_vt_batch/*.test.sql +``` +Expected: All pass (previously failing) + +#### 7.2 start_tasks (uses read_with_poll) +```bash +./scripts/run-test-with-colors supabase/tests/start_tasks/*.test.sql +``` +Expected: All pass + +#### 7.3 Realtime (no longer needs partition helper) +```bash +./scripts/run-test-with-colors supabase/tests/realtime/*.test.sql +``` +Expected: All pass + +### Edge Cases + +#### Fresh Install Test +```bash +pnpm supabase db reset +# Immediately test realtime.send without waiting +PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -d postgres -c \ + "SELECT realtime.send(jsonb_build_object('test', 'data'), 'test', 'test', false)" +# Check message was stored +PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -d postgres -c \ + "SELECT COUNT(*) FROM realtime.messages WHERE payload->>'test' = 'data'" +``` +Expected: Message successfully stored (partition exists) + +--- + +## Summary Checklist + +Before creating PR: +- [ ] All schema changes completed (Tasks 1-2) +- [ ] All code updates completed (Task 3) +- [ ] All cleanup completed (Task 4) +- [ ] Migrations regenerated (Task 5) +- [ ] Documentation updated (Task 6) +- [ ] All tests passing (Task 7) +- [ ] No references to `pgflow.read_with_poll` remain +- [ ] No references to `create_realtime_partition` remain +- [ ] Fresh `db reset` creates partitions automatically + +## Files Changed Summary + +### pkgs/core + +**Modified**: +- `schemas/0110_function_set_vt_batch.sql` - Add headers column to RETURNING +- `src/PgflowSqlClient.ts` - Change pgflow.read_with_poll → pgmq.read_with_poll +- `supabase/tests/start_tasks/*.test.sql` (8 files) - Change pgflow.read_with_poll → pgmq.read_with_poll +- `supabase/tests/realtime/*.test.sql` (5 files) - Remove partition helper calls +- `supabase/tests/map_output_aggregation/broadcast_event_fixed.test.sql` - Remove partition helper call +- `supabase/tests/regressions/step_failed_event_bug.test.sql` - Remove partition helper call +- `CHANGELOG.md` - Document changes +- `README.md` - Update references +- `assets/flow-lifecycle.mermaid` - Update diagram +- `assets/flow-lifecycle.svg` - Update diagram + +**Deleted**: +- `schemas/0080_function_read_with_poll.sql` - Backport no longer needed +- `supabase/tests/pgflow_tests/create_realtime_partition.test.sql` - Helper test obsolete +- `supabase/seed.sql` - Remove `create_realtime_partition` function (~40 lines) + +**Generated**: +- New migration file via Atlas: `supabase/migrations/*_pgflow_upgrade_pgmq_1_5_1.sql` +- Updated `supabase/migrations/atlas.sum` + +### pkgs/edge-worker + +**Modified**: +- `src/queue/Queue.ts` - Change pgflow.read_with_poll → pgmq.read_with_poll (line 82) +- `CHANGELOG.md` - Document changes + +**Auto-synced** (via Nx tasks): +- `supabase/migrations/*` - Auto-copied from core on `nx supabase:reset` + +### pkgs/client + +**Auto-synced** (via Nx tasks): +- `supabase/migrations/*` - Auto-copied from core on `nx supabase:prepare` + +### pkgs/website + +**Modified**: +- `src/content/docs/reference/queue-worker/configuration.mdx` - Update references +- `src/assets/architecture-diagrams/task-execution.mermaid` - Update diagram +- `CHANGELOG.md` - Document changes + +### examples/playground + +**Manual sync required**: +- `supabase/migrations/*_pgflow_upgrade_pgmq_1_5_1.sql` - Copy from core manually + +### Root + +**Modified**: +- `ARCHITECTURE_GUIDE.md` - Update references diff --git a/pkgs/client/supabase/config.toml b/pkgs/client/supabase/config.toml index 19359c0e9..5491e6544 100644 --- a/pkgs/client/supabase/config.toml +++ b/pkgs/client/supabase/config.toml @@ -8,7 +8,7 @@ schemas = ["public", "graphql_public", "pgflow"] [db] port = 50522 # distinct from core:50422 and edge-worker:50322 shadow_port = 50520 -major_version = 15 +major_version = 17 [db.pooler] enabled = true diff --git a/pkgs/core/ATLAS.md b/pkgs/core/ATLAS.md index 8f8e3d651..ecd6690bd 100644 --- a/pkgs/core/ATLAS.md +++ b/pkgs/core/ATLAS.md @@ -16,7 +16,7 @@ The database must be empty, but contain everything needed for the schemas to app We need a configured [PGMQ](https://github.com/tembo-io/pgmq) extension, which Atlas does not support in their dev images. -That's why this setup relies on a custom built image `jumski/postgres-15-pgmq:latest`. +That's why this setup relies on a custom built image `jumski/postgres-17-pgmq:latest`. Inspect `Dockerfile.atlas` to see how it is built. diff --git a/pkgs/core/atlas/Dockerfile b/pkgs/core/atlas/Dockerfile index d6ad9b170..7ab45c41c 100644 --- a/pkgs/core/atlas/Dockerfile +++ b/pkgs/core/atlas/Dockerfile @@ -1,5 +1,5 @@ -# Use official PostgreSQL 15 -FROM postgres:15 +# Use official PostgreSQL 17 +FROM postgres:17 # Set environment variables for postgres ENV POSTGRES_PASSWORD=postgres @@ -10,20 +10,20 @@ RUN apt-get update && \ apt-get install -y \ build-essential \ git \ - postgresql-server-dev-15 + postgresql-server-dev-17 # Clone and install pgmq -RUN mkdir -p /usr/share/postgresql/15/extension && \ +RUN mkdir -p /usr/share/postgresql/17/extension && \ git clone https://github.com/tembo-io/pgmq.git /tmp/pgmq && \ cd /tmp/pgmq/pgmq-extension && \ - git checkout v1.4.4 && \ - # Copy extension files manually to PostgreSQL 15 extensions directory + git checkout v1.5.1 && \ + # Copy extension files manually to PostgreSQL 17 extensions directory make && \ - cp pgmq.control /usr/share/postgresql/15/extension/ && \ - cp sql/pgmq--*.sql /usr/share/postgresql/15/extension/ + cp pgmq.control /usr/share/postgresql/17/extension/ && \ + cp sql/pgmq--*.sql /usr/share/postgresql/17/extension/ # Clean up -RUN apt-get remove -y build-essential git postgresql-server-dev-15 && \ +RUN apt-get remove -y build-essential git postgresql-server-dev-17 && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* /tmp/pgmq diff --git a/pkgs/core/atlas/atlas.hcl b/pkgs/core/atlas/atlas.hcl index 5a20124d1..5b0e0e33a 100644 --- a/pkgs/core/atlas/atlas.hcl +++ b/pkgs/core/atlas/atlas.hcl @@ -12,10 +12,10 @@ env "local" { } docker "postgres" "pgflow" { - # image = "postgres:15" + # image = "postgres:17" # custom image is built and pushed to speed up schema verification, # otherwise it takes around 30s - image = "jumski/postgres-15-pgmq:latest" + image = "jumski/postgres-17-pgmq:latest" baseline = file(".supabase-baseline-schema.sql") build { dockerfile = "atlas/Dockerfile" diff --git a/pkgs/core/package.json b/pkgs/core/package.json index b77dd5d41..a62eb8432 100644 --- a/pkgs/core/package.json +++ b/pkgs/core/package.json @@ -20,7 +20,7 @@ }, "devDependencies": { "@types/node": "^22.14.1", - "supabase": "2.21.1" + "supabase": "2.54.11" }, "dependencies": { "@pgflow/dsl": "workspace:*", diff --git a/pkgs/core/scripts/build-atlas-postgres-image b/pkgs/core/scripts/build-atlas-postgres-image index 9e24d0251..0fdee36a4 100755 --- a/pkgs/core/scripts/build-atlas-postgres-image +++ b/pkgs/core/scripts/build-atlas-postgres-image @@ -1,2 +1,2 @@ #!/bin/bash -docker build -t jumski/postgres-15-pgmq:latest --file atlas/Dockerfile atlas/ +docker build -t jumski/postgres-17-pgmq:latest --file atlas/Dockerfile atlas/ diff --git a/pkgs/core/scripts/push-atlas-postgres-image b/pkgs/core/scripts/push-atlas-postgres-image index b3be2cdf2..56576da6b 100755 --- a/pkgs/core/scripts/push-atlas-postgres-image +++ b/pkgs/core/scripts/push-atlas-postgres-image @@ -1,2 +1,2 @@ #!/bin/bash -docker push jumski/postgres-15-pgmq:latest +docker push jumski/postgres-17-pgmq:latest diff --git a/pkgs/core/src/database-types.ts b/pkgs/core/src/database-types.ts index d5a7c13ef..179139ddb 100644 --- a/pkgs/core/src/database-types.ts +++ b/pkgs/core/src/database-types.ts @@ -348,6 +348,7 @@ export type Database = { Functions: { add_step: { Args: { + step_type?: string flow_slug: string step_slug: string deps_slugs?: string[] @@ -355,7 +356,6 @@ export type Database = { base_delay?: number timeout?: number start_delay?: number - step_type?: string } Returns: { created_at: string @@ -371,7 +371,7 @@ export type Database = { } } calculate_retry_delay: { - Args: { base_delay: number; attempts_count: number } + Args: { attempts_count: number; base_delay: number } Returns: number } cascade_complete_taskless_steps: { @@ -404,10 +404,10 @@ export type Database = { } create_flow: { Args: { + timeout?: number flow_slug: string max_attempts?: number base_delay?: number - timeout?: number } Returns: { created_at: string @@ -455,31 +455,31 @@ export type Database = { } poll_for_tasks: { Args: { - queue_name: string vt: number - qty: number - max_poll_seconds?: number poll_interval_ms?: number + max_poll_seconds?: number + qty: number + queue_name: string } Returns: Database["pgflow"]["CompositeTypes"]["step_task_record"][] } read_with_poll: { Args: { + qty: number queue_name: string vt: number - qty: number + conditional?: Json max_poll_seconds?: number poll_interval_ms?: number - conditional?: Json } Returns: Database["pgmq"]["CompositeTypes"]["message_record"][] } set_vt_batch: { - Args: { queue_name: string; msg_ids: number[]; vt_offsets: number[] } + Args: { vt_offsets: number[]; queue_name: string; msg_ids: number[] } Returns: Database["pgmq"]["CompositeTypes"]["message_record"][] } start_flow: { - Args: { flow_slug: string; input: Json; run_id?: string } + Args: { flow_slug: string; run_id?: string; input: Json } Returns: { completed_at: string | null failed_at: string | null @@ -493,7 +493,7 @@ export type Database = { }[] } start_flow_with_states: { - Args: { flow_slug: string; input: Json; run_id?: string } + Args: { flow_slug: string; run_id?: string; input: Json } Returns: Json } start_ready_steps: { @@ -501,7 +501,7 @@ export type Database = { Returns: undefined } start_tasks: { - Args: { flow_slug: string; msg_ids: number[]; worker_id: string } + Args: { worker_id: string; flow_slug: string; msg_ids: number[] } Returns: Database["pgflow"]["CompositeTypes"]["step_task_record"][] } } @@ -569,16 +569,16 @@ export type Database = { } archive: { Args: - | { queue_name: string; msg_id: number } - | { queue_name: string; msg_ids: number[] } + | { msg_id: number; queue_name: string } + | { msg_ids: number[]; queue_name: string } Returns: boolean } convert_archive_partitioned: { Args: { table_name: string - partition_interval?: string retention_interval?: string leading_partition?: number + partition_interval?: string } Returns: undefined } @@ -592,9 +592,9 @@ export type Database = { } create_partitioned: { Args: { + retention_interval?: string queue_name: string partition_interval?: string - retention_interval?: string } Returns: undefined } @@ -604,7 +604,7 @@ export type Database = { } delete: { Args: - | { queue_name: string; msg_id: number } + | { msg_id: number; queue_name: string } | { queue_name: string; msg_ids: number[] } Returns: boolean } @@ -617,7 +617,7 @@ export type Database = { Returns: boolean } format_table_name: { - Args: { queue_name: string; prefix: string } + Args: { prefix: string; queue_name: string } Returns: string } list_queues: { @@ -641,29 +641,29 @@ export type Database = { Returns: number } read: { - Args: { queue_name: string; vt: number; qty: number } + Args: { queue_name: string; qty: number; vt: number } Returns: Database["pgmq"]["CompositeTypes"]["message_record"][] } read_with_poll: { Args: { - queue_name: string - vt: number qty: number max_poll_seconds?: number poll_interval_ms?: number + queue_name: string + vt: number } Returns: Database["pgmq"]["CompositeTypes"]["message_record"][] } send: { - Args: { queue_name: string; msg: Json; delay?: number } + Args: { delay?: number; msg: Json; queue_name: string } Returns: number[] } send_batch: { - Args: { queue_name: string; msgs: Json[]; delay?: number } + Args: { delay?: number; queue_name: string; msgs: Json[] } Returns: number[] } set_vt: { - Args: { queue_name: string; msg_id: number; vt: number } + Args: { queue_name: string; vt: number; msg_id: number } Returns: Database["pgmq"]["CompositeTypes"]["message_record"][] } validate_queue_name: { diff --git a/pkgs/core/supabase/.temp/pgmeta-version b/pkgs/core/supabase/.temp/pgmeta-version deleted file mode 100644 index 6097d5224..000000000 --- a/pkgs/core/supabase/.temp/pgmeta-version +++ /dev/null @@ -1 +0,0 @@ -v0.88.2 \ No newline at end of file diff --git a/pkgs/core/supabase/.temp/postgres-version b/pkgs/core/supabase/.temp/postgres-version deleted file mode 100644 index 243b1c31c..000000000 --- a/pkgs/core/supabase/.temp/postgres-version +++ /dev/null @@ -1 +0,0 @@ -15.8.1.069 \ No newline at end of file diff --git a/pkgs/core/supabase/config.toml b/pkgs/core/supabase/config.toml index 8c2180fd6..05ded3e42 100644 --- a/pkgs/core/supabase/config.toml +++ b/pkgs/core/supabase/config.toml @@ -3,7 +3,7 @@ project_id = "core" [db] port = 50422 shadow_port = 50420 -major_version = 15 +major_version = 17 # disable unused features [db.seed] diff --git a/pkgs/edge-worker/supabase/config.toml b/pkgs/edge-worker/supabase/config.toml index c850dd51b..4d93832d3 100644 --- a/pkgs/edge-worker/supabase/config.toml +++ b/pkgs/edge-worker/supabase/config.toml @@ -7,7 +7,7 @@ port = 50321 [db] port = 50322 shadow_port = 50320 -major_version = 15 +major_version = 17 [db.pooler] enabled = true diff --git a/pkgs/edge-worker/tests/db/compose.yaml b/pkgs/edge-worker/tests/db/compose.yaml index 799f0ef44..dd1647c4e 100644 --- a/pkgs/edge-worker/tests/db/compose.yaml +++ b/pkgs/edge-worker/tests/db/compose.yaml @@ -1,8 +1,8 @@ services: db: - image: jumski/postgres-15-pgmq:latest - # image: postgres:15.8-alpine - # image: supabase/postgres:15.8.1.020 + image: jumski/postgres-17-pgmq:latest + # image: postgres:17-alpine + # image: supabase/postgres:17.x ports: - '5432:5432' volumes: diff --git a/pkgs/website/src/content/docs/deploy/update-pgflow.mdx b/pkgs/website/src/content/docs/deploy/update-pgflow.mdx index ad69c019d..f68a0eef8 100644 --- a/pkgs/website/src/content/docs/deploy/update-pgflow.mdx +++ b/pkgs/website/src/content/docs/deploy/update-pgflow.mdx @@ -85,6 +85,18 @@ npx supabase stop npx supabase start ``` +### PostgreSQL 17 Compatibility + + + ### 5. Apply new migrations Apply the new migrations to your database: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f8d7a0b8..a9a11caef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -299,8 +299,8 @@ importers: specifier: ^22.14.1 version: 22.19.0 supabase: - specifier: 2.21.1 - version: 2.21.1 + specifier: 2.54.11 + version: 2.54.11 pkgs/dsl: devDependencies: