From 320c9ce25b93a5039005f3282adf7d3fe224d85e Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Wed, 10 Dec 2025 17:31:49 +0100 Subject: [PATCH] split compilation to startup and manual compilation docs --- pkgs/website/astro.config.mjs | 8 +- pkgs/website/redirects.config.mjs | 2 + .../content/docs/build/local-development.mdx | 4 +- .../src/content/docs/concepts/compilation.mdx | 202 ------------------ .../src/content/docs/concepts/index.mdx | 6 +- .../docs/concepts/manual-compilation.mdx | 116 ++++++++++ .../docs/concepts/startup-compilation.mdx | 118 ++++++++++ .../docs/concepts/worker-lifecycle.mdx | 6 +- .../deploy/supabase/configure-secrets.mdx | 8 +- .../docs/get-started/flows/create-flow.mdx | 2 +- .../docs/get-started/flows/quickstart.mdx | 8 +- .../content/docs/reference/compile-api.mdx | 4 + .../docs/reference/control-plane-api.mdx | 6 +- 13 files changed, 268 insertions(+), 222 deletions(-) delete mode 100644 pkgs/website/src/content/docs/concepts/compilation.mdx create mode 100644 pkgs/website/src/content/docs/concepts/manual-compilation.mdx create mode 100644 pkgs/website/src/content/docs/concepts/startup-compilation.mdx diff --git a/pkgs/website/astro.config.mjs b/pkgs/website/astro.config.mjs index 9c9178351..31015f79e 100644 --- a/pkgs/website/astro.config.mjs +++ b/pkgs/website/astro.config.mjs @@ -356,8 +356,12 @@ export default defineConfig({ }, { label: 'Data model', link: '/concepts/data-model/' }, { - label: 'Compilation', - link: '/concepts/compilation/', + label: 'Startup Compilation', + link: '/concepts/startup-compilation/', + }, + { + label: 'Manual Compilation', + link: '/concepts/manual-compilation/', }, { label: 'Worker lifecycle', diff --git a/pkgs/website/redirects.config.mjs b/pkgs/website/redirects.config.mjs index b6ba3c6a1..57ac7852a 100644 --- a/pkgs/website/redirects.config.mjs +++ b/pkgs/website/redirects.config.mjs @@ -77,6 +77,8 @@ export const redirects = { '/concepts/context/': '/concepts/context-object/', '/concepts/flow-dsl/': '/concepts/understanding-flows/', + '/concepts/compilation/': '/concepts/startup-compilation/', + '/reference/compilation-workflow/': '/concepts/manual-compilation/', // ============================================================================ // MAIN BRANCH PATH MIGRATIONS (how-to split to build/deploy/reference/concepts) diff --git a/pkgs/website/src/content/docs/build/local-development.mdx b/pkgs/website/src/content/docs/build/local-development.mdx index fc4f285ac..fa1355904 100644 --- a/pkgs/website/src/content/docs/build/local-development.mdx +++ b/pkgs/website/src/content/docs/build/local-development.mdx @@ -63,8 +63,8 @@ After this initial registration, pgflow's cron keeps the worker running automati description="How workers poll and restart" /> diff --git a/pkgs/website/src/content/docs/concepts/compilation.mdx b/pkgs/website/src/content/docs/concepts/compilation.mdx deleted file mode 100644 index 4a5180cb4..000000000 --- a/pkgs/website/src/content/docs/concepts/compilation.mdx +++ /dev/null @@ -1,202 +0,0 @@ ---- -title: Compilation -description: How pgflow compiles TypeScript flows to SQL via HTTP -sidebar: - order: 25 ---- - -import { Aside } from "@astrojs/starlight/components"; - -pgflow compiles TypeScript flow definitions to SQL migrations via an HTTP-based architecture. This design eliminates the need for a local Deno installation and ensures compilation uses the same runtime as production. - -## How It Works - -When you run `pgflow compile greetUser`, the following happens: - -``` -┌─────────────┐ HTTP GET ┌─────────────────────┐ -│ pgflow CLI │ ─────────────────>│ ControlPlane Edge │ -│ │ │ Function │ -│ │ │ │ -│ │ SQL Array │ 1. Look up flow │ -│ │ <─────────────────│ 2. Call compileFlow│ -│ │ │ 3. Return SQL │ -│ │ └─────────────────────┘ -│ 4. Write │ -│ migration │ -└─────────────┘ -``` - -1. **CLI sends request** - The compile command sends an HTTP GET request to: - `http://127.0.0.1:54321/functions/v1/pgflow/flows/{slug}` - -2. **ControlPlane looks up flow** - The edge function has a registry of flows (from your `index.ts`). It finds the flow by slug. - -3. **Compilation happens in Deno** - The ControlPlane calls `compileFlow()` from `@pgflow/dsl`, which extracts the flow structure and generates SQL. - -4. **SQL returned to CLI** - The response contains an array of SQL statements. - -5. **CLI writes migration** - The CLI joins the SQL and writes it to `supabase/migrations/{timestamp}_create_{slug}_flow.sql`. - -## The ControlPlane Edge Function - -The `pgflow` edge function is created during installation and serves as your project's ControlPlane: - -```typescript title="supabase/functions/pgflow/index.ts" -import { ControlPlane } from '@pgflow/edge-worker'; -import * as flows from '../../flows/index.ts'; - -ControlPlane.serve(flows); -``` - -The ControlPlane: -- **Registers flows** by slug in an in-memory registry -- **Exposes** the `/flows/:slug` endpoint for compilation -- **Returns 404** if a flow slug is not found in the registry -- **Returns 500** with error details if compilation fails - -## Why HTTP-Based Compilation? - -This architecture provides several benefits: - -**No local Deno required** - Users don't need Deno installed on their machine. The Supabase Edge Functions runtime handles everything. - -**Same runtime as production** - Flows are compiled using the exact same Deno environment they'll run in, eliminating "works on my machine" issues. - -**Consistent dependency resolution** - The `deno.json` import map in your edge function ensures consistent package versions. - -**Simpler CLI** - The CLI is a lightweight Node.js package that makes HTTP requests, rather than needing to bundle the entire compilation infrastructure. - -## Auto-Compilation - -When you start an Edge Worker, pgflow automatically compares your TypeScript flow definition with the database schema and takes action based on the environment. - -```d2 -...@../../../assets/pgflow-theme.d2 - -direction: right - -start: "Worker\nstarts" -start.class: step_started - -exists: "Exists?" -exists.shape: diamond -exists.class: info - -compile: "Compile" -compile.class: step_completed - -match: "Same\nshape?" -match.shape: diamond -match.class: info - -verified: "Verified" -verified.class: step_completed - -local: "Local\ndev?" -local.shape: diamond -local.class: warning - -recompile: "Delete old\n& compile new" -recompile.class: step_completed - -fail: "Fail\nto start" -fail.class: step_failed - -polling: "Start\npolling" -polling.class: step_completed - -start -> exists: "lookup flow\nin DB by slug" -exists -> compile: "no" -exists -> match: "yes" -match -> verified: "yes" -match -> local: "no" -local -> recompile: "yes" -local -> fail: "prod" -compile -> polling -verified -> polling -recompile -> polling -``` - -### Local Development - -In local development, auto-compilation enables seamless iteration: - -1. **No flow exists** → compiles and creates it -2. **Flow exists with same shape** → uses existing schema -3. **Flow exists with different shape** → **deletes the existing flow and all its run data**, then compiles the new version - -The workflow is fully automatic: -1. You change your flow code -2. Edge runtime detects the change and restarts the function -3. pgflow cron triggers the function via HTTP request -4. Worker starts, detects shape mismatch, deletes old flow, compiles new one - - - -To disable auto-compilation, set `ensureFlowCompiled: false` in worker options. - -### Production - -In production, shape mismatches cause the worker to **refuse to start**: -- This prevents accidental data loss from running flows -- Worker fails with `FlowShapeMismatchError` -- Resolution: create a versioned flow (e.g., `processOrderV2`) and deploy a new worker - -### What is "Flow Shape"? - -**Changes that affect shape (require versioning):** -- Step slugs -- Step order / dependencies -- Step types (single vs map) - -**Changes that DON'T affect shape (safe to change):** -- Step options (timeout, retries, maxConcurrency) -- Handler code logic - -## Manual Compilation (Optional) - -Some teams prefer explicit compilation for: -- SQL migrations in source control -- Code review of flow structure -- CI/CD validation - -To compile manually: -```bash frame="none" -npx pgflow compile myFlow -``` - -This generates a migration file you can review and apply. - -## Adding New Flows - -To make a flow available for compilation: - -1. Create the flow definition in `supabase/flows/` -2. Export it from `supabase/flows/index.ts` - -The ControlPlane automatically picks up all flows exported from your `flows/index.ts`. - - - -## Troubleshooting - -### "Could not connect to ControlPlane" - -Make sure edge functions are running: - -```bash frame="none" -npx supabase functions serve --no-verify-jwt -``` - -### "Flow not found" - -Register your flow in `supabase/functions/pgflow/index.ts` and restart the functions server. - -## API Reference - -For detailed HTTP endpoint documentation, see [ControlPlane API Reference](/reference/control-plane-api/). diff --git a/pkgs/website/src/content/docs/concepts/index.mdx b/pkgs/website/src/content/docs/concepts/index.mdx index 3e556ed95..bdad8a007 100644 --- a/pkgs/website/src/content/docs/concepts/index.mdx +++ b/pkgs/website/src/content/docs/concepts/index.mdx @@ -30,9 +30,9 @@ Steps execute through **tasks** (the actual units of work) - regular steps have description="Understanding pgflow's database schema design and table relationships" /> +**Manual compilation via ControlPlane may be removed in a future release.** [Startup Compilation](/concepts/startup-compilation/) is the preferred approach - flows are automatically compiled when workers start, eliminating the need for manual compilation in most workflows. + + +Some teams prefer explicit compilation for: +- SQL migrations in source control +- Code review of flow structure changes +- CI/CD validation before deployment + +## The Compile Command + +To compile a flow manually: + +```bash frame="none" +npx pgflow compile myFlow +``` + +This generates a migration file at `supabase/migrations/{timestamp}_create_{slug}_flow.sql`. + +The CLI: +1. Sends an HTTP GET request to the ControlPlane edge function +2. Receives the compiled SQL statements +3. Writes them to a timestamped migration file + +## How Compilation Works + +``` +┌─────────────┐ HTTP GET ┌─────────────────────┐ +│ pgflow CLI │ ─────────────────>│ ControlPlane Edge │ +│ │ │ Function │ +│ │ │ │ +│ │ SQL Array │ 1. Look up flow │ +│ │ <─────────────────│ 2. Call compileFlow│ +│ │ │ 3. Return SQL │ +│ │ └─────────────────────┘ +│ 4. Write │ +│ migration │ +└─────────────┘ +``` + +The ControlPlane edge function is created during installation: + +```typescript title="supabase/functions/pgflow/index.ts" +import { ControlPlane } from '@pgflow/edge-worker'; +import * as flows from '../../flows/index.ts'; + +ControlPlane.serve(flows); +``` + +It registers all flows by slug and exposes the `/flows/:slug` endpoint for compilation. + +## Why HTTP-Based Compilation? + +**No local Deno required** - The Supabase Edge Functions runtime handles everything. Users don't need Deno installed. + +**Same runtime as production** - Flows are compiled using the exact same Deno environment they'll run in, eliminating "works on my machine" issues. + +**Consistent dependency resolution** - The `deno.json` import map in your edge function ensures consistent package versions. + +**Simpler CLI** - The CLI is a lightweight Node.js package that makes HTTP requests, rather than needing to bundle the entire compilation infrastructure. + +## Adding New Flows + +To make a flow available for compilation: + +1. Create the flow definition in `supabase/flows/` +2. Export it from `supabase/flows/index.ts` + +The ControlPlane automatically picks up all flows exported from your `flows/index.ts`. + + + +## Troubleshooting + +### "Could not connect to ControlPlane" + +Make sure edge functions are running: + +```bash frame="none" +npx supabase functions serve --no-verify-jwt +``` + +### "Flow not found" + +Register your flow in `supabase/functions/pgflow/index.ts` and restart the functions server. + +## Related + + + + + + diff --git a/pkgs/website/src/content/docs/concepts/startup-compilation.mdx b/pkgs/website/src/content/docs/concepts/startup-compilation.mdx new file mode 100644 index 000000000..e9d84528e --- /dev/null +++ b/pkgs/website/src/content/docs/concepts/startup-compilation.mdx @@ -0,0 +1,118 @@ +--- +title: Startup Compilation +description: How pgflow automatically compiles flows when workers start +sidebar: + order: 25 +--- + +import { Aside, CardGrid, LinkCard } from "@astrojs/starlight/components"; + +When an Edge Worker starts, pgflow automatically verifies and compiles flows. This eliminates manual compilation steps in most workflows. + +## How It Works + +```d2 +...@../../../assets/pgflow-theme.d2 + +direction: right + +start: "Worker\nstarts" +start.class: step_started + +exists: "Exists?" +exists.shape: diamond +exists.class: info + +compile: "Compile" +compile.class: step_completed + +match: "Same\nshape?" +match.shape: diamond +match.class: info + +verified: "Verified" +verified.class: step_completed + +local: "Local\ndev?" +local.shape: diamond +local.class: warning + +recompile: "Delete old\n& compile new" +recompile.class: step_completed + +fail: "Fail\nto start" +fail.class: step_failed + +polling: "Start\npolling" +polling.class: step_completed + +start -> exists: "lookup flow\nin DB by slug" +exists -> compile: "no" +exists -> match: "yes" +match -> verified: "yes" +match -> local: "no" +local -> recompile: "yes" +local -> fail: "prod" +compile -> polling +verified -> polling +recompile -> polling +``` + +When a worker starts, it: +1. **Looks up the flow** in the database by slug +2. **Compiles if missing** - If the flow doesn't exist, compiles and creates it +3. **Verifies shape** - If the flow exists, compares the TypeScript definition with the database schema +4. **Handles mismatches** - Behavior depends on environment (see below) + +## Local Development + +In local development, startup compilation enables seamless iteration: + +1. **No flow exists** - compiles and creates it +2. **Flow exists with same shape** - uses existing schema +3. **Flow exists with different shape** - **deletes the existing flow and all its run data**, then compiles the new version + +The workflow is fully automatic: +1. You change your flow code +2. Edge runtime detects the change and restarts the function +3. pgflow cron triggers the function via HTTP request +4. Worker starts, detects shape mismatch, deletes old flow, compiles new one + + + +To disable startup compilation, set `ensureFlowCompiled: false` in worker options. + +## Production + +In production, shape mismatches cause the worker to **refuse to start**: +- This prevents accidental data loss from running flows +- Worker fails with `FlowShapeMismatchError` +- Resolution: create a versioned flow (e.g., `processOrderV2`) and deploy a new worker + +## What is "Flow Shape"? + +**Changes that affect shape (require versioning):** +- Step slugs +- Step order / dependencies +- Step types (single vs map) + +**Changes that DON'T affect shape (safe to change):** +- Step options (timeout, retries, maxConcurrency) +- Handler code logic + +## Related + + + + + diff --git a/pkgs/website/src/content/docs/concepts/worker-lifecycle.mdx b/pkgs/website/src/content/docs/concepts/worker-lifecycle.mdx index 5d2bf4179..580aea349 100644 --- a/pkgs/website/src/content/docs/concepts/worker-lifecycle.mdx +++ b/pkgs/website/src/content/docs/concepts/worker-lifecycle.mdx @@ -85,7 +85,7 @@ cron_trigger -> auto_compile: "starts\nworker" auto_compile -> code_change: "ready for\nnext change" {style.stroke-dash: 3} ``` -When you modify your flow code locally, everything restarts automatically (see [Auto-Compilation](/concepts/compilation/#auto-compilation) for the detailed decision tree): +When you modify your flow code locally, everything restarts automatically (see [Startup Compilation](/concepts/startup-compilation/) for the detailed decision tree): 1. **Edge runtime detects change** - Supabase Edge Runtime watches for file changes 2. **Function restarts** - The modified function is reloaded @@ -114,8 +114,8 @@ You don't need to manually curl or restart anything after the initial setup. description="How to register, deprecate, and track workers" /> Project Settings -> General Highlight/arrow pointing to "Reference ID" field Show the alphanumeric project ID value - --> + */} 1. Open [Supabase Dashboard](https://supabase.com/dashboard) 2. Select your project @@ -26,11 +26,11 @@ pgflow needs vault secrets to automatically manage workers in production. 2. ### Get your service role key - + */} 1. Go to **Project Settings** -> **API** 2. Under **Project API keys**, find `service_role` (labeled "secret") diff --git a/pkgs/website/src/content/docs/get-started/flows/create-flow.mdx b/pkgs/website/src/content/docs/get-started/flows/create-flow.mdx index 6f2be87c0..8d800e36d 100644 --- a/pkgs/website/src/content/docs/get-started/flows/create-flow.mdx +++ b/pkgs/website/src/content/docs/get-started/flows/create-flow.mdx @@ -137,7 +137,7 @@ ControlPlane.serve(flows); 4. Trigger a new run to test your changes :::note[Key Concepts] diff --git a/pkgs/website/src/content/docs/get-started/flows/quickstart.mdx b/pkgs/website/src/content/docs/get-started/flows/quickstart.mdx index 80eaaca6d..a7f8b9f56 100644 --- a/pkgs/website/src/content/docs/get-started/flows/quickstart.mdx +++ b/pkgs/website/src/content/docs/get-started/flows/quickstart.mdx @@ -37,7 +37,7 @@ See pgflow in action using the GreetUser flow scaffolded during installation. curl http://localhost:54321/functions/v1/greet-user-worker ``` - You only need to curl once - pgflow [auto-compiles](/concepts/compilation/) and [keeps workers running](/concepts/worker-lifecycle/). + You only need to curl once - pgflow [auto-compiles](/concepts/startup-compilation/) and [keeps workers running](/concepts/worker-lifecycle/). 3. ### Trigger your first flow @@ -91,7 +91,7 @@ Just change your flow code - the rest is automatic: ## Next steps @@ -103,8 +103,8 @@ Recompiling **deletes the existing flow and all its run data**. This is intentio description="Learn how the scaffolded GreetUser flow works" /> +**The ControlPlane and manual compilation may be removed in a future release.** [Startup Compilation](/concepts/startup-compilation/) is the preferred approach - flows are automatically compiled when workers start, eliminating the need for manual compilation in most workflows. + + The `compileFlow` function converts Flow objects into SQL statements that can be executed directly in PostgreSQL. Use this when you need to inspect the generated SQL or integrate flow compilation into custom tooling. ## Usage diff --git a/pkgs/website/src/content/docs/reference/control-plane-api.mdx b/pkgs/website/src/content/docs/reference/control-plane-api.mdx index 7c8410efa..0297cebf7 100644 --- a/pkgs/website/src/content/docs/reference/control-plane-api.mdx +++ b/pkgs/website/src/content/docs/reference/control-plane-api.mdx @@ -7,10 +7,14 @@ sidebar: import { Aside } from "@astrojs/starlight/components"; + + The ControlPlane edge function exposes HTTP endpoints for compiling TypeScript flows to SQL. The `pgflow compile` CLI uses these endpoints internally. ## Base URL