From 93bf3a34be8dc4d98d1d2597900dad9431edd9a1 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 13:10:55 -0700 Subject: [PATCH 01/14] Slot event identity: client half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Number a run's events by dense per-run position instead of by ULID, and make the runtime claim its own event ids. - `@workflow/world`: slot id format/parse helpers, `SPEC_VERSION_SLOT_IDENTITY`, `mintedSpecVersion()` (default on, `WORKFLOW_SLOT_IDENTITY=0` opts out), `eventId`/`maxSlot` on `CreateEventParams`. - `@workflow/core`: contiguous slot reservation off the mutable event log, tail-tight one-at-a-time claims, and a merge/replay/re-claim loop on rejection with a per-slot-run restart budget and randomized backoff. - `@workflow/errors`: `SlotConflictError` carrying the inline event delta. - `@workflow/world-vercel`: sends the claimed id and decodes the 409 delta. - `@workflow/world-local`, `@workflow/world-postgres`: slot allocators, plus a run-scoped events primary key, since `evnt_…001` now exists once per run. Correlation ids are untouched: steps, waits, hooks and attributes keep their seeded ULIDs. --- .changeset/inline-claim-reclaim.md | 5 + .changeset/late-hook-delivery-divergence.md | 6 + .changeset/local-slot-order.md | 5 + .changeset/precondition-restart-backoff.md | 6 + .changeset/slimy-weeks-act.md | 2 + .changeset/slot-claims-tail-tight.md | 7 + .changeset/slot-duplicate-start-skip.md | 6 + .changeset/slot-event-identity-client.md | 9 + .changeset/slot-event-identity-worlds.md | 8 + .changeset/slot-identity-default-on.md | 8 + .changeset/slot-restart-budget.md | 6 + .changeset/slot-restart-cursor-top-up.md | 6 + .changeset/tidy-moons-observe.md | 5 + .changeset/turbo-run-started-occurred-at.md | 5 + .../render-event-log-race-repro-results.js | 5 + .gitignore | 3 + .../api-reference/workflow-errors/index.mdx | 3 + .../api-reference/workflow-errors/meta.json | 1 + .../workflow-errors/slot-conflict-error.mdx | 76 ++ .../docs/v5/configuration/runtime-tuning.mdx | 31 +- .../core/e2e/event-log-race-repro.test.ts | 136 +++- packages/core/e2e/utils.ts | 6 +- packages/core/src/events-consumer.test.ts | 69 +- packages/core/src/events-consumer.ts | 118 ++- packages/core/src/log-format.test.ts | 26 +- packages/core/src/log-format.ts | 20 +- packages/core/src/logger.test.ts | 4 +- packages/core/src/private.ts | 15 +- packages/core/src/runtime.test.ts | 80 +- packages/core/src/runtime.ts | 381 ++++++--- packages/core/src/runtime/constants.test.ts | 60 ++ packages/core/src/runtime/constants.ts | 85 +- packages/core/src/runtime/helpers.test.ts | 383 ++++++++- packages/core/src/runtime/helpers.ts | 312 +++++++- .../runtime/precondition-guard-replay.test.ts | 232 +++++- packages/core/src/runtime/start.test.ts | 33 +- .../core/src/runtime/step-executor.test.ts | 155 +++- packages/core/src/runtime/step-executor.ts | 236 ++++-- .../core/src/runtime/suspension-handler.ts | 41 +- .../core/src/runtime/world-compatibility.ts | 27 +- packages/core/src/workflow.ts | 47 +- packages/core/src/workflow/hook.test.ts | 66 +- packages/core/src/workflow/hook.ts | 41 +- packages/errors/src/index.ts | 53 ++ packages/workflow/src/internal/errors.ts | 1 + packages/world-local/src/fs.ts | 22 +- packages/world-local/src/index.ts | 7 +- .../world-local/src/storage/events-storage.ts | 741 ++++++++++++++---- packages/world-local/src/storage/helpers.ts | 92 ++- .../src/storage/slot-identity.test.ts | 484 ++++++++++++ .../world-local/src/storage/slots.test.ts | 288 +++++++ packages/world-local/src/storage/slots.ts | 327 ++++++++ .../migrations/0018_run_scoped_event_keys.sql | 10 + .../src/drizzle/migrations/meta/_journal.json | 7 + packages/world-postgres/src/drizzle/schema.ts | 8 +- packages/world-postgres/src/index.ts | 7 +- packages/world-postgres/src/slots.ts | 231 ++++++ packages/world-postgres/src/storage.ts | 647 ++++++++++----- .../world-postgres/test/slot-identity.test.ts | 426 ++++++++++ packages/world-vercel/src/event-retry.test.ts | 11 + packages/world-vercel/src/event-retry.ts | 5 + packages/world-vercel/src/events-v4.test.ts | 222 +++++- packages/world-vercel/src/events-v4.ts | 141 +++- packages/world-vercel/src/events.ts | 50 +- packages/world-vercel/src/index.ts | 11 +- packages/world-vercel/src/utils.ts | 5 +- packages/world/package.json | 1 + packages/world/src/events.ts | 31 + packages/world/src/index.ts | 18 + packages/world/src/slot-identity.test.ts | 107 +++ packages/world/src/slot-identity.ts | 121 +++ packages/world/src/spec-version.test.ts | 63 +- packages/world/src/spec-version.ts | 75 +- packages/world/src/ulid.ts | 12 + 74 files changed, 6309 insertions(+), 691 deletions(-) create mode 100644 .changeset/inline-claim-reclaim.md create mode 100644 .changeset/late-hook-delivery-divergence.md create mode 100644 .changeset/local-slot-order.md create mode 100644 .changeset/precondition-restart-backoff.md create mode 100644 .changeset/slimy-weeks-act.md create mode 100644 .changeset/slot-claims-tail-tight.md create mode 100644 .changeset/slot-duplicate-start-skip.md create mode 100644 .changeset/slot-event-identity-client.md create mode 100644 .changeset/slot-event-identity-worlds.md create mode 100644 .changeset/slot-identity-default-on.md create mode 100644 .changeset/slot-restart-budget.md create mode 100644 .changeset/slot-restart-cursor-top-up.md create mode 100644 .changeset/tidy-moons-observe.md create mode 100644 .changeset/turbo-run-started-occurred-at.md create mode 100644 docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx create mode 100644 packages/world-local/src/storage/slot-identity.test.ts create mode 100644 packages/world-local/src/storage/slots.test.ts create mode 100644 packages/world-local/src/storage/slots.ts create mode 100644 packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_keys.sql create mode 100644 packages/world-postgres/src/slots.ts create mode 100644 packages/world-postgres/test/slot-identity.test.ts create mode 100644 packages/world/src/slot-identity.test.ts create mode 100644 packages/world/src/slot-identity.ts diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md new file mode 100644 index 0000000000..9d57541efc --- /dev/null +++ b/.changeset/inline-claim-reclaim.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md new file mode 100644 index 0000000000..148f7df032 --- /dev/null +++ b/.changeset/local-slot-order.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/.changeset/precondition-restart-backoff.md b/.changeset/precondition-restart-backoff.md new file mode 100644 index 0000000000..8e739cd8f0 --- /dev/null +++ b/.changeset/precondition-restart-backoff.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Space out in-process replay restarts with a randomized backoff so concurrent replays of one run stop contending in lockstep diff --git a/.changeset/slimy-weeks-act.md b/.changeset/slimy-weeks-act.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/slimy-weeks-act.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.changeset/slot-claims-tail-tight.md b/.changeset/slot-claims-tail-tight.md new file mode 100644 index 0000000000..346b2c92de --- /dev/null +++ b/.changeset/slot-claims-tail-tight.md @@ -0,0 +1,7 @@ +--- +'@workflow/core': patch +'@workflow/world-local': patch +'@workflow/world-postgres': patch +--- + +Take event slot claims one at a time against the event log's tail, so a replay that decided from a log missing an event is rejected instead of committing. diff --git a/.changeset/slot-duplicate-start-skip.md b/.changeset/slot-duplicate-start-skip.md new file mode 100644 index 0000000000..fc434c6abf --- /dev/null +++ b/.changeset/slot-duplicate-start-skip.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Skip a step whose concurrent start another handler already wrote, instead of restarting the replay to rediscover it diff --git a/.changeset/slot-event-identity-client.md b/.changeset/slot-event-identity-client.md new file mode 100644 index 0000000000..e6d2c5feee --- /dev/null +++ b/.changeset/slot-event-identity-client.md @@ -0,0 +1,9 @@ +--- +'@workflow/world-vercel': minor +'@workflow/core': minor +'@workflow/errors': minor +'@workflow/world': minor +'workflow': minor +--- + +Event creations on runs that number events by slot now claim their own event id and merge, replay and re-claim when a `SlotConflictError` shows another writer took it first. diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md new file mode 100644 index 0000000000..863a2f860b --- /dev/null +++ b/.changeset/slot-event-identity-worlds.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/core': minor +--- + +Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md new file mode 100644 index 0000000000..55e3973586 --- /dev/null +++ b/.changeset/slot-identity-default-on.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +--- + +Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/.changeset/slot-restart-budget.md b/.changeset/slot-restart-budget.md new file mode 100644 index 0000000000..6a4c52c909 --- /dev/null +++ b/.changeset/slot-restart-budget.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Let a run that numbers its events by position absorb more concurrent-write rejections in one invocation, instead of falling back to a delayed re-invocation diff --git a/.changeset/slot-restart-cursor-top-up.md b/.changeset/slot-restart-cursor-top-up.md new file mode 100644 index 0000000000..18752ca81f --- /dev/null +++ b/.changeset/slot-restart-cursor-top-up.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Recover faster from a concurrent write on runs that number their events by slot, by topping the event log up from its cursor instead of reloading it in full diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md new file mode 100644 index 0000000000..9c6c5b7692 --- /dev/null +++ b/.changeset/tidy-moons-observe.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/.github/scripts/render-event-log-race-repro-results.js b/.github/scripts/render-event-log-race-repro-results.js index bbead520b2..c0cc1aadfd 100644 --- a/.github/scripts/render-event-log-race-repro-results.js +++ b/.github/scripts/render-event-log-race-repro-results.js @@ -407,6 +407,11 @@ function renderLatestFailures(entry) { `\nShowing 20 of ${entry.failing.length + entry.truncatedFailingCount} non-completed runs.` ); } + // Deliberately not inlined here: the slices are large and this comment has a + // size limit, while the artifact has none. + console.log( + '\nThe `event-log-race-repro-results` artifact carries a window of the committed log around the divergent event, for a sample of the corruptions.' + ); console.log(''); } diff --git a/.gitignore b/.gitignore index a3f813e31a..3241f9667c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ workbench/nextjs-*/public/.well-known/workflow workbench/sveltekit/static/.well-known/workflow +# Local e2e diagnostics dumps +e2e-diagnostics-*.json + # Event log race repro output (written to the repo root by the harness and by # scripts/event-log-race-repro-local.sh) event-log-race-repro-results.json diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index bd4c061ada..540fc5ccef 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -82,6 +82,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when an event creation is rejected because the client's event-log snapshot is stale. + + Thrown when an event creation is rejected because another writer already took the event's slot. + Thrown when a request is made before the system is ready to process it. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/meta.json b/docs/content/docs/v5/api-reference/workflow-errors/meta.json index a84eddd1ad..b2de769e0e 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/meta.json +++ b/docs/content/docs/v5/api-reference/workflow-errors/meta.json @@ -15,6 +15,7 @@ "throttle-error", "entity-conflict-error", "precondition-failed-error", + "slot-conflict-error", "run-expired-error", "run-not-supported-error", "too-early-error" diff --git a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx new file mode 100644 index 0000000000..96a4f67f17 --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx @@ -0,0 +1,76 @@ +--- +title: SlotConflictError +description: Thrown when an event creation is rejected because another writer already took the event's slot. +type: reference +summary: Catch SlotConflictError when a world rejects an event creation whose slot in the run's event log was already taken. +related: + - /docs/api-reference/workflow-errors/workflow-world-error + - /docs/api-reference/workflow-errors/precondition-failed-error +--- + +`SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics. + +On a run that numbers its events by slot, each event's id encodes its position in the log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. + +The rejection carries the missing events inline so that merge usually costs no extra round-trip: + +- `events` — the events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them, in which case the client reloads the log itself. +- `cursor` — cursor to continue the delta from. +- `hasMore` — whether events beyond `events` remain to be fetched. + +This is the slot-numbering counterpart to [`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error), which is how the same staleness is reported for runs guarded by an event-log snapshot watermark instead. A run uses one scheme or the other for its whole life, decided when it is created. + + +The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly. + + +```typescript lineNumbers +import { SlotConflictError } from "workflow/errors" +declare const world: { events: { create(...args: any[]): Promise } }; // @setup +declare const runId: string; // @setup +declare const event: any; // @setup + +try { + await world.events.create(runId, event); +} catch (error) { + if (SlotConflictError.is(error)) { // [!code highlight] + console.log(`Slot ${error.eventId} taken; ${error.events.length} event(s) missed`); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `SlotConflictError.is(value)` + +Type-safe check for `SlotConflictError` instances. Preferred over `instanceof` because it works across module boundaries and VM contexts. + +```typescript +import { SlotConflictError } from "workflow/errors" +declare const error: unknown; // @setup + +if (SlotConflictError.is(error)) { + // error is typed as SlotConflictError +} +``` diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index c460380f88..d094be19e6 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -59,9 +59,11 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` -- Default: `3` +- Default: `3`, or `12` on a run numbering its events by position (see [`WORKFLOW_SLOT_IDENTITY`](#workflow_slot_identity)) - How many times a single invocation restarts its replay in-process after a rejected event creation before it falls back to a re-invocation. - A restart reloads the event log and rebuilds the workflow from scratch, so it costs a replay but no queue round trip. A World may attach the missing events to its rejection, in which case the first restart needs no event-log request at all. +- A run numbering its events by position reads only the page after its cursor instead of reloading the log, since positions are allocated in order and every event it was missing sorts above what it already has. Restarts are cheap enough there that the higher default is worth taking before a re-invocation and its delay. +- Setting this overrides both defaults. ### `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS` @@ -75,6 +77,27 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Delay before a re-invocation caused by a rejected event creation. - Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up — so the delay gives the other writers a moment to quiesce. +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS` + +- Default: `15` +- Base for the randomized wait before an in-process replay restart re-derives, in milliseconds. The wait doubles with each restart the invocation has spent, up to [`WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`](#workflow_precondition_restart_backoff_max_ms), and is drawn uniformly from zero to that bound. +- The wait exists for runs with several replays in flight at once — a fan-out of steps completing together, or a burst of hooks. A rejected event creation means another replay got there first; if every loser re-derives immediately they all contend again, and none of them pulls far enough ahead to finish. Drawing each wait from the full range spreads the retries apart. +- Set to `0` to restart without waiting. + +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS` + +- Default: `400` +- Ceiling on the wait described above. + +### `WORKFLOW_SLOT_IDENTITY` + +- Default: enabled +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the run replays from the top against a log that now includes them. +- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. +- Set `0` or `false` to disable, which numbers new runs by ULID as before. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` @@ -216,6 +239,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index b577df7b5c..c2895ff944 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -137,6 +137,31 @@ interface ReproRunResult { resumesFailed: number; stragglers?: number; }; + /** + * The committed log around the divergent event, for corruptions only. A + * divergence is a disagreement between the order the log records and the + * order a replay reconstructs, so the log's own ordering is the only + * evidence that distinguishes the candidate causes — and the run is on an + * ephemeral preview deployment, so it has to be captured while the job is + * still running rather than read back afterwards. + */ + logSlice?: LogSliceEntry[]; +} + +/** + * One committed event, projected to the fields that decide replay order: + * its position (`slot`), what it resolves (`eventType`/`correlationId`), and + * both clock domains. `occurredAt` is the client/VM moment and `createdAt` + * the persisted event time the sandbox clock is driven from; a race between a + * `sleep` and a step is decided by that clock, so the two have to be + * comparable side by side. + */ +interface LogSliceEntry { + slot: number | string; + eventType: string; + correlationId?: string; + occurredAt?: string; + createdAt?: string; } function envNumber(name: string, fallback: number) { @@ -392,6 +417,98 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + +/** + * How many events either side of the divergent one to keep. The window has to + * span a whole round of the storm — width branches, each with a step create, + * start and completion, plus the round's waits — or it can miss the very + * event whose position explains the divergence. + */ +const LOG_SLICE_RADIUS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RADIUS', 45); + +/** + * Cap on how many corruptions carry a slice. The results JSON is rendered into + * a PR comment, and a body over GitHub's limit is rejected outright, so the + * slices are a sample rather than a complete record. + */ +const LOG_SLICE_MAX_RUNS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RUNS', 6); + +let logSlicesCaptured = 0; + +/** Worlds hand timestamps back as a Date or as the stored ISO string. */ +function isoOrUndefined(value: unknown): string | undefined { + if (value instanceof Date) return value.toISOString(); + return typeof value === 'string' ? value : undefined; +} + +/** Ordinal of a slot-numbered id, or the raw id when it is a ULID. */ +function idOrdinal(id: string): number | string { + const body = id.slice(id.indexOf('_') + 1); + return /^\d+$/.test(body) ? Number(body) : id; +} + +/** + * Reads the committed log and returns the window around the divergent event + * named in `message`. Best-effort: the report is a measurement, so a failed + * read costs a slice rather than the attempt's classification. + */ +async function readLogSlice( + runId: string, + message: string | undefined +): Promise { + if (logSlicesCaptured >= LOG_SLICE_MAX_RUNS) return undefined; + try { + const world = await getWorld(); + const { data: events } = await world.events.list({ runId }); + const projected: LogSliceEntry[] = events.map((event) => ({ + slot: idOrdinal(event.eventId), + eventType: event.eventType, + correlationId: event.correlationId, + occurredAt: isoOrUndefined(event.occurredAt), + createdAt: isoOrUndefined(event.createdAt), + })); + + // The corruption message names the last divergent event; centre on it when + // it is there, and otherwise keep the tail, where a divergence that ran out + // of recovery replays ends up. + const divergent = message?.match(/evnt_[0-9A-Z]+/)?.[0]; + const at = divergent + ? projected.findIndex((entry) => entry.slot === idOrdinal(divergent)) + : -1; + const centre = at >= 0 ? at : projected.length - 1; + logSlicesCaptured += 1; + return projected.slice( + Math.max(0, centre - LOG_SLICE_RADIUS), + centre + LOG_SLICE_RADIUS + 1 + ); + } catch { + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -431,14 +548,27 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); + const outcome = classifyFailure(failure.errorCode); + const errorMessage = hydrated?.message ?? failure.error?.message; return { ...base, - outcome: classifyFailure(failure.errorCode), + outcome, status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, + logSlice: + outcome === 'CORRUPTED_EVENT_LOG' + ? await readLogSlice(run.runId, errorMessage) + : undefined, }; } diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 8c57ebed70..46f78b501e 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -659,7 +659,11 @@ async function getRunDiagnostics(tracked: TrackedRun): Promise { const elapsed = baseTime ? ((event.createdAt?.getTime?.() ?? 0) - baseTime) / 1000 : 0; - const prefix = ` +${elapsed.toFixed(1)}s`; + // The event's own id leads the line: on a run numbering its events + // by position it says where in the log the event sits, which is what + // a diagnosis of an out-of-order or gapped log needs and what the + // correlation id below cannot report. + const prefix = ` ${event.eventId} +${elapsed.toFixed(1)}s`; let detail = event.eventType; if ('eventData' in event) { const data = (event as any).eventData; diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..5db54b4169 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -84,6 +115,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } /** @@ -200,32 +232,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..c558208a77 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_01ABCDEFGHJKMNPQRSTVWXYZ.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_01ABCDEFGHJKMNPQRSTVWXYZ. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 16fb6a3e4e..0fd7d0a17c 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -547,12 +547,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 750cef009f..5500f1f4ff 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -19,6 +22,7 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from './serialization.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; // Capture every promise handed to `waitUntil` so tests can assert that // progress-critical sends are never registered on a detached, unconsumed @@ -1523,6 +1527,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1535,12 +1550,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1556,8 +1574,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1576,6 +1596,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1621,7 +1642,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1629,7 +1650,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1715,6 +1736,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ @@ -1761,6 +1809,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 40c25db803..11696080ad 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -5,7 +5,6 @@ import { FatalError, HookNotFoundError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -19,13 +18,17 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, + maxSlotOf, ROOT_RUN_ID_ATTRIBUTE, type RunInput, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, + usesSlotIdentity, type WorkflowInvokePayload, WorkflowInvokePayloadSchema, type WorkflowRun, @@ -52,23 +55,26 @@ import { getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, + preconditionRestartBackoffMs, } from './runtime/constants.js'; import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, + claimFenceFor, type EventCreator, + eventCreateFenceFor, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, insertEventByEventId, isPreconditionGuardEnabled, - type LoadedEventLog, + isStaleWriteRejection, loadWorkflowRunEvents, memoizeEncryptionKey, parseHealthCheckPayload, preconditionEventDelta, - preconditionSnapshotParams, queueMessage, + toMutableEventLog, withHealthCheck, } from './runtime/helpers.js'; import { @@ -714,6 +720,12 @@ export function workflowEntrypoint( let cachedEvents: Event[] | null = null; let eventsCursor: string | null = null; + // Set when a restarted replay is recovering by topping its + // cached log up from the cursor rather than reloading it + // whole. The next load verifies the result is dense before + // the replay trusts it; see the consume site below the load. + let slotTopUpPending = false; + // Inline-delta optimization: when an inline step's terminal // write returns the event-log delta since the pre-write // cursor (a supporting World only), we stash it here so the @@ -738,6 +750,19 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published on a slot-numbered run, + // for the writes whose snapshot cannot show it: turbo + // backgrounds `run_started` and replays against an empty log, + // so a claim numbered from that log alone would propose a slot + // `run_started` already holds. 0 when the run is not + // slot-numbered — its ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -901,6 +926,14 @@ export function workflowEntrypoint( // Precondition (412) recovery: how many times this invocation // has thrown away its replay and started over in-process. let preconditionRestarts = 0; + /** + * Wait the next loop iteration owes before re-deriving, set + * by `restartReplayInProcess`. Held here rather than awaited + * at the restart's call sites because the restart is decided + * from three of them, in synchronous code, while the loop head + * is the single point every restart passes through. + */ + let pendingRestartBackoffMs = 0; /** * Event ids the discarded replay held, kept until the next * load resolves so a restart can report what its reload @@ -916,7 +949,7 @@ export function workflowEntrypoint( ids: Set; restart: number; reason: string; - source: 'inline-delta' | 'full-reload'; + source: 'inline-delta' | 'full-reload' | 'slot-top-up'; } | null = null; /** * Report what a stale-snapshot restart's reload found, once @@ -966,13 +999,15 @@ export function workflowEntrypoint( * per-invocation budget is spent (the caller then falls back * to a fresh invocation). * - * A 412 means the log this replay derived its events from was - * missing an event the backend had already recorded. The - * rejected write cannot simply be retried: correlation ids - * are positional ordinals of one seeded sequence, so a replay - * over the corrected log mints a different id for the same - * logical event, and re-posting this one would persist an - * event no correct replay ever produces. The whole replay has + * A rejection — 412 for the event-log watermark, 409 for a + * lost slot claim — means the log this replay derived its + * events from was missing an event the backend had already + * recorded. The rejected write cannot simply be retried: + * correlation ids are positional ordinals of one seeded + * sequence, and a slot event id is a position in the log, so + * a replay over the corrected log mints different ids for the + * same logical event, and re-posting this one would persist + * an event no correct replay ever produces. The whole replay has * to be re-derived — which the loop does by discarding its * cached log, since `runWorkflow` then builds a fresh VM, * seed and correlation-id sequence from the reloaded events. @@ -993,12 +1028,15 @@ export function workflowEntrypoint( ): boolean => { if ( preconditionRestarts >= - getPreconditionMaxInProcessRestarts() + getPreconditionMaxInProcessRestarts( + usesSlotIdentity(workflowRun?.specVersion) + ) ) { return false; } preconditionRestarts++; - // A World MAY return the events we were missing on the 412. + // A World MAY return the events we were missing on the + // rejection. // Trust it only on the FIRST restart: its completeness proof // leans on the backend's own bookkeeping, so if that // under-counts, a "complete" delta can still leave a hole. @@ -1012,6 +1050,25 @@ export function workflowEntrypoint( // to merge it into; with no base log the restart has to load // the whole thing anyway. const usedDelta = Boolean(delta && cachedEvents); + // Without a delta, a slot-numbered log still heals from its + // cursor rather than from a full reload. Slot ids sort in + // write order, so every event this replay was missing is + // strictly above the cursor and one incremental page brings + // it in — and density (a dense log from slot 1 holds + // exactly `maxSlot` events) proves afterwards that it did, + // so nothing is being trusted here that is not checked. + // Neither property holds under ULID ids, which is why those + // restarts reload whole. + const topsUpFromCursor = + !usedDelta && + usesSlotIdentity(workflowRun?.specVersion) && + cachedEvents !== null && + eventsCursor !== null; + const restartSource = usedDelta + ? 'inline-delta' + : topsUpFromCursor + ? 'slot-top-up' + : 'full-reload'; // Snapshot the set being discarded while it is still in // hand; the comparison happens once the next load resolves. preconditionRestartBaseline = cachedEvents @@ -1021,7 +1078,7 @@ export function workflowEntrypoint( ), restart: preconditionRestarts, reason, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } : null; if (usedDelta) { @@ -1029,6 +1086,18 @@ export function workflowEntrypoint( // (`pendingInlineDelta && cachedEvents`) with no // events.list round trip at all. pendingInlineDelta = delta; + } else if (topsUpFromCursor) { + // Keep the cached log and its cursor: the loop's + // incremental branch fetches the page above the cursor + // and appends it, and the density check below the load + // sends the restart to a full reload if that page did not + // close the gap. Appends land above everything already + // scanned for payload prewarming, so no rescan is needed + // unless that fallback fires. + slotTopUpPending = true; + preloadedEvents = undefined; + preloadedEventsCursor = undefined; + pendingInlineDelta = null; } else { // MUST be a full, cursor-less reload. The cursor filters // by lexicographic event id while a hole is defined by @@ -1055,12 +1124,14 @@ export function workflowEntrypoint( reason, loopIteration, preconditionRestarts, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } ); span?.setAttributes({ 'workflow.precondition_restarts': preconditionRestarts, }); + pendingRestartBackoffMs = + preconditionRestartBackoffMs(preconditionRestarts); return true; }; @@ -1101,7 +1172,7 @@ export function workflowEntrypoint( return { reinvoked: false, error: new WorkflowRuntimeError( - `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts()} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, + `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts(usesSlotIdentity(workflowRun?.specVersion))} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, { cause: error } ), }; @@ -1438,6 +1509,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = createEvent( runStartedEvent, // We background this purely as a write barrier and @@ -1447,7 +1528,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1460,6 +1541,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1476,7 +1561,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; - const now = new Date(); + // A slot-numbered run's first two positions are the run's + // own: `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } workflowRun = { runId, status: 'running', @@ -1525,6 +1623,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1779,6 +1878,19 @@ export function workflowEntrypoint( while (true) { loopIteration++; + // A restart lost a slot to a concurrent replay of this same + // run. Pause before re-deriving so the winner gets a clear + // window to extend the log: re-deriving immediately puts + // every loser back in contention at once, and none of them + // pulls ahead. + if (pendingRestartBackoffMs > 0) { + const backoffMs = pendingRestartBackoffMs; + pendingRestartBackoffMs = 0; + await new Promise((resolve) => + setTimeout(resolve, backoffMs) + ); + } + // Replay-budget check: bail out (retry or fail) if // non-step time within this invocation has exceeded // the configured budget. Step bodies are excluded @@ -1918,6 +2030,26 @@ export function workflowEntrypoint( // the wait pass, which may swap in a freshly loaded array. cachedEvents = events; + if (slotTopUpPending) { + slotTopUpPending = false; + // A slot-numbered log is dense from slot 1, so a + // complete one holds exactly `maxSlot` events. A short + // count means the page above the cursor did not bring + // in everything the restart was missing — the only + // other reading, a permanent hole from a write that + // took a slot and then failed, is equally unrecoverable + // from here — so fall back to the authoritative load. + if (maxSlotOf(events) !== events.length) { + const loaded = await loadWorkflowRunEvents(runId); + events = loaded.events; + eventsCursor = loaded.cursor; + cachedEvents = events; + // The reload can insert events below the prefix + // already scanned for payload prewarming. + replayPayloadCache.resetScan(); + } + } + reportPreconditionRestartReload(events); // Detect concurrent completion via the event log: if @@ -1959,12 +2091,24 @@ export function workflowEntrypoint( }, })); + // One log for the whole loop: `events` is appended to in + // place by the guard's reloads, so a per-iteration + // rescan for the slot high-water mark would be wasted + // work on an array that never changes identity here. + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + const waitClaim = claimFenceFor( + waitLog, + workflowRun.specVersion + ); for (const waitEvent of waitsToComplete) { try { - await createEvent(waitEvent, { - requestId, - ...preconditionSnapshotParams(events, eventsCursor), - }); + await waitClaim((fence) => + createEvent(waitEvent, { requestId, ...fence }) + ); } catch (err) { if (EntityConflictError.is(err)) { runtimeLogger.info( @@ -2073,6 +2217,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + // Replay workflow runtimeLogger.debug('Starting workflow replay', { workflowRunId: runId, @@ -2098,7 +2256,8 @@ export function workflowEntrypoint( // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. runReadyBarrier, - world.capabilities + world.capabilities, + replayWriteLog ); await payloadPrewarm; runtimeLogger.debug('Workflow replay completed', { @@ -2110,10 +2269,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below restarts the replay in-process. + // computed by this replay, so a rejection proving the view + // was incomplete — 412 stale watermark, or 409 taken slot + // — must force a *fresh replay* (which may observe the new + // event and produce a different result), not re-commit the + // stale result. The catch below restarts the replay + // in-process. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -2127,7 +2288,10 @@ export function workflowEntrypoint( }, { requestId, - ...preconditionSnapshotParams(events, eventsCursor), + ...eventCreateFenceFor( + replayWriteLog, + workflowRun.specVersion + ), } ); } catch (err) { @@ -2196,12 +2360,13 @@ export function workflowEntrypoint( } // V2: handle suspension without queuing steps. - // Each event creation inside handleSuspension carries the - // precondition snapshot of the loaded event log, so a - // backend holding an event this replay never saw rejects - // the write (412) instead of accepting a divergent one. - // The rejection is handled here, by restarting the - // replay — never by re-posting the same event. + // Each event creation inside handleSuspension carries + // the run's concurrency fence — its own event slot, or + // the loaded snapshot's watermark — so a backend holding + // an event this replay never saw rejects the write + // (409/412) instead of accepting a divergent one. The + // rejection is handled here, by restarting the replay — + // never by re-posting the same event. const suspensionStart = Date.now(); // The snapshot refresh above always sets cachedEvents // before the replay can suspend. Re-narrow it for this @@ -2214,10 +2379,11 @@ export function workflowEntrypoint( 'Invariant violation: workflow suspended before its event log was loaded' ); } - const suspensionLog: LoadedEventLog = { - events: cachedEvents, - cursor: eventsCursor, - }; + const suspensionLog = toMutableEventLog( + cachedEvents, + eventsCursor, + knownSlotFloor + ); let suspensionResult: Awaited< ReturnType >; @@ -2233,14 +2399,15 @@ export function workflowEntrypoint( replayRecoveryReporter, }); } catch (suspensionError) { - // A suspension create was rejected as stale: re-derive - // the replay from a corrected log in this invocation. - // Once the in-process budget is spent, fall back to an + // A suspension create was rejected as stale (412 + // watermark, or 409 taken slot): re-derive the replay + // from a corrected log in this invocation. Once the + // in-process budget is spent, fall back to an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (isStaleWriteRejection(suspensionError)) { if ( restartReplayInProcess( 'suspension-create', @@ -2705,11 +2872,15 @@ export function workflowEntrypoint( // rejected with 412 — its guarded suspension creates // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started - // claim of its next inline step, which carries the - // snapshot too (threaded below via - // `stateUpdatedAt`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // claim of its next inline step, which is fenced too + // (threaded below via `claimFenceFor`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot-numbered run gets there differently — the + // claim merges the missed events and retries in + // place, so the same events are observed without + // discarding the batch. See claimFenceFor. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -2849,21 +3020,23 @@ export function workflowEntrypoint( turbo, }); - // Precondition-guard snapshot for the inline - // step_started claims: the lazy claim is the first - // durable write of a hot-path step (its step_created - // is deferred), so without a snapshot it would bypass - // the guard entirely and a stale replay could claim — - // and commit — a step scheduled off a view that misses - // an event it never loaded. - // `preconditionSnapshotParams` returns an empty object - // when the guard env flag is off, so this is a no-op - // outside guarded deployments; Worlds that don't - // enforce the guard ignore it. - const inlineClaimSnapshot = preconditionSnapshotParams( - cachedEvents ?? [], - preInlineWriteCursor - ); + // Concurrency fence for the inline step_started claims: + // the lazy claim is the first durable write of a + // hot-path step (its step_created is deferred), so + // without one it would be unguarded and a stale replay + // could claim — and commit — a step scheduled off a view + // that misses an out-of-band event. One log for the + // whole batch so each claim draws its own event slot; + // for a run fenced by the watermark instead, every claim + // in the batch carries the same snapshot. + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -2871,6 +3044,26 @@ export function workflowEntrypoint( >[]; const stepExecutionPromises = inlineExecutions.map( (s, stepIndex) => { + // Drawn here — synchronously, in replay order — + // rather than inside `run`: a slot claim is + // positional, so it has to be assigned before these + // executions start racing each other, and the order + // it is assigned in has to be replay-stable. + const claimFence = claimFenceFor( + inlineClaimLog, + workflowRun.specVersion, + { + // A lazy start publishes two events: the World + // writes the step's deferred `step_created` + // alongside the claim, so the batch has to + // reserve a slot for that one too — otherwise + // it lands on the slot the next start in the + // batch is holding and costs that start its + // claim. + extraEvents: + s.lazyStepInput !== undefined ? 1 : 0, + } + ); const run = () => executeStep({ world, @@ -2942,7 +3135,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - preconditionSnapshot: inlineClaimSnapshot, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2984,23 +3177,26 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is missing an event the backend already has, - // so the claim was fenced by the guard and no step - // events were written. Abandon the batch — any - // optimistic body result is discarded by executeStep's - // reconciliation — and restart the replay so it - // observes the missing event. Wait for the sibling - // executions to settle first so no owned body is in - // flight when the restart (or the ack path) runs. - if (PreconditionFailedError.is(stepErr)) { + // An incomplete-view rejection of an inline + // step_started claim — 412 for the event-log + // watermark, 409 for a lost slot claim: the loaded + // view this batch was scheduled from is missing an + // event the backend already has, so the claim was + // fenced and no step events were written. Abandon the + // batch — any optimistic body result is discarded by + // executeStep's reconciliation — and restart the + // replay so it observes the missing event. Wait for + // the sibling executions to settle first so no owned + // body is in flight when the restart (or the ack + // path) runs. + if (isStaleWriteRejection(stepErr)) { const settled = await Promise.allSettled( stepExecutionPromises ); // A sibling whose claim was accepted wrote step // events of its own, possibly after the World built - // this 412's delta — so that delta can no longer be + // the rejection's delta — so that delta can no + // longer be // assumed to complete the log, and the restart has // to reload it in full. `skipped` (the step already // existed), `gone` and `throttled` (claim rejected) @@ -3138,7 +3334,7 @@ export function workflowEntrypoint( // correlationId so it dedupes against the // keyed re-dispatch the suspension handler // performs on replay (it also uses - // `idempotencyKey: step.correlationId`). + // the same run-scoped correlationId key). // // Without this, a mixed batch where one step // `completed` with unflushed background ops @@ -3215,24 +3411,25 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a guarded write made - // directly by the replay loop — the result-bearing - // `run_completed`, or the `wait_completed` of the wait - // pass. Both reach this one catch and the rejection - // does not say which, hence the neutral label. - // Neither may be re-posted in place: the correlation id - // and (for run_completed) the result itself came from - // this replay, and a corrected log may produce - // different ones. Don't fail the run — restart the - // replay in this invocation, and only once that budget - // is spent schedule an explicit re-invocation. + // Incomplete-view rejection of a guarded write made + // directly by the replay loop — a stale watermark (412) + // or a taken slot (409), on the result-bearing + // `run_completed` or on the `wait_completed` of the + // wait pass. Both reach this one catch and the + // rejection does not say which, hence the neutral + // label. Neither may be re-posted in place: the + // correlation id and (for run_completed) the result + // itself came from this replay, and a corrected log may + // produce different ones. Don't fail the run — restart + // the replay in this invocation, and only once that + // budget is spent schedule an explicit re-invocation. // Rethrowing instead would rely on redelivery of the // CURRENT message, which the turbo path has already // acked — empirically the run then stalls for the // queue's ~300s default visibility timeout before // completing. let terminalError = err; - if (PreconditionFailedError.is(err)) { + if (isStaleWriteRejection(err)) { if (restartReplayInProcess('replay-write', err)) { continue; } @@ -3375,10 +3572,10 @@ export function workflowEntrypoint( // type identity and custom properties round-trip // through the event log. // - // Precondition-guard asymmetry: unlike `run_completed`, - // this terminal `run_failed` sends no `stateUpdatedAt` - // snapshot, so it is never 412-rejected even if a hook - // landed mid-replay and could have changed the path that + // Fencing asymmetry: unlike `run_completed`, this + // terminal `run_failed` carries no concurrency fence, so + // it is never rejected even if a hook landed mid-replay + // and could have changed the path that // threw. This is intentional and fail-open: a spurious // failure is recoverable (the run can be re-run from the // dashboard), whereas a spurious *completion* commits a diff --git a/packages/core/src/runtime/constants.test.ts b/packages/core/src/runtime/constants.test.ts index 208250e8e6..919c4edd14 100644 --- a/packages/core/src/runtime/constants.test.ts +++ b/packages/core/src/runtime/constants.test.ts @@ -18,6 +18,9 @@ import { MAX_REPLAY_TIMEOUT_MS, MIN_MAX_INLINE_STEPS, MIN_REPLAY_TIMEOUT_MS, + PRECONDITION_RESTART_BACKOFF_BASE_MS, + PRECONDITION_RESTART_BACKOFF_MAX_MS, + preconditionRestartBackoffMs, REPLAY_TIMEOUT_MS, } from './constants.js'; @@ -393,3 +396,60 @@ describe('getInlineOwnershipLeaseSeconds', () => { expect(getInlineOwnershipLeaseSeconds()).toBe(1); }); }); + +describe('preconditionRestartBackoffMs', () => { + const BASE_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS'; + const MAX_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS'; + + beforeEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + afterEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + it('doubles the window on each successive restart', () => { + // Draw the top of the window so the window itself is observable. + const top = () => 0.999_999; + expect(preconditionRestartBackoffMs(1, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS - 1 + ); + expect(preconditionRestartBackoffMs(2, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 2 - 1 + ); + expect(preconditionRestartBackoffMs(3, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 4 - 1 + ); + }); + + it('draws over the whole window, not a fixed delay plus noise', () => { + // Full jitter is the property that decorrelates concurrent replays; a + // floor would keep them in lockstep however long the wait. + expect(preconditionRestartBackoffMs(5, () => 0)).toBe(0); + expect(preconditionRestartBackoffMs(5, () => 0.5)).toBeLessThan( + preconditionRestartBackoffMs(5, () => 0.999_999) + ); + }); + + it('caps the window', () => { + expect(preconditionRestartBackoffMs(40, () => 0.999_999)).toBe( + PRECONDITION_RESTART_BACKOFF_MAX_MS - 1 + ); + }); + + it('is disabled by a zero base', () => { + process.env[BASE_ENV] = '0'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(0); + expect(preconditionRestartBackoffMs(9, () => 0.999_999)).toBe(0); + }); + + it('honours overrides of both the base and the cap', () => { + process.env[BASE_ENV] = '100'; + process.env[MAX_ENV] = '150'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(99); + expect(preconditionRestartBackoffMs(2, () => 0.999_999)).toBe(149); + }); +}); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index d5fea8383b..9a42399899 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -397,14 +397,32 @@ export function getReplayDivergenceMaxRetries(): number { // run-level budget below then applies. export const PRECONDITION_MAX_INPROCESS_RESTARTS = 3; +// A run that numbers its events by slot recovers by topping its log up from +// its cursor, so a restart costs one incremental page instead of a full +// reload. The tight bound above is priced for the reload; spending it here +// buys a re-invocation — a queue hop plus `PRECONDITION_REINVOKE_DELAY_SECONDS` +// — in place of restarts that are orders of magnitude cheaper than the thing +// the bound was protecting against. Measured on the step-storm repro against +// world-postgres at 6-way concurrency, raising this to 12 took the six runs +// from 196–241s (two of them exceeding the harness timeout) to 119–148s with +// none timing out. +export const PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL = 12; + /** * Effective in-process replay-restart budget for stale-snapshot rejections. * Override via `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`. + * + * @param incremental Whether a restart heals from the log's cursor rather than + * reloading it whole, which holds for runs on slot identity. */ -export function getPreconditionMaxInProcessRestarts(): number { +export function getPreconditionMaxInProcessRestarts( + incremental = false +): number { return envNumber( 'WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS', - PRECONDITION_MAX_INPROCESS_RESTARTS, + incremental + ? PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL + : PRECONDITION_MAX_INPROCESS_RESTARTS, { integer: true } ); } @@ -447,3 +465,66 @@ export function getPreconditionReinvokeDelaySeconds(): number { { integer: true } ); } + +// Concurrent replays of one run contend for the same event slots, and a +// rejected write costs its loser a restart. With no pause the losers re-derive +// at full speed and collide again immediately, so a run under heavy fan-out can +// spend its whole invocation budget with no writer ever pulling far enough +// ahead to finish. A short randomized wait between restarts breaks the lockstep +// by spreading the retries. +export const PRECONDITION_RESTART_BACKOFF_BASE_MS = 15; + +// Ceiling on that wait. The restart is cheap (one incremental page plus a +// re-derive), so the backoff must stay well under the cost of the +// re-invocation it is competing with. +export const PRECONDITION_RESTART_BACKOFF_MAX_MS = 400; + +/** + * Effective base for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS`; `0` disables the wait entirely. + */ +export function getPreconditionRestartBackoffBaseMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS', + PRECONDITION_RESTART_BACKOFF_BASE_MS, + { integer: true } + ); +} + +/** + * Effective ceiling for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`. + */ +export function getPreconditionRestartBackoffMaxMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS', + PRECONDITION_RESTART_BACKOFF_MAX_MS, + { integer: true } + ); +} + +/** + * Full-jitter backoff for the `restarts`-th in-process replay restart, in + * milliseconds. + * + * Full jitter (a uniform draw over the whole window, not a fixed delay plus + * noise) is what actually decorrelates the racers: equal-length waits would + * keep colliding replays in lockstep no matter how long they were. + * + * @param restarts 1-based count of restarts spent so far, including this one. + * @param random Injectable uniform source; defaults to `Math.random`. The delay + * never reaches the event log, so it does not affect replay determinism. + */ +export function preconditionRestartBackoffMs( + restarts: number, + random: () => number = Math.random +): number { + const base = getPreconditionRestartBackoffBaseMs(); + if (base <= 0) return 0; + const exponent = Math.max(0, restarts - 1); + const window = Math.min( + getPreconditionRestartBackoffMaxMs(), + base * 2 ** exponent + ); + return Math.floor(random() * window); +} diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index d71fdfd6a3..78b11985a2 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,5 +1,15 @@ -import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; -import type { Event, World } from '@workflow/world'; +import { + EntityConflictError, + PreconditionFailedError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; +import { + type Event, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + type World, +} from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; @@ -12,15 +22,21 @@ import { } from '../serialization.js'; import { appendUniqueEvents, + claimFenceFor, + eventCreateFenceFor, getWorkflowQueueName, handleHealthCheckMessage, healthCheck, insertEventByEventId, + isStaleWriteRejection, latestEventStateUpdatedAt, loadWorkflowRunEvents, memoizeEncryptionKey, + mergeLoadedEvents, preconditionEventDelta, preconditionSnapshotParams, + reserveSlot, + toMutableEventLog, } from './helpers.js'; // Mock the logger to suppress output during tests @@ -626,6 +642,313 @@ describe('latestEventStateUpdatedAt', () => { }); }); +describe('slot bookkeeping', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + it('reads maxSlot from the highest slot present, not the last element', () => { + // Nothing forces a caller's array into slot order — a World is free to + // hand back a page in whatever order its index produced — so the highest + // slot is a scan, not a peek at the last element. + const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); + expect(log.maxSlot).toBe(3); + expect(log.nextSlot).toBe(4); + }); + + it('reports maxSlot 0 for an empty or ULID-numbered log', () => { + expect(toMutableEventLog([], null).maxSlot).toBe(0); + expect( + toMutableEventLog([makeUlidEvent(1_700_000_000_000)], null).maxSlot + ).toBe(0); + }); + + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + + it('never lowers maxSlot when an older delta is merged in', () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); + mergeLoadedEvents(log, [slotEvent(2)]); + expect(log.maxSlot).toBe(3); + expect(log.events).toHaveLength(3); + }); + + it('raises the reservation pointer past a newer delta', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + reserveSlot(log); + reserveSlot(log); + expect(log.nextSlot).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); + + expect(log.maxSlot).toBe(5); + expect(reserveSlot(log)).toBe(6); + }); + + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + + it('deduplicates merged events by id', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + ]); + }); + + it('restores slot order when a merge brings in a lower slot', () => { + // Arrival order is not log order: a slot is reserved when its event is + // issued and written when the issue resolves, so a lower slot can be + // learned after a higher one. The replay consumes this array positionally, + // so an event left sitting ahead of the one it followed decides races the + // wrong way. + const log = toMutableEventLog([slotEvent(1), slotEvent(4)], 'c0'); + mergeLoadedEvents(log, [slotEvent(3), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + slotEventId(3), + slotEventId(4), + ]); + }); + + it('leaves a ULID log in the order the World returned it', () => { + // ULID ids are minted at write time, so arrival order *is* log order and + // the World's ordering is the authority. + const events = [makeUlidEvent(1_700_000_000_000)]; + const later = makeUlidEvent(1_700_000_001_000); + const earlier = makeUlidEvent(1_699_999_999_000); + const log = toMutableEventLog(events, 'c0'); + mergeLoadedEvents(log, [later, earlier]); + expect(log.events).toEqual([events[0], later, earlier]); + }); + + it('hands out contiguous distinct slots for a synchronous burst', () => { + // The suspension flush issues every operation synchronously and awaits + // them together; without contiguous reservation they would all propose the + // same slot and all but one would conflict. + const log = toMutableEventLog([slotEvent(4)], 'c0'); + const burst = Array.from({ length: 20 }, () => reserveSlot(log)); + expect(burst).toEqual(Array.from({ length: 20 }, (_, i) => 5 + i)); + expect(new Set(burst).size).toBe(burst.length); + // Reservations sit past maxSlot rather than moving it: only merged events + // prove a slot is taken. + expect(log.maxSlot).toBe(4); + }); + + it('proposes a padded event id only for a slot-identity run', () => { + const log = toMutableEventLog([], null); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(1), + maxSlot: 0, + }); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(2), + maxSlot: 0, + }); + }); + + it('reserves a slot per extra event and names the top one', () => { + // A lazy inline `step_started` publishes two events: the World also writes + // the `step_created` it deferred, which takes the slot below the claim. + // Reserving it here is what keeps it off the slot the next write of the + // same batch will claim. + const log = toMutableEventLog([slotEvent(1)], null); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(3), maxSlot: 1 }); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(5), maxSlot: 1 }); + // A single-event write in the same batch still gets the next free slot. + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(6), + maxSlot: 1, + }); + }); + + it('burns no slot on an extra event of a ULID-numbered run', () => { + const log = toMutableEventLog([], null); + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { + extraEvents: 1, + }); + expect(log.nextSlot).toBe(1); + }); + + it('proposes no event id for a ULID-numbered run', () => { + // A run whose ids the backend mints must not burn slots either. + const log = toMutableEventLog([], null); + const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + expect(fence?.eventId).toBeUndefined(); + expect(log.nextSlot).toBe(1); + }); +}); + +describe('claimFenceFor', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('claims the next free slot and passes the observed maxSlot alongside it', async () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(2)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async () => 'ok'); + + await expect(claim(op)).resolves.toBe('ok'); + expect(op).toHaveBeenCalledWith({ eventId: slotEventId(3), maxSlot: 2 }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('takes each claim only once the create ahead of it has committed', async () => { + // A claim fences out a concurrent writer only while it names the slot right + // after the tail the writer saw, so a second create cannot be numbered + // until the first has landed. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const claimed: (string | undefined)[] = []; + let releaseFirst!: () => void; + const firstLanded = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const first = claim(async (fence) => { + claimed.push(fence?.eventId); + await firstLanded; + return 'first'; + }); + const second = claim(async (fence) => { + claimed.push(fence?.eventId); + return 'second'; + }); + + await Promise.resolve(); + expect(claimed).toEqual([slotEventId(2)]); + + releaseFirst(); + await expect(first).resolves.toBe('first'); + await expect(second).resolves.toBe('second'); + expect(claimed).toEqual([slotEventId(2), slotEventId(3)]); + }); + + it('rejects a lost claim rather than re-addressing the write', async () => { + // A 409 says this replay decided from a log missing an event. Moving the + // same write to a free slot would commit that decision anyway, so the + // rejection propagates and the run replays. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async (fence?: { eventId?: string }) => { + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(SlotConflictError); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('leaves the rest of the batch claiming into the occupied range', async () => { + // The tail stops advancing for this log while the backend's moves on, so + // the siblings behind a rejected claim propose slots the backend has + // already filled and are rejected with it. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const loser = claim(async (fence) => { + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + }); + }); + await expect(loser).rejects.toBeInstanceOf(SlotConflictError); + + await expect(claim(async (fence) => fence?.eventId)).resolves.toBe( + slotEventId(2) + ); + }); + + it('numbers a batch in the order its claims fire, each one above the last', async () => { + // A step that publishes the `step_created` it deferred takes the two slots + // below its claim, and the next claim starts above both. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const withCreate = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const plain = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + await withCreate(record); + await plain(record); + + expect(claimed).toEqual([slotEventId(3), slotEventId(4)]); + }); + + it('rethrows a non-conflict error immediately, without merging', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unserialized', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith( + expect.objectContaining({ stateUpdatedAt: time }) + ); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + describe('preconditionSnapshotParams', () => { let originalGuard: string | undefined; @@ -833,6 +1156,62 @@ describe('preconditionEventDelta', () => { expect(delta({ events: [null] })).toBe(null); expect(delta('not-an-object')).toBe(null); }); + + it('reads the delta a lost slot claim carries as typed fields', () => { + const event = makeUlidEvent(1_700_000_000_000); + + expect( + preconditionEventDelta( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [event], + cursor: 'eid:next', + hasMore: false, + }), + RUN_ID + ) + ).toEqual({ events: [event], cursor: 'eid:next' }); + }); + + it('refuses a truncated slot-conflict delta', () => { + // A partial delta would restart the replay on a log that is still missing + // events; only a full reload can prove it saw all of them. + expect( + preconditionEventDelta( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [makeUlidEvent(1_700_000_000_000)], + cursor: 'eid:next', + hasMore: true, + }), + RUN_ID + ) + ).toBe(null); + }); +}); + +describe('isStaleWriteRejection', () => { + it('accepts both rejections that prove the replay read an incomplete log', () => { + expect(isStaleWriteRejection(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + isStaleWriteRejection( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [], + cursor: null, + hasMore: false, + }) + ) + ).toBe(true); + }); + + it('rejects every other failure, which is not recovered by a restart', () => { + expect(isStaleWriteRejection(new WorkflowWorldError('boom'))).toBe(false); + expect(isStaleWriteRejection(new Error('boom'))).toBe(false); + expect(isStaleWriteRejection(undefined)).toBe(false); + }); }); describe('memoizeEncryptionKey', () => { diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 29096e7400..d9e367cf05 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1,6 +1,7 @@ import { PreconditionFailedError, RUN_ERROR_CODES, + SlotConflictError, WorkflowWorldError, } from '@workflow/errors'; import type { @@ -17,10 +18,14 @@ import { getQueueTopicPrefix, HealthCheckPayloadSchema, HOOK_RESUME_INPUT_VERSION, + isSlotId, + maxSlotOf, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + slotEventId, ulidToDate, + usesSlotIdentity, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; @@ -482,7 +487,22 @@ function recordRequestedEventCursor( } /** - * Appends events whose IDs are not already present in `target`. + * Appends events whose IDs are not already present in `target`, keeping a + * slot-numbered log in slot order. + * + * Arrival order is not log order under slot identity. A slot is reserved when + * its event is issued and written when that issue resolves, so a lower slot can + * be committed after a higher one, and a merge that only appends leaves the + * array in the order the events were *learned*, not the order they occupy. + * That difference decides races: the replay consumes this array positionally + * — the delivery barriers in `pendingDeliveryBarriers` are keyed on the index — + * so a `step_completed` sitting ahead of a `wait_completed` it actually + * follows makes the replay take the branch the log does not record, and the + * next event it reads belongs to a step it never started. + * + * Sorting by event id *is* sorting by slot: ids are zero-padded to a fixed + * width, and a slot-numbered run's log carries no ULID ids to interleave with + * them. * * Pass the IDs currently present in `target` when appending repeatedly to the * same array. The set is updated alongside `target`. @@ -506,13 +526,19 @@ export function appendUniqueEvents( } const ids = targetIds ?? new Set(target.map((event) => event.eventId)); + let outOfOrder = false; for (const event of events) { if (ids.has(event.eventId)) { continue; } ids.add(event.eventId); + outOfOrder ||= + target.length > 0 && event.eventId < target[target.length - 1].eventId; target.push(event); } + if (outOfOrder && isSlotId(target[0].eventId)) { + target.sort((a, b) => (a.eventId < b.eventId ? -1 : 1)); + } } /** @@ -708,6 +734,89 @@ export interface LoadedEventLog { cursor: string | null; } +/** + * A loaded snapshot that also tracks the run's event slots, for the numbering + * where the event id is itself the concurrency fence. Slot state is per-replay: + * it is rebuilt from the events every time the log is loaded, never carried + * across a restart. + */ +export interface MutableEventLog extends LoadedEventLog { + /** + * Highest slot present in `events`, or 0 for a log that is empty or + * ULID-numbered. Maintained by `mergeLoadedEvents` from the events merged in, + * never from the array's last element: events are appended without sorting, + * so after a merge the last element need not be the newest. + */ + maxSlot: number; + /** + * Next slot `reserveSlot` will hand out. Only a writer holding the log's + * write chain may draw from it, and it is rewound to `maxSlot + 1` when a + * claim is rejected so the rest of the batch claims into the occupied range + * and is rejected with it. + */ + nextSlot: number; + /** + * Tail of the chain of creates numbered off this log, or `undefined` when + * none is in flight. + * + * Slot claims are taken one at a time. A claim only fences out a concurrent + * writer if it names the slot right after the log's committed tail: numbering + * a whole concurrent batch up front hands its later writes slots far enough + * above the tail that a foreign event landing in between clears every fence + * they carry, and the batch commits decisions taken without it. + */ + writeChain?: Promise; +} + +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ +export function toMutableEventLog( + events: Event[], + cursor: string | null, + slotFloor = 0 +): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); + return { + events, + cursor, + maxSlot, + nextSlot: maxSlot + 1, + }; +} + +/** + * Merges loaded events into `log` in place, keeping `maxSlot` current and + * advancing the reservation pointer past the events merged in. + */ +export function mergeLoadedEvents( + log: MutableEventLog, + events: readonly Event[] +): void { + appendUniqueEvents(log.events, events); + log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); +} + +/** + * Claims the next free slot in `log`. + * + * Only call this while holding the log's write chain: the claim is the fence, + * and it only fences anything while it names the slot immediately after the + * tail this writer has seen. + */ +export function reserveSlot(log: MutableEventLog): number { + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; +} + /** * Whether the optimistic-concurrency guard for event creation is enabled. * **On by default** where the runtime executes: replay-context creates send a @@ -824,8 +933,25 @@ export function preconditionSnapshotParams( } /** - * The events a rejecting World attached to a `PreconditionFailedError`, when it - * returned the ones the client's snapshot was missing inline. + * Whether a World rejected an event creation because the replay that produced it + * had not seen the whole event log. + * + * The two schemes reject differently and stay separately countable — 412 for the + * event-log watermark, 409 for a lost slot claim, both live at once while runs on + * the older numbering drain — but they prove the same thing and are recovered the + * same way, by restarting the replay over the corrected log. + */ +export function isStaleWriteRejection(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + +/** + * The events a rejecting World attached to its rejection, when it returned the + * ones the client's snapshot was missing inline. + * + * A slot conflict carries them as typed fields; the watermark guard carries them + * in `details`. Both are read here so callers recover from either without + * branching. * * Returns `null` for anything else — no details, a World that did not implement * this, or a payload that does not narrow cleanly. Callers fall back to @@ -841,14 +967,25 @@ export function preconditionEventDelta( error: unknown, runId: string ): { events: Event[]; cursor: string | null } | null { - if (!PreconditionFailedError.is(error)) { - return null; - } - const details = error.details; - if (typeof details !== 'object' || details === null) { + let events: unknown; + let cursor: unknown; + if (SlotConflictError.is(error)) { + // A truncated delta is not a delta: the restart has to see every event it + // was missing, and only a full reload can guarantee that. + if (error.hasMore) { + return null; + } + events = error.events; + cursor = error.cursor; + } else if (PreconditionFailedError.is(error)) { + const details = error.details; + if (typeof details !== 'object' || details === null) { + return null; + } + ({ events, cursor } = details as { events?: unknown; cursor?: unknown }); + } else { return null; } - const { events, cursor } = details as { events?: unknown; cursor?: unknown }; if (!Array.isArray(events) || events.length === 0) { return null; } @@ -874,6 +1011,163 @@ export type EventCreator = ( params?: CreateEventParams ) => Promise; +/** + * The concurrency fence a replay-context event creation carries. Exactly one of + * the two schemes is ever populated: the precondition snapshot for a run guarded + * by the event-log watermark, `eventId`/`maxSlot` for a run that numbers its + * events by slot. + */ +export interface EventCreateFence extends PreconditionSnapshotParams { + eventId?: string; + maxSlot?: number; +} + +/** + * The fence to attach to a replay-context event creation, under whichever + * scheme the run uses. + * + * Neither scheme is retried in place. A rejection under either one proves the + * replay derived this event from a log that was missing another, and correlation + * ids are positional ordinals of one sequence — so a replay over the corrected + * log mints different ids and re-posting this write would persist an event no + * correct replay produces. Recovery is a restarted replay + * ({@link isStaleWriteRejection}), never a re-send. + * + * Claims a slot off `log` for a slot-numbered run — which counts as a + * reservation, so a caller that fences several creates from one log gets a + * distinct slot per create. Empty when the run is fenced neither way, leaving + * the create exactly as unfenced as it was before either mechanism existed. + * + * `extraEvents` is how many events *besides* the one being created this write + * publishes: a lazy inline `step_started` also materializes the `step_created` + * it deferred. Those events take the slots immediately below the claim, so this + * reserves them too and names the top one — a World that writes a pair + * derives the lower id from the one it was given. + * + * The reservation has to happen here rather than at the World or its backend. + * Slots are handed out for a whole concurrent batch synchronously, before any of + * it lands, so a second event numbered off the log as the backend sees it would + * take the slot already promised to the next write in the batch — and every + * write after the first in a fan-out would lose its claim. + */ +export function eventCreateFenceFor( + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): EventCreateFence { + if (usesSlotIdentity(specVersion)) { + return reserveSlotFence(log, options?.extraEvents ?? 0); + } + return preconditionSnapshotParams(log.events, log.cursor); +} + +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): EventCreateFence { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; +} + +/** + * Runs one slot-numbered create with the log's claim to itself, taking its slot + * only once every create ahead of it on the log has settled. + * + * A slot claim is an assertion about the tail: "nothing has been published + * since the view I decided from". Claims handed out up front to a concurrent + * batch can only assert that about the first of them — the rest sit above slots + * their own siblings have yet to fill, so a foreign event landing in that space + * satisfies their fences too and they commit on a view that is already missing + * it. Taking claims one at a time keeps every write's fence tight against the + * tail the writer actually saw. + * + * A rejection therefore stops the whole batch rather than only its own write: + * the log's tail stops advancing while the backend's moves on, so the claims + * behind it fall inside the occupied range and are rejected in turn. That is + * the intent — the batch was decided from a log missing an event, so none of it + * should land. + */ +async function withSerializedClaim( + log: MutableEventLog, + extraEvents: number, + op: (fence: EventCreateFence) => Promise +): Promise { + const ahead = log.writeChain; + let done!: () => void; + log.writeChain = new Promise((resolve) => { + done = resolve; + }); + if (ahead) { + await ahead; + } + try { + const fence = reserveSlotFence(log, extraEvents); + const result = await op(fence); + log.maxSlot = Math.max( + log.maxSlot, + maxSlotOf([{ eventId: fence.eventId ?? '' }]) + ); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); + return result; + } catch (error) { + // The slots this attempt drew are not the writer's, and the tail is at + // least as high as the claim that lost. Rewinding onto the occupied range + // is what makes the rest of the batch fail with it. + log.nextSlot = log.maxSlot + 1; + throw error; + } finally { + done(); + } +} + +/** + * Runs one event create under whichever fence its run uses. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fenced create for a claim issued from a concurrent batch — an inline + * step's `step_started`, or a suspension flush's writes. + * + * Under slot numbering this serializes the batch's claims so each one is taken + * against the tail its writer actually saw ({@link withSerializedClaim}); under + * the watermark every claim in the batch legitimately carries the same + * snapshot, so they all run concurrently off one fence. + * + * Neither scheme re-issues a rejected claim at a free number. A rejection says + * this replay decided from a log it had not fully seen, and the missing event + * can be the one that would have sent the workflow down another branch — with + * correlation ids counted in branch order, every id after that branch moves + * with it, so the re-issued write would land under an identity that now names a + * different step. The rejection propagates and the run replays over the + * corrected log ({@link isStaleWriteRejection}). + */ +export function claimFenceFor( + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + return (op) => withSerializedClaim(log, options?.extraEvents ?? 0, op); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index fefa5a0bcf..3a879bc8d1 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -10,7 +10,10 @@ * 2. The restart reloads the whole event log with no cursor, because a hole is * defined by ULID time while a cursor filters lexicographically — unless * the World attached the missing events to the 412, which the runtime - * consumes with no events.list round trip at all (first restart only). + * consumes with no events.list round trip at all (first restart only). A run + * numbering its events by slot instead heals from its cursor, since slot ids + * sort in write order and density proves afterwards that the page closed the + * gap; a short count sends it back to the full reload. * 3. Restarts are bounded; once the bound is spent the runtime schedules a * delayed re-invocation instead of failing the run — and that escalation is * itself counted on the queue message, so a run that can never observe its @@ -19,11 +22,19 @@ * Modeled on wait-completion-replay.test.ts, but with real ULID event IDs so * latestEventStateUpdatedAt() actually derives snapshot times. */ -import { PreconditionFailedError, RUN_ERROR_CODES } from '@workflow/errors'; +import { + PreconditionFailedError, + RUN_ERROR_CODES, + SlotConflictError, +} from '@workflow/errors'; import { type CreateEventRequest, type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, + slotIdBody, type WorkflowRun, type World, } from '@workflow/world'; @@ -42,6 +53,7 @@ import { getPreconditionMaxInProcessRestarts, getPreconditionMaxReinvocations, getPreconditionReinvokeDelaySeconds, + PRECONDITION_MAX_INPROCESS_RESTARTS, } from './constants.js'; import { setWorld } from './world.js'; @@ -88,6 +100,8 @@ interface SnapshotParams { stateUpdatedAt: number | undefined; stateEventCount: number | undefined; stateCursor: string | undefined; + /** The claimed position, on a run that numbers its events by slot. */ + eventId?: string | undefined; } async function runPreconditionScenario(options: { @@ -98,6 +112,17 @@ async function runPreconditionScenario(options: { * or a payload the runtime must refuse to narrow (`malformed`). */ attachDelta?: 'complete' | 'malformed'; + /** + * Number the run's events by slot rather than by ULID, and reject with the + * 409 a taken slot produces instead of the 412. + */ + slotIdentity?: boolean; + /** + * Slot mode only: land a second out-of-band event ahead of the hook and hide + * it from the page above the cursor, so a restart that tops up incrementally + * ends holding a log that is short of its own highest slot. + */ + hideFirstOutsideEventFromCursor?: boolean; }) { vi.spyOn(Date, 'now').mockReturnValue(+fixedNow); @@ -112,11 +137,17 @@ async function runPreconditionScenario(options: { undefined ); + const SPEC = options.slotIdentity + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; + const { globalThis: vmGlobalThis } = createContext({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); + // Correlation ids are ULIDs in both modes: slot identity numbers event ids + // only. const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; const syncStep0CorrelationId = `step_${vmUlid(+startedAt)}`; const waitCorrelationId = `wait_${vmUlid(+startedAt)}`; @@ -127,23 +158,35 @@ async function runPreconditionScenario(options: { status: 'running', input: workflowArgs, deploymentId, - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, startedAt, createdAt: startedAt, updatedAt: startedAt, }; // Real ULID event IDs at controlled times so latestEventStateUpdatedAt() - // resolves an actual epoch-ms snapshot from the loaded log. + // resolves an actual epoch-ms snapshot from the loaded log. Under slot + // identity the id is instead the next free slot, or the one the writer + // claimed, and the log stays dense from slot 1. const hostUlid = monotonicFactory(); let eventIndex = 0; - const event = (data: CreateEventRequest, atMs?: number): Event => { + let nextSlot = FIRST_SLOT; + const event = ( + data: CreateEventRequest, + atMs?: number, + claimedEventId?: string + ): Event => { const t = atMs ?? +startedAt + ++eventIndex * 100; + let eventId = `evnt_${hostUlid(t)}`; + if (options.slotIdentity) { + eventId = claimedEventId ?? `evnt_${slotIdBody(nextSlot)}`; + nextSlot = Math.max(nextSlot, slotFromId(eventId) ?? nextSlot) + 1; + } return { ...data, - specVersion: data.specVersion ?? SPEC_VERSION_CURRENT, + specVersion: data.specVersion ?? SPEC, runId, - eventId: `evnt_${hostUlid(t)}`, + eventId, createdAt: new Date(t), } as Event; }; @@ -151,19 +194,19 @@ async function runPreconditionScenario(options: { const staleEvents: Event[] = [ event({ eventType: 'run_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, eventData: { deploymentId, workflowName, input: workflowArgs }, }), - event({ eventType: 'run_started', specVersion: SPEC_VERSION_CURRENT }), + event({ eventType: 'run_started', specVersion: SPEC }), event({ eventType: 'hook_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { token: hookToken }, }), event({ eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { stepName: 'syncStep', @@ -176,12 +219,12 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, }), event({ eventType: 'step_completed', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { result: await dehydrateStepReturnValue(undefined, runId, undefined), @@ -189,7 +232,7 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'wait_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: waitCorrelationId, eventData: { resumeAt: new Date(+startedAt - 1_000) }, }), @@ -199,10 +242,30 @@ async function runPreconditionScenario(options: { const staleEventsCursor = 'cursor-after-stale-events'; const OUTSIDE_EVENT_MS = +startedAt + 5_000; + // Lands ahead of the winning delivery and is withheld from the page above + // the cursor, so an incremental top-up comes back holding fewer events than + // its own highest slot names. + const hiddenOutsideEvent = options.hideFirstOutsideEventFromCursor + ? event( + { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: hookCorrelationId, + eventData: { + payload: await dehydrateStepReturnValue( + { value: 'hook-poke' }, + runId, + undefined + ), + }, + }, + OUTSIDE_EVENT_MS - 1 + ) + : undefined; const hookReceivedEvent = event( { eventType: 'hook_received', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { payload: await dehydrateStepReturnValue( @@ -233,7 +296,9 @@ async function runPreconditionScenario(options: { }) => { const data = params.pagination?.cursor === staleEventsCursor - ? durableEvents.slice(staleEvents.length) + ? durableEvents + .slice(staleEvents.length) + .filter((e) => e !== hiddenOutsideEvent) : [...durableEvents]; return { data, @@ -262,6 +327,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt?: number; stateEventCount?: number; stateCursor?: string; + eventId?: string; } ) => { createParams.push({ @@ -269,6 +335,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt: params?.stateUpdatedAt, stateEventCount: params?.stateEventCount, stateCursor: params?.stateCursor, + eventId: params?.eventId, }); if (request.eventType === 'run_started') { @@ -278,11 +345,32 @@ async function runPreconditionScenario(options: { if (request.eventType === 'wait_completed') { // The out-of-band hook payload becomes durable just before the // wait_completed commit — this is the exact race the guard closes. + if (hiddenOutsideEvent && !durableEvents.includes(hiddenOutsideEvent)) { + durableEvents.push(hiddenOutsideEvent); + } if (!durableEvents.includes(hookReceivedEvent)) { durableEvents.push(hookReceivedEvent); } if (waitCompletedRejections < (options.rejectWaitCompletedTimes ?? 0)) { waitCompletedRejections++; + // Both rejections say the same thing — "you decided from a log you + // had not fully seen" — and differ only in how the writer found out: + // a watermark comparison, or the slot it claimed already being + // occupied by the event it was missing. + if (options.slotIdentity) { + throw new SlotConflictError( + `Event ${params?.eventId} is already taken.`, + { + eventId: params?.eventId ?? '', + events: + options.attachDelta === 'complete' + ? [hookReceivedEvent] + : undefined, + cursor: + options.attachDelta === 'complete' ? 'cursor-409' : undefined, + } + ); + } throw new PreconditionFailedError( 'Run state is stale: the client event log is missing at least one event at or before its snapshot.', options.attachDelta === undefined @@ -303,17 +391,30 @@ async function runPreconditionScenario(options: { !!request.eventData && (request.eventData as { input?: unknown }).input !== undefined; let effectiveRequest = request; + // A claim covers the whole write, so a lazy start's synthesized + // step_created takes the claimed position and the step_started the one + // after it — the two extra slots the caller reserved for exactly this. + let claimedEventId = params?.eventId; + const takeClaim = () => { + const claim = claimedEventId; + claimedEventId = undefined; + return claim; + }; if (lazyStepStart) { const lazyData = request.eventData as { stepName?: string; input?: unknown; }; - const syntheticStepCreated = event({ - eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: request.correlationId, - eventData: { stepName: lazyData.stepName, input: lazyData.input }, - } as CreateEventRequest); + const syntheticStepCreated = event( + { + eventType: 'step_created', + specVersion: SPEC, + correlationId: request.correlationId, + eventData: { stepName: lazyData.stepName, input: lazyData.input }, + } as CreateEventRequest, + undefined, + takeClaim() + ); durableEvents.push(syntheticStepCreated); createdEvents.push(syntheticStepCreated); const { input: _strippedInput, ...startEventData } = lazyData; @@ -323,7 +424,7 @@ async function runPreconditionScenario(options: { } as CreateEventRequest; } - const created = event(effectiveRequest); + const created = event(effectiveRequest, undefined, takeClaim()); durableEvents.push(created); createdEvents.push(created); if (effectiveRequest.eventType === 'step_started') { @@ -343,7 +444,7 @@ async function runPreconditionScenario(options: { const queue = vi.fn().mockResolvedValue({ messageId: 'msg_step' }); const fakeWorld = { - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, createQueueHandler: vi.fn((_prefix, handler) => { capturedHandler = handler; return vi.fn(); @@ -1051,6 +1152,89 @@ describe('precondition guard through the real replay loop', () => { ); }); + it('heals a slot-numbered log from its cursor instead of reloading it whole', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + }); + await result.handlerInvocation; + + expect(result.waitCompletedRejectionCount()).toBe(1); + // Slot ids sort in write order, so the event the snapshot was missing is + // strictly above the cursor and one incremental page brings it in. The + // full reload the ULID path needs exists only because a hole defined by + // ULID time can sort below the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + + const waitCreates = result.createParams.filter( + (c) => c.eventType === 'wait_completed' + ); + expect(waitCreates).toHaveLength(2); + // No watermark is sent on a run that fences by position. + expect(waitCreates[0]?.stateUpdatedAt).toBeUndefined(); + // The rejected claim sat at the position the hook had just taken; the + // restarted replay claims the one after it. + expect(slotFromId(waitCreates[1]?.eventId ?? '')).toBe( + (slotFromId(waitCreates[0]?.eventId ?? '') ?? 0) + 1 + ); + + // Replay after the restart observed the hook and took the hook branch. + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + + it('falls back to the full reload when a slot top-up leaves the log short of its own highest slot', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + hideFirstOutsideEventFromCursor: true, + }); + await result.handlerInvocation; + + // A slot log is dense from slot 1, so a complete one holds exactly + // `maxSlot` events. The page above the cursor withheld one, the count came + // up short, and the restart fell through to the authoritative load — the + // check is what lets the cheap path be taken in the first place. + expect(cursorlessLoads(result.listEvents)).toBe(1); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(2); + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + + it('spends a larger in-process restart budget when restarts heal from the cursor', async () => { + // One more rejection than a reloading run is allowed to absorb. That run + // would be out of budget and re-invoked; this one keeps recovering in + // process, because each of its restarts costs one incremental page rather + // than a full reload and so is not worth a queue hop to avoid. + const rejections = PRECONDITION_MAX_INPROCESS_RESTARTS + 1; + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: rejections, + }); + + await expect(result.handlerInvocation).resolves.toBeUndefined(); + expect(result.waitCompletedRejectionCount()).toBe(rejections); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(rejections + 1); + // Every one of them healed from the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + }); + it('falls back to the full reload when the 412 payload does not narrow to events', async () => { const result = await runPreconditionScenario({ rejectWaitCompletedTimes: 1, diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 8904600556..9d96422c7c 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 4bbe37fa56..a0e9e615a3 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -1,6 +1,7 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; import type { Event, World } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { createWorld } from '@workflow/world-local'; @@ -171,7 +172,7 @@ describe('executeStep — compute instance stamping', () => { counter += 1; }); - it('stamps computeInstanceId on step_started without displacing the precondition snapshot', async () => { + it('stamps computeInstanceId on step_started without displacing the claim fence', async () => { const world = makeWorld(); const stepName = uniqueStepName(); const { runId, stepId } = await setupRunningStep({ @@ -197,7 +198,7 @@ describe('executeStep — compute instance stamping', () => { workflowStartedAt: Date.now(), stepId, stepName, - preconditionSnapshot, + claimFence: (op) => op(preconditionSnapshot), }); const started = createSpy.mock.calls.filter( @@ -210,3 +211,153 @@ describe('executeStep — compute instance stamping', () => { expect(started[0]?.[2]).toMatchObject(preconditionSnapshot); }); }); + +// A run that numbers its events by position rejects a claim whose slot was +// taken, and the rejection carries the events that took it. When those events +// show the same step already started, the loser is in the ordinary "another +// handler owns this step" position and skips — the outcome an unfenced write +// reaches via EntityConflictError. Anything less than proof of the same call +// must propagate instead, because correlation ids are positional: a replay +// that diverged can reach the same step number naming a different call. +describe('executeStep — slot rejection carrying a duplicate start', () => { + afterEach(() => { + counter += 1; + }); + + function startedEvent(opts: { + stepId: string; + stepName: string; + input?: unknown; + }) { + return { + eventType: 'step_started', + correlationId: opts.stepId, + eventData: { + stepName: opts.stepName, + ...(opts.input !== undefined ? { input: opts.input } : {}), + }, + }; + } + + async function runAgainstRejection(opts: { + events: unknown[]; + lazyStepInput?: Uint8Array; + onBody?: () => void; + }) { + const world = makeWorld(); + const stepName = uniqueStepName(); + let bodyRuns = 0; + const { runId, stepId } = await setupRunningStep({ + world, + stepName, + onBody: () => { + bodyRuns += 1; + opts.onBody?.(); + }, + }); + + const rejection = new SlotConflictError('slot taken', { + eventId: 'evnt_00000000000000000000000007', + events: opts.events.map((build) => + typeof build === 'function' + ? (build as (ids: { stepId: string; stepName: string }) => unknown)({ + stepId, + stepName, + }) + : build + ), + }); + + const run = () => + executeStep({ + world, + workflowRunId: runId, + workflowName: 'wf', + workflowStartedAt: Date.now(), + stepId, + stepName, + lazyStepInput: opts.lazyStepInput, + // Take the awaited claim path so the rejection is translated before a + // body ever runs; the optimistic path reconciles the same way but + // would run the body first and muddy the assertion. + suppressOptimisticStart: true, + claimFence: () => Promise.reject(rejection), + }); + + return { run, bodyRuns: () => bodyRuns, stepName, stepId }; + } + + it('skips when the delta already started this step (no lazy input to compare)', async () => { + const { run, bodyRuns } = await runAgainstRejection({ + events: [ + (ids: { stepId: string; stepName: string }) => startedEvent(ids), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('skips when the delta started this step with byte-identical input', async () => { + const input = new Uint8Array([1, 2, 3]); + const { run, bodyRuns } = await runAgainstRejection({ + lazyStepInput: input, + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 3]) }), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('propagates when the delta started the same slot under a different step name', async () => { + const { run } = await runAgainstRejection({ + events: [ + (ids: { stepId: string }) => + startedEvent({ ...ids, stepName: 'step//./other//someOtherStep' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta started this step with different input bytes', async () => { + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 4]) }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta start carries a remote ref instead of inline bytes', async () => { + // A ref says nothing about the value behind it, so identity is unprovable + // and the replay restart is the only correct answer. + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: { ref: 'payload_abc' } }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta holds no start for this step', async () => { + const { run } = await runAgainstRejection({ + events: [ + { eventType: 'step_completed', correlationId: 'step_other' }, + (ids: { stepName: string }) => + startedEvent({ ...ids, stepId: 'step_other' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); +}); diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index c412271a2a..dd8a36d2d6 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -4,6 +4,7 @@ import { FatalError, RetryableError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowRuntimeError, @@ -51,9 +52,10 @@ import { } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; import { + type EventCreateFence, type EventCreator, + type FencedCreate, memoizeEncryptionKey, - type PreconditionSnapshotParams, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; import { @@ -86,6 +88,73 @@ function extractInlineDelta( }; } +/** + * Byte-equality for two dehydrated step inputs, used to decide whether a + * conflicting `step_started` describes the same call as ours. Only inline + * bytes can answer that: a `SerializedData` that came back as a remote ref + * carries no payload, and two refs being unequal says nothing about the + * values behind them, so anything that is not a pair of `Uint8Array`s is + * reported as "cannot tell". + */ +function sameSerializedInput( + a: SerializedData | undefined, + b: SerializedData | undefined +): boolean { + if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false; + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} + +/** + * Decide whether a slot rejection carrying a delta is a *benign duplicate*: + * the events we lost the slot to already contain a `step_started` for this + * exact step, so a concurrent handler is running (or has run) the very call + * we were about to claim. The step is then someone else's to finish and we + * can skip, exactly as we do on the `EntityConflictError` the unfenced path + * would have raised instead. + * + * The identity test is deliberately strict. Correlation ids are positional + * ordinals of one seeded sequence, so a replay that diverged — a + * `Promise.race` between a hook and a step resolving the other way, say — can + * arrive at the same correlation id naming a different call. Matching on it + * alone would let such a replay adopt a foreign step's completion as its own. So + * the step name must match, and on the lazy path (where we hold the input) + * the inputs must be byte-identical. Anything we cannot prove identical + * returns false and takes the replay restart, which is always correct and + * merely slower. + */ +function isBenignDuplicateStart( + events: unknown[], + stepId: string, + stepName: string, + lazyStepInput: SerializedData | undefined +): boolean { + for (const candidate of events) { + if (!candidate || typeof candidate !== 'object') continue; + const event = candidate as { + eventType?: unknown; + correlationId?: unknown; + eventData?: { stepName?: unknown; input?: unknown } | null; + }; + if (event.eventType !== 'step_started') continue; + if (event.correlationId !== stepId) continue; + if (event.eventData?.stepName !== stepName) continue; + if (lazyStepInput === undefined) return true; + if ( + sameSerializedInput( + lazyStepInput, + event.eventData?.input as SerializedData + ) + ) { + return true; + } + } + return false; +} + export interface StepExecutorParams { world: World; workflowRunId: string; @@ -141,20 +210,27 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Precondition-guard snapshot of the event log the caller's replay loaded, to - * attach to this step's `step_started` claim. On the lazy inline path the - * claim is the step's FIRST durable write (its `step_created` is deferred), - * so without this the claim would bypass the optimistic-concurrency guard - * entirely: a replay working from a stale view could claim — and then commit - * — a step scheduled without observing an event it never loaded. A - * guard-enforcing World rejects a stale claim with `PreconditionFailedError` - * (412); executeStep does NOT translate that rejection (re-claiming in place - * would still commit the stale schedule), so it propagates for the caller to - * abandon the batch and restart its replay. Undefined when the guard is - * disabled or the caller has no snapshot; Worlds that don't enforce the guard - * ignore it. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies, or the caller's replay snapshot + * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the + * older numbering. + * + * On the lazy inline path the claim is the step's FIRST durable write (its + * `step_created` is deferred), so without a fence it would be unguarded + * entirely: a replay working from a stale view could claim — and then commit — + * a step scheduled without observing an out-of-band event. A fencing World + * rejects such a claim with `SlotConflictError` (409) or + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, made per + * scheme — see `claimFenceFor`. Either way executeStep does NOT translate a + * rejection that reaches it, so an unretried one propagates for the caller to + * abandon the batch and force a fresh replay. + * + * Undefined when the caller has no snapshot, or when the watermark guard is + * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - preconditionSnapshot?: PreconditionSnapshotParams; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -268,6 +344,10 @@ export async function executeStep( // put us on the Vercel branch with nothing to build a host from, making // `https://` the base URL of every step. const isVercel = Boolean(process.env.VERCEL_URL); + // Unfenced when the caller passes no fence — every World that fences + // ignores the field it does not understand, so this is the same create it + // was before either mechanism existed. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -488,6 +568,37 @@ export async function executeStep( }); return { type: 'skipped' }; } + if ( + SlotConflictError.is(err) && + isBenignDuplicateStart( + err.events, + stepId, + stepName, + params.lazyStepInput + ) + ) { + // We lost the slot to a writer that had already started this same + // step, so this is the ordinary "another handler owns it" outcome + // wearing a slot rejection — the same thing an unfenced write reports + // as EntityConflictError. Skip rather than restart the replay: the + // restart would re-derive this identical claim and lose again. + // + // The delta on the error is deliberately NOT merged into the log. + // Adopting events this VM never replayed would let the next claim + // fence itself against a tail it cannot account for, which is the + // one thing the fence exists to prevent. Siblings in the batch fail + // with their own rejection and the handler defers as usual. + runtimeLogger.debug('Step already started by a concurrent writer', { + stepName, + stepId, + workflowRunId, + }); + span?.setAttributes({ + ...Attribute.StepSkipped(true), + ...Attribute.StepSkipReason('completed'), + }); + return { type: 'skipped' }; + } if (TooEarlyError.is(err)) { const timeoutSeconds = Math.max(1, err.retryAfter ?? 1); runtimeLogger.debug('Step retryAfter timestamp not yet reached', { @@ -529,13 +640,15 @@ export async function executeStep( let step: Step; // Params for the `step_started` create on either path below: the ambient - // compute-instance stamp plus the optimistic-concurrency claim guard. - const startEventParams: CreateEventParams = { + // compute-instance stamp plus whichever fence the claim is running under. + const startEventParams = ( + fence: EventCreateFence | undefined + ): CreateEventParams => ({ computeInstanceId: COMPUTE_INSTANCE_ID, - // Spread as a unit: the three snapshot fields describe one snapshot and - // must travel together — see StepExecutorParams.preconditionSnapshot. - ...params.preconditionSnapshot, - }; + // Spread as a unit: the fence's fields describe one fence and must + // travel together — see StepExecutorParams.claimFence. + ...fence, + }); // `Date.now()` taken immediately before the `step_started` create is // issued (either path below) — anchors RSFS's end point. See // StepLatencyEventData.rsfs and the call sites below. @@ -575,26 +688,29 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart: + // the body result is discarded, and unless the rejection proves + // another writer already started this same step (a benign + // duplicate, skipped) it propagates to the caller. + return runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Guard the claim — see StepExecutorParams.preconditionSnapshot. A - // stale (412) rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - startEventParams + startEventParams(fence) + ) ); } ); @@ -635,26 +751,28 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Guard the claim — see StepExecutorParams.preconditionSnapshot. A - // stale (412) rejection is intentionally NOT translated by - // startErrorToResult below, so it propagates to the caller for a - // fresh replay. - startEventParams + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry propagates to the caller for a fresh replay, + // except where startErrorToResult below can read the rejection's delta + // as "another writer already started this exact step" and skip. + const startResult = await runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + startEventParams(fence) + ) ); if (!startResult.step) { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 8cc8dbf208..e167d83b05 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -30,9 +30,9 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { + claimFenceFor, type EventCreator, - type LoadedEventLog, - preconditionSnapshotParams, + type MutableEventLog, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; @@ -44,14 +44,15 @@ export interface SuspensionHandlerParams { requestId?: string; /** * The runtime's loaded event log. Every event creation this suspension makes - * is sent with the precondition snapshot derived from it, so a backend that - * has recorded an event the replay did not see rejects the write with a 412 - * instead of accepting a divergent event. The rejection is not retried here: - * the event's correlation id was minted by *this* replay's seeded sequence, - * so re-committing it against a corrected log would persist an event no - * correct replay produces. The caller restarts the replay instead. + * carries a fence derived from it — its own event slot, or the snapshot's + * watermark for a run on the older numbering — so a backend that has recorded + * an event the replay did not see rejects the write (409/412) instead of + * accepting a divergent event. The rejection is not retried here: the event's + * correlation id was minted by *this* replay's seeded sequence, so + * re-committing it against a corrected log would persist an event no correct + * replay produces. The caller restarts the replay instead. */ - eventLog?: LoadedEventLog; + eventLog?: MutableEventLog; /** * Turbo mode only: a promise that resolves once the backgrounded * `run_started` has landed (the run exists). When present, every world write @@ -268,19 +269,19 @@ export async function handleSuspension({ reporter.withEventCreate(params, (p) => world.events.create(runId, data, p) ); - // Adds the optimistic-concurrency guard when the caller supplied a loaded - // event log; without one it creates directly (callers with no replay - // snapshot, e.g. tests). A stale (412) rejection propagates to the caller, - // which restarts the replay from a corrected log — it is not retried here, - // because the event's correlation id was minted by *this* replay's seeded - // sequence, so re-committing it against a corrected log would persist an - // event no correct replay produces. + // Fences the create against the run's event log when the caller supplied a + // loaded one; without it the create goes out unfenced (callers with no replay + // snapshot, e.g. tests). A rejection propagates to the caller, which restarts + // the replay from a corrected log — it is not retried here, because the + // event's correlation id was minted by *this* replay's seeded sequence, so + // re-committing it against a corrected log would persist an event no correct + // replay produces. const createGuarded: EventCreator = (data, params) => eventLog - ? createEvent(data, { - ...params, - ...preconditionSnapshotParams(eventLog.events, eventLog.cursor), - }) + ? claimFenceFor( + eventLog, + run.specVersion + )((fence) => createEvent(data, { ...params, ...fence })) : createEvent(data, params); // Separate queue items by type const stepItems = suspension.steps.filter( diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 6c09703327..81acb1fe95 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -14,9 +14,13 @@ import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -73,7 +77,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -100,6 +112,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -136,7 +149,13 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain, whose writes have to be ordered with the caller's terminal write — + * see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -195,22 +214,34 @@ export async function runWorkflow( // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; workflowDiscontinuation.reject( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, @@ -855,7 +886,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + eventLog ); return dehydrated; @@ -872,7 +904,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + eventLog ); throw err; diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 0cc4a950e0..7fdbaaf8cd 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -23,7 +23,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -42,7 +45,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), @@ -466,6 +469,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index d5d5bed8ec..5114eeeed7 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -175,8 +175,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -192,6 +193,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -457,8 +483,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -578,8 +606,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index e552e8e28a..0c547c00aa 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -821,6 +821,59 @@ export class PreconditionFailedError extends WorkflowWorldError { } } +/** + * Thrown when the backend rejects an event creation because the event slot the + * client named was already taken by another writer (HTTP 409). + * + * On a run that numbers its events by slot, whoever writes a slot first owns + * it, and the loser has by definition been replaying against an event log + * missing at least one event. Retrying the same write can therefore never + * succeed: the client has to merge the events it was missing, replay, and + * propose whatever slot that replay lands on. The rejection carries those + * events inline so the common case costs no extra round-trip. + * + * Distinct from `PreconditionFailedError` (412), which is the equivalent + * rejection for a run guarded by the `stateUpdatedAt` watermark instead. Both + * mechanisms are live at once while runs on the older numbering drain. + * + * The workflow runtime handles this automatically. Users interacting with world + * storage backends directly may encounter it. + * + * @property eventId - The slot-numbered event id that was already taken. + * @property events - The events recorded after the client's snapshot, in + * ascending slot order. Empty when the backend could not read them, in which + * case the client reloads the log itself. + * @property cursor - Cursor to continue the delta from, or `null`. + * @property hasMore - Whether events beyond `events` remain to be fetched. + */ +export class SlotConflictError extends WorkflowWorldError { + readonly eventId: string; + readonly events: unknown[]; + readonly cursor: string | null; + readonly hasMore: boolean; + + constructor( + message: string, + options: { + eventId: string; + events?: unknown[]; + cursor?: string | null; + hasMore?: boolean; + } + ) { + super(message, { status: 409 }); + this.name = 'SlotConflictError'; + this.eventId = options.eventId; + this.events = options.events ?? []; + this.cursor = options.cursor ?? null; + this.hasMore = options.hasMore ?? false; + } + + static is(value: unknown): value is SlotConflictError { + return isError(value) && value.name === 'SlotConflictError'; + } +} + /** * Thrown when awaiting `run.returnValue` on a workflow run that was cancelled. * diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 4490e7ed4f..9e0cdb3774 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -5,6 +5,7 @@ export { PreconditionFailedError, RunExpiredError, RunNotSupportedError, + SlotConflictError, StepNotRegisteredError, ThrottleError, TooEarlyError, diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 70178b344c..8d2ec5d203 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -466,6 +466,11 @@ export async function deleteJSON(filePath: string): Promise { await withWindowsRetry(() => fs.unlink(filePath)); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } @@ -579,6 +584,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * A slot-numbered event log orders by slot — the position is the order — and + * a writer that loses a slot re-proposes above the winner while keeping the + * stamp it started with, so `createdAt` there disagrees with the log. Such an + * item reports one shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -615,6 +628,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -718,7 +732,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -746,8 +760,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -768,7 +782,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 598cceb93f..3c7c0beefe 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,10 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. + specVersion: mintedSpecVersion(), capabilities: { // world-local deduplicates concurrent `hook_received` writes sharing a // `(runId, resumeId)` via a filesystem sidecar claim (see diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index d161d1ef8d..b29d9a8a29 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -5,11 +5,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -33,9 +35,15 @@ import { isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, + SLOT_RETRY_BUDGET_MS, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, + slotRetryDelay, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -86,6 +94,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -123,6 +132,20 @@ function getMaxEventsPerRun(): number { // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. +/** + * The time an event orders and paginates by. A slot-numbered run's order is its + * slot order — the position *is* the order, the way the sort key is for the + * other backends — so every such event reports the same time and lets the + * event-id tie-break do the ordering. Ordering those by `createdAt` reads the + * log in an order no replay produced: a writer that loses a slot re-proposes + * above the winner while keeping the stamp it started with, and a caller that + * reserves slots for a whole flush commits them in whatever order the network + * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree + * with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + const HookTokenClaimSchema = z.object({ // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed @@ -277,6 +300,7 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; @@ -524,6 +548,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -543,6 +569,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -550,6 +577,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -610,6 +638,65 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -661,6 +748,58 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A slot-numbered create reserves its position before running the + // validation and materialization that may still reject it. Handing the + // reservation back on the way out is what keeps the log dense: an + // abandoned slot below a sibling's published one is a hole that can never + // be filled, and a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ + let eventCommitted = false; + /** + * Hands the slots of a create that never published back to the allocator, + * so an abandoned reservation below a sibling's published slot does not + * become a hole the run can never fill. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } + throw error; + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -671,7 +810,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -700,9 +841,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -737,6 +880,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -818,8 +973,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -838,6 +999,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -854,6 +1016,18 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and a ULID names no position: the replay + // would read it wherever its timestamp happens to sort rather than where + // the writer meant it to go. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -875,7 +1049,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -891,20 +1065,109 @@ export function createEventsStorage( } } - // ============================================================ - // VALIDATION: Terminal state and event ordering checks - // ============================================================ - // Lazy step start: a step_started carrying step-creation data // (stepName + input) is allowed to arrive with no prior step_created // — it creates the step on the fly (see the materialization block // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. const createsChildEntity = isChildEntityCreationEvent(data); const lazyStepStart = createsChildEntity && data.eventType === 'step_started'; + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so it takes that position outright instead of + // allocating one, and every other event allocates above it even when it + // is the first to arrive here. Allocation is append-only (see SlotBook), + // so `run_created` cannot get the first position by asking for the lowest + // free one: a `run_started` racing it (start() issues the creation and + // the queue send in parallel) may already have moved the book past it. + const ownsFirstSlot = data.eventType === 'run_created'; + // A slot-numbered run's ids name positions in its log, so an id is + // either claimed by a caller that holds the log (and is therefore + // asserting the log is complete up to that position) or allocated here + // for a caller that has no log — a step completion reporting in, a + // cancellation from an API call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // A claim has to clear the log's tail, not merely be free: a + // position left unwritten by an abandoned reservation stays empty + // for good, and a caller numbering from a stale snapshot aims + // straight at it, landing an event below events another replay + // already consumed. Rejected before the materialization below + // creates the step, hook or wait this event will now never + // accompany. See SlotBook.highestWritten. + const tail = await slots.highestWritten(effectiveRunId); + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (slot <= tail) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + let slot: number; + if (ownsFirstSlot) { + slot = RUN_CREATED_SLOT; + slots.claim(effectiveRunId, slot); + } else { + slot = await slots.reserve(effectiveRunId); + } + reserved.add(slot); + eventId = slotEventId(slot); + } + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1289,13 +1552,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1578,6 +1868,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1599,10 +1890,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -1641,6 +1936,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1656,24 +1952,35 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1686,15 +1993,39 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -2299,6 +2630,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2312,10 +2644,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2377,11 +2713,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2397,115 +2739,178 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + // `run_created` is excluded even though it named its own position: the + // first slot is the only position it can ever occupy, so losing it means + // the run already has a creation event, and appending a second one above + // it would be worse than the duplicate the publish is reporting. + const reallocatesSlot = + slotMode && params?.eventId === undefined && !ownsFirstSlot; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again above whatever the log has reached rather than surfacing a + // conflict the caller cannot act on. Re-reading rather than + // incrementing bounds the search: every round at least one writer + // wins, so the top of the log is never further than the number of + // writers still contending for it. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2526,14 +2931,54 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + if (slotMode) { + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` @@ -2571,6 +3016,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2619,6 +3065,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2679,6 +3126,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2711,6 +3159,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index ffc4b81802..77d9781db6 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { decodeTime, monotonicFactory } from 'ulid'; import { hasTag, @@ -208,6 +209,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -229,38 +270,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * A slot-numbered run takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -279,6 +317,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..f1ee5df07b --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,484 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('rejects a claim on a free position below the log’s tail', async () => { + // The undercut that a "is the position free?" check cannot catch. A + // position claimed by a write that then failed is never filled, so a log + // carries holes below its tail — and a caller numbering from a snapshot + // that predates the events above one of those holes aims straight at it. + // Let it land and the event sits below events another replay has already + // consumed: the log stays internally consistent while its order silently + // changes, which is enough to flip a race between a step and a sleep from + // one replay to the next. A claim asserts a complete log, so a claim that + // does not clear the tail is a conflict, exactly as a taken one is. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await expect( + createStep(runId, 'step_early', slotEventId(2)) + ).rejects.toThrow(SlotConflictError); + // The hole stays a hole, and the log stays in slot order. + await expect(slotsOf(runId)).resolves.toEqual([1, 3]); + }); + + it('accepts the claim immediately above a tail with a hole below it', async () => { + // The fence rejects at-or-below, so the first position above the tail has + // to stay writable — otherwise every write following a hole would conflict + // forever and the run could never make progress again. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + expect(await createStep(runId, 'step_next', slotEventId(4))).toBe( + slotEventId(4) + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 3, 4]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('numbers a run densely when every write it makes lands', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('leaves a rejected write’s position unused instead of recycling it', async () => { + // The rejected op's position sits below its concurrent sibling's, so handing + // it to the next writer would order that writer's event below one that + // already published. The hole costs a reader nothing it was promised; the + // inversion would cost the run. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + const slots = await slotsOf(runId); + expect(slots).toHaveLength(3); + expect(slots[0]).toBe(FIRST_SLOT); + // Both concurrent writers took a position, one abandoned its own, and the + // third write went above them both. + expect(slots[2]).toBe(FIRST_SLOT + 3); + expect(slots[1]).toBeGreaterThan(slots[0]); + expect(slots[1]).toBeLessThan(slots[2]); + }); +}); + +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts when another instance takes a claimed slot', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. + const runId = await newSlotRun(); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..318b95c204 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,288 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + FIRST_SLOT, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts above the slot the run’s own creation event owns', async () => { + // `run_created` takes the first slot outright — nothing can precede it — so + // an allocation never hands that position out. + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('leaves a hole in the persisted log unfilled', async () => { + // Allocation is append-only: the free position sits below a published event, + // and an event placed there would order before one that already happened. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('stays above published events when a lower position comes free', async () => { + // The corruption this rules out: a step completion allocating late, dropping + // into a hole, and landing below the step_started it reports on. Replay reads + // the log in slot order and cannot consume that. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + const published = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(published)); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBeGreaterThan(published); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; a book that only moved + // on publish would give them all the same position and fail all but one. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => RUN_CREATED_SLOT + 1 + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('honours a floor above where the book has reached', async () => { + // start() publishes the run entity before its `run_created` event and issues + // the queue send in parallel, so the delivery's `run_started` can allocate + // while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, 5)).resolves.toBe(5); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); +}); + +describe('release', () => { + it('does not recycle an abandoned interior slot', async () => { + // The position may already sit below a sibling that published, and no + // caller can tell from here. A hole is a position nothing ever wrote; an + // inversion is an event a replay reads before the one it followed. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('highestWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(2); + }); + + it('reports the tail of a log with a hole below it', async () => { + // The case the tail exists to catch: slot 3 was claimed by a write that + // never published, so it is free — but a claim on it would land below the + // event at 4 that a replay may already have consumed. + await writeEvents(1, 2, 4); + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(4); + }); + + it('is below the first slot for a run with no events', async () => { + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); + }); + + it('ignores a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); + }); +}); + +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('stops holding back a claim that resolved', async () => { + // Released before anything allocated for the run, so no position in this + // instance was ever handed out above it and the log — which is the authority + // on what published — reaches only slot 1. Nothing can be inverted by + // seeding the book from disk alone. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + +describe('observe', () => { + it('moves allocation above a position the client claimed', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(RUN_CREATED_SLOT + 1); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..1b1822e704 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,327 @@ +/** + * Slot allocation for the Local World. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second. A replay reads the log in slot + * order, so the order slots are handed out in has to be an order some execution + * could have produced — which makes allocation strictly *append-only*: a slot is + * only ever handed out above every position this book has seen. + * + * Filling a hole is what that rules out, and it is worth naming why, because the + * alternative looks appealing (it keeps the log dense). A position left unwritten + * by an abandoned reservation sits below events that are already published. Hand + * it to the next caller and a `step_completed` lands below its own + * `step_started`; the replay reaches a completion for a step it has not started + * and diverges, and every later replay diverges the same way. A hole costs a + * reader the ability to prove its copy of the log is complete. An inversion + * costs the run. + * + * Three properties do the work: + * + * - Handing out a slot is a *synchronous* set operation, so concurrent + * callers in one process get distinct slots with no lock. The only await is + * seeding from disk, which is memoized per run. + * - An allocation picks the position above the highest one the book knows of, + * written or outstanding, and that ceiling never descends. A reservation that + * is abandoned (its create threw a validation error) leaves its position + * unused rather than being recycled below a sibling that already published. + * - The event publish is `writeExclusive`, which is the authority. The book is + * a hint: when it turns out to be stale (another process wrote the slot), + * the publish fails and the caller is told so, rather than a duplicate being + * written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** + * The highest position this book has ever seen written, claimed or handed out. + * Allocation goes above it and it never descends, which is what keeps a + * released position from being recycled below events already published. + */ + ceiling: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet is not + * slot-numbered, and that answer is not cached: the resilient-start path + * creates the run moments later, and caching "no" would strand it on ULIDs + * for the rest of this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the position above every one this book knows of for `runId`, and at + * or above `minSlot`. Distinct for every concurrent caller; the publish still + * has to prove the position was actually free. + * + * `minSlot` defaults to the position above the run's first slot, which is + * reserved for its own `run_created`: that event needs no allocation, and it + * may not be on disk yet when a concurrent `run_started` allocates (start() + * issues the creation and the queue send in parallel, and the run entity is + * published before its event). + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * The highest position this run has published, seeding from disk if the run + * has not been read yet, or `FIRST_SLOT - 1` for a log with no events. + * + * This is the tail a claim has to clear. "Free" is not the property a claim + * needs: allocation is append-only, so a position left unwritten by an + * abandoned reservation stays empty for good, and a caller numbering from a + * snapshot that predates the events above such a hole aims straight at it. Let + * that write land and the event sits *below* events another replay already + * consumed — the log stays internally consistent while its order silently + * changes, which is enough to flip a race between a step and a sleep from one + * replay to the next. + * + * Reading the tail before the write also lets a doomed claim be rejected + * *before* the create materializes its step, hook or wait: the entity mutation + * runs ahead of the event publish, so a claim that only fails at the publish + * leaves an entity behind with no event, and the caller's re-proposal at the + * next slot then collides with its own orphan. A tail read here is not a + * promise — the publish is still the authority, and another instance sharing + * the data directory may have written above it — but it turns the case that + * actually happens (a caller numbering from a stale log) into a clean + * conflict. + */ + highestWritten(runId: string): Promise; + /** + * Forgets a reserved or claimed slot whose publish is never going to happen, + * so nothing waits on it. The position itself is not handed out again: it may + * already sit below a sibling that published, and recycling it there would put + * a later event below an earlier one. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run is slot-numbered, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const outstanding = new Set(claims.get(runId)); + const book: RunSlots = { + written, + outstanding, + ceiling: Math.max(FIRST_SLOT - 1, ...written, ...outstanding), + }; + books.set(runId, book); + return book; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + + function take(book: RunSlots, minSlot: number): number { + const slot = Math.max(book.ceiling + 1, minSlot); + book.outstanding.add(slot); + book.ceiling = slot; + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { + const opened = open(runId); + // Awaiting a book that is already in hand would yield to the microtask + // queue and let a concurrent caller take the same slot. + return take(opened instanceof Promise ? await opened : opened, minSlot); + }, + + claim(runId, slot) { + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } + const book = books.get(runId); + if (book) { + book.outstanding.add(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + }, + + async highestWritten(runId) { + const book = await open(runId); + return Math.max(FIRST_SLOT - 1, ...book.written); + }, + + release(runId, slot) { + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + return; + } + // The ceiling stays where it is: this position may already sit below one a + // sibling published, and handing it out again would order a later event + // before an earlier one. + book.outstanding.delete(slot); + }, + + observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + }, + + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + } + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. + }, + + clear() { + modes.clear(); + books.clear(); + claims.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_keys.sql b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_keys.sql new file mode 100644 index 0000000000..980739cbbc --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_keys.sql @@ -0,0 +1,10 @@ +-- Event ids are unique per run, not globally. Under slot identity (spec 6) +-- every run numbers its own log from 1, so "evnt_0...001" exists once per run +-- and the old global primary key would make the second run to reach slot 1 +-- collide with the first. +-- +-- The run leads the key so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id index redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index b7fb5d8215..4b82d140c5 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1785283200000, "tag": "0017_add_hook_resume_context", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1785801600000, + "tag": "0018_run_scoped_event_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index af328b043e..ff6d601b8d 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -153,7 +153,11 @@ export const events = schema.table( > >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: under slot identity every run + // numbers its own log from 1, so `evnt_0…001` exists once per run. The run + // leads the key so the range scans in `list` stay a single index seek, and + // it subsumes the plain `run_id` index the table used to carry. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 430cea0812..926d752d3b 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,10 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. + specVersion: mintedSpecVersion(), ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..6a0366a636 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,231 @@ +/** + * Slot identity for the postgres world. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second. Contention on a position is the + * point — it is what makes a concurrent write detectable rather than silent — + * so the two things this module has to get right are that a position is written + * at most once and that a position this allocator loses is retried rather than + * abandoned as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so for a slot-numbered run the highest id names the highest + * written position — and because a log holds ids of exactly one scheme, that id + * also reports which scheme the run was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event of a slot-numbered run, at the position the caller claimed or + * at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; + try { + return await options.write({ + primary: () => primary, + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 140597a6a9..9a13a17abb 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -62,6 +69,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + highestEventId, + highestSlotOf, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; /** @@ -442,7 +456,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -453,7 +467,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -464,6 +478,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { let eventId: string | undefined; @@ -490,6 +562,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -585,7 +671,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -631,7 +723,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -651,6 +743,107 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and a ULID names no position: the replay would read + // it wherever its timestamp happens to sort rather than where the writer + // meant it to go. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway — so a + // slot-numbered run pays nothing extra for this query. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is either + // claimed by a caller that holds the log (and is therefore asserting the + // log is complete up to that position) or allocated at write time for a + // caller that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + // A claim has to clear the log's tail, not merely be free. A position + // claimed by a write that then failed is never filled, so a log can + // carry holes below its tail; a caller numbering from a snapshot that + // predates the events above such a hole aims straight at it, and a + // free-position check would accept the write. The event then lands + // *below* events another replay has already consumed — the log stays + // internally consistent while its order silently changes, which is + // enough to flip a race between a step and a sleep from one replay to + // the next. + // + // Checked here, before the materialization below creates the step, hook + // or wait this event would now never accompany: a caller that + // re-proposes at the next position would otherwise collide with its own + // orphan and read that as "my write already landed". + seedHighestEventId = + seedHighestEventId ?? + (await highestEventId(drizzle, effectiveRunId)) ?? + undefined; + if (claimedSlot <= highestSlotOf(seedHighestEventId)) { + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -676,17 +869,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1192,41 +1387,51 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1236,99 +1441,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1496,7 +1706,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1531,20 +1741,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1622,47 +1833,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1694,9 +1907,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } @@ -1748,17 +1995,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..a6cf498651 --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,426 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(3) } + ); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having + // reserved two positions. A second event numbered off the log as this + // world sees it would take the slot the next start in the batch claimed, + // costing every start after the first its claim. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = await Promise.all( + claims.map((eventId, index) => + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ) + ); + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free rather than abandoning the one it lost, + // so a burst of concurrent writers still numbers itself densely. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('numbers a burst so the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index d514576c7b..c0b7f65491 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -1,6 +1,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -94,6 +95,16 @@ describe('isRetryableEventPostError', () => { expect(isRetryableEventPostError(new ThrottleError('429'))).toBe(false); }); + it('does not retry a lost event slot', () => { + // Re-issuing the same write is guaranteed to lose the slot again; only a + // merge and a replay can produce a write that lands. + expect( + isRetryableEventPostError( + new SlotConflictError('taken', { eventId: 'evnt_x' }) + ) + ).toBe(false); + }); + it('retries a body-parse failure (write may have landed)', () => { expect( isRetryableEventPostError( diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index fc0d3bd87f..25985a6c5c 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -40,6 +40,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -203,9 +204,13 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { export function isRetryableEventPostError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried in-process. // (425/429 are intentionally left to the runtime's retry-after handling.) + // A slot conflict is doubly definitive: the write lost a race for its event + // id, so re-issuing it unchanged is guaranteed to lose again. Only a merge and + // a replay can produce a write that can land. if ( EntityConflictError.is(err) || RunExpiredError.is(err) || + SlotConflictError.is(err) || TooEarlyError.is(err) || ThrottleError.is(err) ) { diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 181c678fec..f788468974 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -2,10 +2,16 @@ import { EntityConflictError, PreconditionFailedError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, } from '@workflow/errors'; +import { + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotIdBody, +} from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -28,7 +34,7 @@ import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; describe('throwForErrorResponse', () => { const call = ( status: number, - body = '{"message":"boom"}', + body: string | Uint8Array = '{"message":"boom"}', headers: Record = {} ) => throwForErrorResponse(status, headers, body, 'createEvent', 'http://x'); @@ -108,6 +114,109 @@ describe('throwForErrorResponse', () => { /createEvent failed: HTTP 500 plain text oops/ ); }); + + /** + * The slot-conflict 409 is the only v4 error body that arrives as CBOR: it + * carries the event-log delta the client replays from, whose payloads are + * byte strings that JSON cannot represent. Decoding it as JSON would lose the + * delta silently and mis-type the error as an entity conflict, which the + * runtime reads as "my write already landed". + */ + describe('slot conflict', () => { + const PAYLOAD = new Uint8Array([1, 2, 3]); + const conflictBody = ( + overrides: Record = {} + ): Uint8Array => + new Uint8Array( + encode({ + success: false, + error: 'slot-conflict', + message: "Event slot 'evnt_…003' is already taken", + details: { eventId: 'evnt_from_details' }, + events: [{ eventId: 'evnt_x', eventData: { output: PAYLOAD } }], + cursor: 'eid:evnt_x', + hasMore: false, + ...overrides, + }) + ); + + it('decodes a CBOR body into SlotConflictError with the delta intact', () => { + try { + call(409, conflictBody(), { + 'content-type': 'application/cbor', + 'x-wf-event-id': 'evnt_from_header', + }); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + const conflict = err as SlotConflictError; + expect(conflict.eventId).toBe('evnt_from_header'); + expect(conflict.cursor).toBe('eid:evnt_x'); + expect(conflict.hasMore).toBe(false); + // Binary payloads survive: this is what a JSON error path destroys. + expect(conflict.events).toHaveLength(1); + expect( + (conflict.events[0] as { eventData: { output: Uint8Array } }) + .eventData.output + ).toEqual(PAYLOAD); + // Not the 409 → EntityConflictError mapping, which the runtime reads as + // "the write I am retrying already landed". + expect(EntityConflictError.is(err)).toBe(false); + } + }); + + it('falls back to the eventId in details when the header is absent', () => { + try { + call(409, conflictBody(), { 'content-type': 'application/cbor' }); + expect.unreachable(); + } catch (err) { + expect((err as SlotConflictError).eventId).toBe('evnt_from_details'); + } + }); + + it('reports an empty delta when the backend could not read one', () => { + try { + call(409, conflictBody({ events: undefined, cursor: null }), { + 'content-type': 'application/cbor', + }); + expect.unreachable(); + } catch (err) { + const conflict = err as SlotConflictError; + expect(conflict.events).toEqual([]); + expect(conflict.cursor).toBeNull(); + } + }); + + it('reads a JSON-encoded slot conflict too', () => { + // Nothing in the protocol forbids a JSON encoding of the same body; only + // the delta's binary payloads require CBOR. + try { + call( + 409, + JSON.stringify({ + error: 'slot-conflict', + message: 'taken', + events: [], + cursor: 'eid:evnt_y', + hasMore: true, + }), + { 'x-wf-event-id': 'evnt_j' } + ); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + expect((err as SlotConflictError).hasMore).toBe(true); + } + }); + + it('leaves an ordinary 409 as EntityConflictError', () => { + // Entity materialization conflicts share the status and are how the + // runtime recognizes a duplicate write. + expect(() => + call(409, JSON.stringify({ error: 'conflict', message: 'exists' })) + ).toThrowError(EntityConflictError); + }); + }); }); /** @@ -954,6 +1063,117 @@ describe('createWorkflowRunEventV4 over HTTP', () => { expect('stateUpdatedAt' in (capturedMeta ?? {})).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('sends the claimed eventId and maxSlot in the frame meta', async () => { + // A slot-numbered run names its own event ids, so the id has to reach the + // wire: the backend reads it from the frame meta and inserts it + // conditionally. Dropped, the backend mints a ULID instead and the run + // silently reverts to server-assigned identity mid-log. + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const eventId = slotEventId(4); + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `wait_${slotIdBody(1)}`, + eventId, + maxSlot: 3, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventId).toBe(eventId); + expect(capturedMeta?.maxSlot).toBe(3); + agent.assertNoPendingInterceptors(); + }); + + it('omits eventId and maxSlot from the frame meta for a ULID-numbered run', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + expect('eventId' in (capturedMeta ?? {})).toBe(false); + expect('maxSlot' in (capturedMeta ?? {})).toBe(false); + agent.assertNoPendingInterceptors(); + }); }); /** diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index f26187f856..170dc3d567 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,6 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ +import { SlotConflictError } from '@workflow/errors'; import { type Event, getEventDataPayloadField } from '@workflow/world'; import { decode } from 'cbor-x'; import { coerceEventDates } from './event-coerce.js'; @@ -74,13 +75,33 @@ async function fetchV4( errorFromV4Response( response.status, headersToRecord(response.headers), - await response.text(), + await readErrorBody(response), opName, url ), }); } +/** + * The error body as bytes when it is CBOR, as text otherwise. + * + * Most v4 error responses are JSON, because the client sends no `Accept` header + * and the backend's error encoder defaults to it. A slot conflict is the + * exception: its body carries the event-log delta the client needs, whose + * payloads are byte strings that JSON cannot represent, so the backend encodes + * that one as CBOR regardless of the `Accept` header. + */ +async function readErrorBody(response: Response): Promise { + if (isCborContentType(response.headers.get('content-type'))) { + return new Uint8Array(await response.arrayBuffer()); + } + return await response.text(); +} + +function isCborContentType(contentType: string | null | undefined): boolean { + return contentType?.toLowerCase().includes('application/cbor') ?? false; +} + /** Flatten a fetch `Headers` into the record shape throwForErrorResponse * expects (it mirrors the v3 `makeRequest` error contract). */ function headersToRecord(headers: Headers): Record { @@ -217,6 +238,22 @@ export interface CreateEventV4Input { * without a loaded event log; older servers ignore it entirely. */ stateUpdatedAt?: number; + /** + * The event's id, claimed by the client instead of minted by the server. + * Sent only for a run that numbers its events by slot, where the id encodes + * the event's position in the log. The server inserts it conditionally and + * answers 409 `slot-conflict` when the slot is already taken; a run on the + * older numbering that sends one is rejected with 400. Older servers ignore + * the field and mint an id as before — which is why only runs stamped with + * slot identity ever send it. + */ + eventId?: string; + /** + * The highest slot the client has seen in the run's event log (0 when it has + * seen none). Observability only: slots are dense, so a persisted slot more + * than one past this is a permanent hole in the log. Ignored by older servers. + */ + maxSlot?: number; /** * Number of loaded events at or below `stateUpdatedAt` (i.e. the loaded * log's length). Sent with `stateUpdatedAt` so the backend can also reject @@ -353,6 +390,8 @@ function buildPostFrameMeta( if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } + if (input.eventId !== undefined) meta.eventId = input.eventId; + if (input.maxSlot !== undefined) meta.maxSlot = input.maxSlot; if (input.stateEventCount !== undefined) { meta.stateEventCount = input.stateEventCount; } @@ -366,6 +405,73 @@ function buildPostFrameMeta( return meta; } +/** + * The backend's machine-readable code for a lost event slot. Paired with 409 + * rather than 412 so a slot conflict stays distinguishable from the + * `stateUpdatedAt` watermark's staleness rejection while both are live. + */ +const V4_SLOT_CONFLICT_CODE = 'slot-conflict'; + +/** The fields a v4 error body may carry, whatever encoding it arrived in. */ +interface V4ErrorBody { + message?: unknown; + /** Machine-readable code. The backend names this field `error`. */ + error?: unknown; + code?: unknown; + events?: unknown; + cursor?: unknown; + hasMore?: unknown; + details?: unknown; +} + +/** Decode an error body as CBOR or JSON, or `undefined` if it is neither. */ +function decodeErrorBody( + errorBody: string | Uint8Array +): V4ErrorBody | undefined { + try { + const value = + typeof errorBody === 'string' + ? (JSON.parse(errorBody) as unknown) + : (decode(errorBody) as unknown); + return value && typeof value === 'object' + ? (value as V4ErrorBody) + : undefined; + } catch { + return undefined; + } +} + +/** + * Build the `SlotConflictError` for a 409 whose body names a taken slot. + * + * The conflicting event id comes from the response header rather than the body + * so the error is still actionable when the body failed to decode; the delta is + * best-effort in the other direction — an absent or malformed `events` leaves + * the runtime to reload the log itself, which is always correct. + */ +function slotConflictFromBody( + message: string, + responseHeaders: Record, + body: V4ErrorBody | undefined +): SlotConflictError { + const details = body?.details; + const detailEventId = + details && typeof details === 'object' && 'eventId' in details + ? (details as { eventId?: unknown }).eventId + : undefined; + const headerEventId = readHeader( + responseHeaders, + V4_RESPONSE_HEADERS.eventId + ); + return new SlotConflictError(message, { + eventId: + headerEventId ?? (typeof detailEventId === 'string' ? detailEventId : ''), + events: Array.isArray(body?.events) ? body.events : [], + cursor: typeof body?.cursor === 'string' ? body.cursor : null, + hasMore: body?.hasMore === true, + }); +} + /** * Build the typed error for a non-2xx v4 response. Reuses the shared * `errorForResponse` status → error-type contract (409→EntityConflictError, @@ -378,26 +484,29 @@ function buildPostFrameMeta( function errorFromV4Response( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; let details: unknown; - try { - const json = JSON.parse(errorBody) as { - message?: string; - code?: string; - events?: unknown; - cursor?: unknown; - }; - if (typeof json.message === 'string') message = json.message; - if (typeof json.code === 'string') code = json.code; - if (statusCode === 412) details = decodePreconditionDetails(json); - } catch { - // body wasn't JSON — keep the default message, append raw text below - if (errorBody) message += ` ${errorBody}`; + const decoded = decodeErrorBody(errorBody); + if (decoded) { + if (typeof decoded.message === 'string') message = decoded.message; + if (typeof decoded.code === 'string') code = decoded.code; + if (statusCode === 412) details = decodePreconditionDetails(decoded); + } else if (typeof errorBody === 'string' && errorBody) { + // Body was neither JSON nor CBOR — keep the default message and append the + // raw text so the response is still diagnosable. + message += ` ${errorBody}`; + } + + // A lost event slot is the one 409 that is not an entity conflict. The + // backend names its machine-readable code `error`; that field is read only + // here, so every other error keeps the status → type mapping below unchanged. + if (statusCode === 409 && decoded?.error === V4_SLOT_CONFLICT_CODE) { + return slotConflictFromBody(message, responseHeaders, decoded); } const retryAfter = parseRetryAfter( @@ -490,7 +599,7 @@ function hasUnusablePayload(candidate: Record): boolean { export function throwForErrorResponse( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): never { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 7e88df8737..34673fcf61 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -32,7 +32,11 @@ * the v3 path. */ -import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; +import { + HookNotFoundError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import { type AnyEventRequest, type CreateEventParams, @@ -434,6 +438,35 @@ function coerceNormalizedEvent(raw: Record): Event { return coerceEventDates(normalizeEventData(raw)); } +/** + * Runs an event create, normalizing the event-log delta a slot conflict carries + * into the same `Event` shape every other read path produces. + * + * The delta arrives as raw CBOR off the error response, so its nested dates are + * still ISO strings; the runtime merges these events into its loaded log and + * calls `.getTime()` on them, exactly as it does for the inline delta on the + * success path. Doing the coercion here rather than in the runtime keeps the + * wire's shape a concern of this adapter, and keeps `SlotConflictError.events` + * meaning the same thing for every World that raises it. + */ +async function withCoercedSlotConflictDelta( + op: () => Promise +): Promise { + try { + return await op(); + } catch (error) { + if (!SlotConflictError.is(error) || error.events.length === 0) { + throw error; + } + throw new SlotConflictError(error.message, { + eventId: error.eventId, + events: (error.events as Record[]).map(coerceEventDates), + cursor: error.cursor, + hasMore: error.hasMore, + }); + } +} + function decodeLegacyStructuredError(payload: Uint8Array): unknown { if (hasSerializedDataFormatPrefix(payload)) { return payload; @@ -587,9 +620,11 @@ export async function createWorkflowRunEvent( // the next queue delivery. Non-retryable // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. - return await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, params, config), - data.eventType + return await withCoercedSlotConflictDelta(() => + withEventPostRetry( + () => createWorkflowRunEventInner(id, data, params, config), + data.eventType + ) ); } catch (err) { // 404 on hook_disposed / hook_received → already-disposed hook. @@ -713,6 +748,13 @@ async function createWorkflowRunEventInner( ...(params?.resumePayloadDigest ? { resumePayloadDigest: params.resumePayloadDigest } : {}), + // Slot identity: the runtime names the event's own id, claiming that + // position in the run's event log. The server inserts it conditionally + // and answers 409 slot-conflict when another writer got there first. + // `maxSlot` rides along so the server can spot a gap, which slots being + // dense makes an unrecoverable corruption. + ...(params?.eventId ? { eventId: params.eventId } : {}), + ...(params?.maxSlot !== undefined ? { maxSlot: params.maxSlot } : {}), remoteRefBehavior, payload, ...meta, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 7d42a8b6ec..78f4412822 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6) unless + // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side + // zstd/gzip payload compression over a superset of the v4 attributes. + // Either way this world reads both, so the stamp only decides how the runs + // it creates from here on are numbered. + specVersion: mintedSpecVersion(), capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 94687e8f20..3f5b32e392 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -32,7 +32,10 @@ import { version } from './version.js'; * `main` — rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. */ -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +// TEMPORARY — revert to '' before merge. Points e2e at the slot-identity +// backend branch deployment. +export const WORKFLOW_SERVER_URL_OVERRIDE = + 'https://workflow-server-git-peter-slot-event-identity.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter. diff --git a/packages/world/package.json b/packages/world/package.json index 681c1d3327..84d5f1b1a0 100644 --- a/packages/world/package.json +++ b/packages/world/package.json @@ -20,6 +20,7 @@ "scripts": { "build": "tsc", "dev": "tsc --watch", + "test": "vitest run src", "clean": "tsc --build --clean && rm -rf dist" }, "dependencies": { diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index b3dc6c9866..f94d707643 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -867,6 +867,37 @@ export interface CreateEventParams { * across the SDK and the backend. */ skipPreload?: boolean; + /** + * The event's id, chosen by the client rather than the World. + * + * Only sent for a run whose spec version numbers events by slot + * (`SPEC_VERSION_SLOT_IDENTITY`), where the id encodes the event's position in + * the log and is therefore the client's claim on that position. + * + * Backend contract (for World implementers who want to support slot + * identity): treat the id as a claim to be won, not a hint. Insert it under a + * uniqueness constraint on `(runId, eventId)` and, when the id is already + * taken, reject the write with `SlotConflictError` (HTTP 409) instead of + * minting a different id — a lost slot means the client replayed against an + * event log missing at least one event, so its whole proposed event, not just + * its id, is suspect. Reject a mismatch in either direction with a 400: a + * ULID names a time and a slot names a position, so a log holding both sorts + * partly by one and partly by the other and no replay can read it in the order + * it was written. + * + * A World that ignores this field keeps minting ids itself, which is correct + * only for runs that were never stamped with slot identity in the first + * place. + */ + eventId?: string; + /** + * The highest slot the client has observed in the run's event log, or 0 for a + * log with no slot-numbered events. Sent alongside {@link eventId} purely as + * an observability signal: because slots are dense, a persisted slot more + * than one past this is a hole, which is unrecoverable and worth alerting on. + * Worlds MAY ignore it. + */ + maxSlot?: number; } /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index d30909aa1e..f0d74f6454 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -106,16 +106,34 @@ export { PaginatedResponseSchema, StructuredErrorSchema, } from './shared.js'; +export { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, + slotEventId, + slotFromId, + slotIdBody, + slotRetryDelay, +} from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + usesSlotIdentity, } from './spec-version.js'; export type * from './steps.js'; export { diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts new file mode 100644 index 0000000000..acdfada627 --- /dev/null +++ b/packages/world/src/slot-identity.test.ts @@ -0,0 +1,107 @@ +import { ulid } from 'ulid'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + slotEventId, + slotFromId, + slotIdBody, +} from './slot-identity.js'; +import { ulidToDate } from './ulid.js'; + +describe('slotIdBody', () => { + it('pads to ULID width so a slot is accepted wherever a ULID is', () => { + const body = slotIdBody(FIRST_SLOT); + expect(body).toHaveLength(SLOT_ID_WIDTH); + expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); + // Crockford base32 starts with the decimal digits, so the padded body + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); + }); + + it('orders lexicographically by slot at a fixed width', () => { + const ascending = [1, 2, 9, 10, 100].map(slotIdBody); + expect([...ascending].sort()).toEqual(ascending); + }); + + it('rejects slots outside the dense numbering', () => { + expect(() => slotIdBody(0)).toThrow(); + expect(() => slotIdBody(-1)).toThrow(); + expect(() => slotIdBody(1.5)).toThrow(); + }); +}); + +describe('slotFromId', () => { + it('round-trips a prefixed id', () => { + expect(slotFromId(`step_${slotIdBody(42)}`)).toBe(42); + }); + + it('round-trips a bare body', () => { + expect(slotFromId(slotIdBody(42))).toBe(42); + }); + + it('reads no slot out of a ULID id', () => { + expect(slotFromId(`evnt_${ulid()}`)).toBeUndefined(); + expect(isSlotId(`evnt_${ulid()}`)).toBe(false); + }); + + it('reads no slot out of the all-zero range fence', () => { + // Slot 0 is the inclusive lower fence for range queries over a run's + // events, never an event. + expect(slotFromId('0'.repeat(SLOT_ID_WIDTH))).toBeUndefined(); + }); + + it('reads no slot out of a body of the wrong width', () => { + expect(slotFromId('evnt_1')).toBeUndefined(); + expect(slotFromId(`evnt_${'1'.repeat(SLOT_ID_WIDTH + 1)}`)).toBeUndefined(); + }); +}); + +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + +describe('maxSlotOf', () => { + it('finds the highest slot regardless of position', () => { + // A log is merged from several loads and is not sorted, so the last element + // is not necessarily the highest slot. + expect( + maxSlotOf([slotEventId(3), slotEventId(7), slotEventId(1)].map(toEvent)) + ).toBe(7); + }); + + it('reports 0 for an empty or ULID-numbered log', () => { + expect(maxSlotOf([])).toBe(0); + expect(maxSlotOf([toEvent(`evnt_${ulid()}`)])).toBe(0); + }); + + it('ignores ULID ids mixed in with slots', () => { + // A mixed log violates slot identity's purity invariant, but the scan must + // still report the highest slot rather than throwing or returning 0. + expect( + maxSlotOf([toEvent(`evnt_${ulid()}`), toEvent(slotEventId(2))]) + ).toBe(2); + }); +}); + +function toEvent(eventId: string): { eventId: string } { + return { eventId }; +} diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts new file mode 100644 index 0000000000..286e910c39 --- /dev/null +++ b/packages/world/src/slot-identity.ts @@ -0,0 +1,121 @@ +/** + * Slot identity: dense, per-run numbering for event ids. + * + * A slot id's body is a decimal counter zero-padded to ULID width. Crockford's + * base32 alphabet begins with the ten decimal digits, so that body is a + * syntactically valid ULID body — every schema, sort key, range fence and + * cursor that accepted a ULID keeps accepting a slot — and because the width is + * fixed, lexicographic order is numeric order. + * + * Slots start at 1 and are handed out above every position the allocator has + * seen, never into a lower one that happens to be free. That makes contention + * explicit — two writers proposing one position cannot both win, and the loser + * is told which events it was missing — and it keeps slot order, which is the + * order a replay reads the log in, a linear extension of what actually + * happened. Filling a hole would place an event below ones that preceded it, + * and a replay reaching a `step_completed` below its own `step_started` diverges + * for good. Zero is left unused because the inclusive lower fence for + * range queries over a run's events is the all-zero id. + * + * Allocation being append-only does not make a published log gap-free, so + * `events.length === maxSlot` is not a completeness proof. A slot claimed by an + * operation that then fails for a reason of its own is never filled, and if a + * later slot has already been published the gap is permanent. Nothing may treat + * a missing slot as an event still on its way. + * + * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so + * nothing may read a time out of one. Use the event's own `createdAt` / + * `occurredAt`. + */ + +/** Width of a slot id's body: ULID width, so a slot is accepted wherever a ULID is. */ +export const SLOT_ID_WIDTH = 26; + +/** First slot in a run. Slot 0 is unused — it is the inclusive range fence. */ +export const FIRST_SLOT = 1; + +const SLOT_BODY_PATTERN = new RegExp(`^[0-9]{${SLOT_ID_WIDTH}}$`); + +/** + * The id body naming `slot`, e.g. `1` → `00000000000000000000000001`. Callers + * prepend their own prefix (`evnt_`, `step_`, `wait_`). + */ +export function slotIdBody(slot: number): string { + if (!Number.isInteger(slot) || slot < FIRST_SLOT) { + throw new Error( + `Slot must be an integer >= ${FIRST_SLOT}, received ${slot}` + ); + } + const body = String(slot).padStart(SLOT_ID_WIDTH, '0'); + if (body.length > SLOT_ID_WIDTH) { + throw new Error(`Slot ${slot} does not fit in ${SLOT_ID_WIDTH} digits`); + } + return body; +} + +/** + * The slot named by an id, or undefined if the id is not a slot id. Accepts + * both a prefixed id (`step_0…001`) and a bare body. + */ +export function slotFromId(id: string): number | undefined { + const underscore = id.indexOf('_'); + const body = underscore === -1 ? id : id.slice(underscore + 1); + if (!SLOT_BODY_PATTERN.test(body)) { + return undefined; + } + const slot = Number(body); + return slot >= FIRST_SLOT ? slot : undefined; +} + +/** Whether an id numbers itself by slot rather than by ULID. */ +export function isSlotId(id: string): boolean { + return slotFromId(id) !== undefined; +} + +/** The event id occupying `slot`. */ +export function slotEventId(slot: number): string { + return `evnt_${slotIdBody(slot)}`; +} + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** + * The highest slot named by any of `events`, or 0 when none is slot-numbered. + * + * Scans rather than reading the last element: a log is merged from several + * loads and is not necessarily sorted, and callers use this value to pick the + * next free slot. + */ +export function maxSlotOf(events: readonly { eventId: string }[]): number { + let max = 0; + for (const event of events) { + const slot = slotFromId(event.eventId); + if (slot !== undefined && slot > max) { + max = slot; + } + } + return max; +} diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..0125e2fb1c 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +17,21 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than it mints', () => { + // Slot identity is readable by every world before any world mints it, so + // that turning it on for new runs cannot make those same worlds reject + // them. Once slots are the default the two constants coincide again. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +73,32 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('mintedSpecVersion', () => { + it('mints slot identity by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); + }); + + it('mints the previous version when the flag is switched off', () => { + for (const value of ['0', 'false']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('treats any other value as on', () => { + // An unset-but-present variable is the shape a shell leaves behind, and it + // must not silently switch a deployment's event identity scheme. Opting + // out takes an explicit `0`/`false`. + for (const value of ['', '1', 'true', 'yes']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_SLOT_IDENTITY + ); + } + }); + + it('mints nothing a world cannot read', () => { + expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index cf516b772b..874bda0e24 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -31,13 +31,72 @@ export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; +/** + * Runs at this spec version or later number their events by dense per-run slots + * (`evnt_…001`) instead of ULIDs. Ids are minted by the client, and a write + * that proposes an event id already taken is rejected rather than renumbered — + * which is what lets a client prove its loaded log is complete. + * + * A run is in exactly one mode for life: the mode is read from the run's + * persisted `specVersion`, never from the build. A run whose log holds ULID + * event ids and is replayed by a slot-capable build would otherwise propose + * `evnt_…001`, a position its very first event already occupies. + * + * Correlation ids are unaffected: steps, waits, hooks and attributes keep their + * seeded ULIDs in both modes. + */ +export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; + /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * The floor a world stamps on new runs, and a *lower* bar than the newest + * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a + * world actually stamps comes from {@link mintedSpecVersion}; this is what it + * falls back to when slot identity is switched off. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to + * read a version before anything may mint it, and because a world that mints + * slot identity still has to read the spec-5 runs it created before the switch. + * Worlds opt into minting individually, via the `specVersion` they declare. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs out of slot identity. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; + +/** + * The spec version a world should stamp on the runs it creates: slot identity + * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case + * {@link SPEC_VERSION_CURRENT}. + * + * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this + * returns, so turning the flag off in one place does not make the runs another + * process created unreadable here. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + const value = env[SLOT_IDENTITY_ENV_VAR]; + return value === '0' || value === 'false' + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SLOT_IDENTITY; +} + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -55,7 +114,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -63,5 +122,17 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; +} + +/** + * Whether a run numbers its events by slot. Always pass the run's persisted + * `specVersion`; see `SPEC_VERSION_SLOT_IDENTITY`. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run uses slot identity + */ +export function usesSlotIdentity(v: number | undefined | null): boolean { + if (v === undefined || v === null) return false; + return v >= SPEC_VERSION_SLOT_IDENTITY; } diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..cc893834c3 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every slot-numbered event. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; From ac429edae118b4f0b86adb8a393cbb8013280148 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 15:18:36 -0700 Subject: [PATCH 02/14] world-local: one hook_received per delivery, dense across a converged retry A hook delivery has two writers, the request that resumed it and the queue consumer that replays the run, and each takes a position to publish at. Both could publish: the writer that lost the race at the position its resume claim named treated it as ordinary contention and moved to a free one, so one delivery ended up with two events. The claim's `eventId` is now a hint and the `resumeId` on the persisted event is the identity, so a writer that loses the pinned position resolves against the other writer of the same delivery instead of reallocating, and a position another instance took for an unrelated event no longer answers for the delivery. A writer that converges on the other's event also hands its own position back, so the retry does not leave the run one position short of complete. --- .changeset/local-resume-one-event.md | 5 + .../world-local/src/storage/events-storage.ts | 197 ++++++++++++++++-- .../src/storage/slot-identity.test.ts | 131 ++++++++++-- .../world-local/src/storage/slots.test.ts | 34 +++ packages/world-local/src/storage/slots.ts | 128 ++++++++---- 5 files changed, 416 insertions(+), 79 deletions(-) create mode 100644 .changeset/local-resume-one-event.md diff --git a/.changeset/local-resume-one-event.md b/.changeset/local-resume-one-event.md new file mode 100644 index 0000000000..e3e888b290 --- /dev/null +++ b/.changeset/local-resume-one-event.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Record one `hook_received` per hook delivery when the delivery is written twice concurrently, and keep the position-numbered log dense across it diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index b29d9a8a29..d7b22d358f 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -204,6 +204,19 @@ const HookResumeClaimSchema = z.object({ payloadDigest: z.string().optional(), }); +/** + * How long a writer that adopted a claim's pinned event id waits for that event + * to become reader-visible. + * + * It is waiting on the other writer of the same resume, which holds the position + * staged under `.locks` while it re-checks the run's terminal marker and links + * the file into `events/`: two filesystem operations, so the window is short and + * the wait almost always ends on the first look. The bound exists for the one + * case that never ends — a writer that died between staging and promoting, whose + * staged file no one will ever promote or clean up. + */ +const PINNED_EVENT_WAIT_MS = 2_000; + async function readHookTokenClaim( constraintPath: string ): Promise | null> { @@ -306,6 +319,34 @@ async function findExistingHookCreatedEventId( return result.data[0]?.eventId ?? null; } +/** + * The run's committed `hook_received` for one resume, if it has one. + * + * The resume claim records where the event is going to be published, and in a + * slot-numbered run that is only a hint: another instance can allocate the same + * position for an unrelated event from its own book, and then the claim points + * at a stranger. The key the event itself carries is the durable identity, so + * this scan is what decides whether a resume has already been recorded. + */ +async function findCommittedResumeEvent( + basedir: string, + runId: string, + resumeId: string +): Promise { + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + filePrefix: `${runId}-`, + filter: (event) => + event.eventType === 'hook_received' && event.resumeId === resumeId, + limit: 1, + getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, + getId: (event) => event.eventId, + }); + return result.data[0] ?? null; +} + /** * Repair an "event-first orphan": the hook entity write is deferred * until after the `hook_created` event publish commits (so a failed @@ -754,9 +795,10 @@ export function createEventsStorage( // A slot-numbered create reserves its position before running the // validation and materialization that may still reject it. Handing the - // reservation back on the way out is what keeps the log dense: an - // abandoned slot below a sibling's published one is a hole that can never - // be filled, and a log with a hole can no longer prove it is complete. + // reservation back on the way out — whether the create throws or returns + // another writer's event — is what keeps the log dense: an abandoned slot + // below a sibling's published one is a hole that can never be filled, and + // a log with a hole can no longer prove it is complete. const reserved = new Set(); let reservedRunId: string | undefined; /** @@ -774,9 +816,14 @@ export function createEventsStorage( */ let eventCommitted = false; /** - * Hands the slots of a create that never published back to the allocator, - * so an abandoned reservation below a sibling's published slot does not - * become a hole the run can never fill. + * Hands the slots of a create that never published back to the allocator. + * + * The allocator recycles a returned position only while it is still the + * top of the book, which is the case that keeps a run dense: a create that + * takes the next position and then publishes nothing would otherwise leave + * the log one short forever. A position that already sits below a + * sibling's published event stays empty by design — reusing it there would + * put a later event below an earlier one. */ async function releasingSlots( result: Promise @@ -784,11 +831,6 @@ export function createEventsStorage( try { return await result; } catch (error) { - if (reservedRunId !== undefined) { - for (const slot of reserved) { - slots.release(reservedRunId, slot); - } - } if (!eventCommitted) { for (const undo of abandonedClaims.reverse()) { // Best effort: the throw the caller sees is the one that matters, @@ -798,6 +840,19 @@ export function createEventsStorage( } } throw error; + } finally { + // Every reservation this create still holds goes back, including on + // the paths that succeed without publishing anything: a create that + // converged on another writer's event returns a result and leaves its + // own position untouched, and keeping it reserved would stop the next + // event of the run from taking it. Handing back a position that *was* + // published is a no-op — the publish recorded it as written, so it + // cannot be handed out again. + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } } } @@ -854,6 +909,16 @@ export function createEventsStorage( // so concurrent / cross-process workers converge on a single // event in the log. let eventId = `evnt_${monotonicUlid()}`; + // Set when a resume claim names `eventId` — either this write reserved + // the claim, or it adopted the position another writer of the same resume + // pinned. Losing the publish at a pinned position is resolved against + // that other writer instead of by moving to a free position, which would + // give one resume two events. + let pinnedEventId: string | undefined; + // Whether this write is the one that reserved the resume claim. The + // claim's owner is the only writer allowed to move off the pinned + // position while the resume is still unrecorded. + let ownsResumeClaim = false; const now = new Date(); // For run_created events, use client-provided runId or generate one server-side @@ -1364,10 +1429,11 @@ export function createEventsStorage( // converges too. Gated on `resumeId` so the historical single-write // path is untouched. if (data.eventType === 'hook_received' && params?.resumeId) { + const resumeId = params.resumeId; const claimPath = hookResumeClaimPath( basedir, effectiveRunId, - params.resumeId + resumeId ); const converge = async ( claim: z.infer @@ -1396,20 +1462,35 @@ export function createEventsStorage( `hook_received resumeId "${params.resumeId}" already recorded with a different payload` ); } - const existing = await readJSONWithFallback( + const pinned = await readJSONWithFallback( basedir, 'events', `${effectiveRunId}-${claim.eventId}`, EventSchema, tag ); - if (existing) { - return { event: existing }; + // The event the claim points at answers for this resume only if it + // is this resume's. In a slot-numbered run the pinned position can + // hold an unrelated event another instance allocated it for, and + // returning that would report a step's event as the resume's. + if (pinned?.resumeId === resumeId) { + return { event: pinned }; + } + const committed = await findCommittedResumeEvent( + basedir, + effectiveRunId, + resumeId + ); + if (committed) { + return { event: committed }; } - // Claim exists but its event is not yet visible (a crash between - // the claim write and the append). Adopt the pinned eventId and - // fall through to (re)write the event idempotently at that path. + // The resume has no event yet: the pinning writer is mid-publish + // (its event is staged, not yet linked into `events/`), or it + // crashed between the claim write and the append. Adopt the pinned + // position and fall through to (re)write the event there, so the + // two writers of this resume still contend for one position. eventId = claim.eventId; + pinnedEventId = claim.eventId; return null; }; @@ -1438,7 +1519,15 @@ export function createEventsStorage( : {}), } satisfies z.infer) ); - if (!won) { + if (won) { + // The claim now names this position for this resume, and the + // other writer is on its way to adopting it. Losing the publish + // here therefore has to be resolved against that writer rather + // than by moving to a free position, which would give one resume + // two events. + pinnedEventId = eventId; + ownsResumeClaim = true; + } else { const winner = await readJSON(claimPath, HookResumeClaimSchema); if (winner) { const converged = await converge(winner); @@ -2435,10 +2524,15 @@ export function createEventsStorage( // Rebuild `event` with the canonical eventId and a // deterministic `createdAt` derived from the eventId // (a ULID) so two workers writing the same event - // produce byte-identical content. + // produce byte-identical content. A slot id carries no + // time, and reading one as a ULID yields the epoch — + // so a slot-numbered run keeps the wall clock, and the + // two workers converge on one event through the + // position instead of through the bytes. eventId = canonicalEventId; - const canonicalCreatedAt = - ulidToDate(eventId.replace(/^evnt_/, '')) ?? now; + const canonicalCreatedAt = slotMode + ? now + : (ulidToDate(eventId.replace(/^evnt_/, '')) ?? now); event = { ...data, runId: effectiveRunId, @@ -2871,6 +2965,7 @@ export function createEventsStorage( const reallocatesSlot = slotMode && params?.eventId === undefined && !ownsFirstSlot; const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + const pinnedDeadline = Date.now() + PINNED_EVENT_WAIT_MS; let compositeKey = ''; let eventPath = ''; let serializedEvent = ''; @@ -2887,6 +2982,64 @@ export function createEventsStorage( if (eventPublished) { break; } + if (pinnedEventId === eventId && params?.resumeId) { + // This resume's claim names this position and something else holds + // it. Taking a free position on the strength of that alone would + // publish a second event for one resume — the duplicate the claim + // exists to prevent, and one the log can never be rid of. So find out + // who holds it first. + const occupant = await readJSONWithFallback( + basedir, + 'events', + compositeKey, + EventSchema, + tag + ); + if (occupant?.resumeId === params.resumeId) { + // The other writer of this same resume got there first; its event + // is this write's answer. + return { event: occupant }; + } + const committed = await findCommittedResumeEvent( + basedir, + effectiveRunId, + params.resumeId + ); + if (committed) { + // The other writer published elsewhere: it reallocated around a + // stranger sitting on the pinned position. + return { event: committed }; + } + // Nobody has recorded this resume yet, so both of its writers are + // still live candidates and only one of them may leave the pinned + // position: if both moved, they could publish two events for one + // resume. That one is the writer that owns the claim, and only once + // it can see a stranger on the position — an unrelated event another + // instance allocated it for, which will never become this resume's. + // Anyone else waits for the owner's event to show up anywhere in the + // log, which the scan above will find. + const strangerHoldsPosition = occupant !== null; + if ( + !(ownsResumeClaim && strangerHoldsPosition) && + Date.now() < pinnedDeadline + ) { + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + continue; + } + // Out of patience. Placing the resume's event somewhere beats leaving + // it unrecorded: an owner that died between staging and promoting, or + // between the claim and the append, is never coming back, and a resume + // with no event hangs the run for good. The claim keeps pointing where + // it always did, which costs nothing — a recorded resume is found by + // the scan above, keyed on the resume itself. + if (!reallocatesSlot) { + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` + ); + } + } if (reallocatesSlot && Date.now() < slotDeadline) { // The position is someone else's — either published there or // staged for it. Record that, top the book up from disk, and try diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index f1ee5df07b..983cda0079 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -178,32 +178,22 @@ describe('numbering', () => { expect(maxSlotOf(data)).toBe(data.length); }); - it('leaves a rejected write’s position unused instead of recycling it', async () => { - // The rejected op's position sits below its concurrent sibling's, so handing - // it to the next writer would order that writer's event below one that - // already published. The hole costs a reader nothing it was promised; the - // inversion would cost the run. + it('reuses a rejected write’s position when nothing published above it', async () => { + // The rejected op took a position and gave it back with nothing above it, so + // the next write lands there and the log stays dense. Below a published + // event the position stays a hole instead: see the SlotBook's own tests, + // where the ordering can be forced. const runId = await newSlotRun(); - const [rejected, accepted] = await Promise.allSettled([ + await expect( storage.events.create(runId, { eventType: 'step_completed', specVersion: SPEC_VERSION_SLOT_IDENTITY, correlationId: 'step_never_created', eventData: { output: new Uint8Array() }, - }), - createStep(runId, 'step_a'), - ]); - expect(rejected.status).toBe('rejected'); - expect(accepted.status).toBe('fulfilled'); - await createStep(runId, 'step_b'); - const slots = await slotsOf(runId); - expect(slots).toHaveLength(3); - expect(slots[0]).toBe(FIRST_SLOT); - // Both concurrent writers took a position, one abandoned its own, and the - // third write went above them both. - expect(slots[2]).toBe(FIRST_SLOT + 3); - expect(slots[1]).toBeGreaterThan(slots[0]); - expect(slots[1]).toBeLessThan(slots[2]); + }) + ).rejects.toThrow(); + await createStep(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT, FIRST_SLOT + 1]); }); }); @@ -482,3 +472,104 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); }); }); + +async function createHook(runId: string, hookId: string, token: string) { + await storage.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: hookId, + eventData: { token, hookId }, + }); + return { hookId, token }; +} + +/** + * One writer of a resume. A resume has two of these — `resumeHook`'s direct + * write and the queue consumer's re-ensure — and they are byte-for-byte + * identical, which is what makes them converge on one event. + */ +function resume( + from: Storage, + runId: string, + hook: { hookId: string; token: string }, + resumeId: string, + payload = new Uint8Array([1, 2, 3]) +) { + return from.events.create( + runId, + { + eventType: 'hook_received', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: hook.hookId, + eventData: { token: hook.token, payload }, + }, + { resumeId, resumePayloadDigest: `digest_${resumeId}` } + ); +} + +async function hookReceivedCount(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.filter((event) => event.eventType === 'hook_received').length; +} + +describe('a resume with two writers', () => { + it('leaves no hole behind a converged redelivery', async () => { + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + + const first = await resume(storage, runId, hook, 'resume_1'); + const second = await resume(storage, runId, hook, 'resume_1'); + expect(second.event.eventId).toBe(first.event.eventId); + + // The redelivery took a position to write at and then converged on the + // event that already existed, so it published nothing. Unless it hands that + // position back, the next event lands above a hole and the run can no longer + // prove its log is complete. + await resume(storage, runId, hook, 'resume_2'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + + it('commits one event when both writers race across instances', async () => { + // The two writers of a resume are usually two processes: the request that + // resumed the hook and the queue consumer that replays the run. They share + // the data directory and nothing else, so the claim on disk is the only + // thing that can converge them. + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + const other = createStorage(testDir); + + const [a, b] = await Promise.all([ + resume(storage, runId, hook, 'resume_1'), + resume(other, runId, hook, 'resume_1'), + ]); + + expect(b.event.eventId).toBe(a.event.eventId); + await expect(hookReceivedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('numbers distinct resumes of one hook densely under load', async () => { + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + const other = createStorage(testDir); + const resumeIds = ['r_0', 'r_1', 'r_2', 'r_3', 'r_4']; + + const results = await Promise.all( + resumeIds.flatMap((resumeId, index) => { + const payload = new Uint8Array([index]); + return [ + resume(storage, runId, hook, resumeId, payload), + resume(other, runId, hook, resumeId, payload), + ]; + }) + ); + + // Each resume's pair shares one event; the five resumes are distinct and sit + // in the five positions above the hook. + expect(new Set(results.map((r) => r.event.eventId)).size).toBe( + resumeIds.length + ); + await expect(hookReceivedCount(runId)).resolves.toBe(resumeIds.length); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5, 6, 7]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 318b95c204..c7ade143a9 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -145,6 +145,20 @@ describe('reserve', () => { expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); }); + it('clears events another instance published after this book was read', async () => { + // The inversion this rules out, and the reason every allocation re-reads the + // log: a second storage instance over the same data directory publishes + // without touching this book, so a book that trusted itself would hand out a + // position below events that already exist. The publish cannot catch it — + // the position is genuinely free — and the replay that reads a completion + // before its own start diverges for good. + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3, 4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + it('honours a floor above where the book has reached', async () => { // start() publishes the run entity before its `run_created` event and issues // the queue send in parallel, so the delivery's `run_started` can allocate @@ -178,6 +192,26 @@ describe('release', () => { await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); }); + it('recycles the top slot, so an abandoned write leaves no hole', async () => { + // Nothing was handed out above it and nothing on disk sits above it, so the + // position provably precedes no event and can be handed out again. The common + // abandonment is a create that converged on another writer's event before + // publishing anything, and leaving those behind would make a log that lost + // nothing unable to prove it. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBe(abandoned); + }); + + it('does not recycle the top slot once the log has moved above it', async () => { + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + await writeEvents(1, abandoned, abandoned + 1); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBe(abandoned + 2); + }); + it('does not resurrect a slot that was published', async () => { const book = createSlotBook(basedir); const slot = await book.reserve(RUN_ID); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 1b1822e704..2b5c3948b4 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -9,26 +9,34 @@ * * Filling a hole is what that rules out, and it is worth naming why, because the * alternative looks appealing (it keeps the log dense). A position left unwritten - * by an abandoned reservation sits below events that are already published. Hand - * it to the next caller and a `step_completed` lands below its own + * by an abandoned reservation can sit below events that are already published. + * Hand it to the next caller and a `step_completed` lands below its own * `step_started`; the replay reaches a completion for a step it has not started * and diverges, and every later replay diverges the same way. A hole costs a * reader the ability to prove its copy of the log is complete. An inversion * costs the run. * + * An abandoned reservation is recycled in the one case where it provably sits + * below nothing: it was the top of the book when it was released, so no sibling + * was ever handed a position above it and none was read from disk above it. That + * keeps the common abandonment (a create that converged on another writer's event + * before publishing anything) from leaving the log permanently un-provable. + * * Three properties do the work: * - * - Handing out a slot is a *synchronous* set operation, so concurrent - * callers in one process get distinct slots with no lock. The only await is - * seeding from disk, which is memoized per run. - * - An allocation picks the position above the highest one the book knows of, - * written or outstanding, and that ceiling never descends. A reservation that - * is abandoned (its create threw a validation error) leaves its position - * unused rather than being recycled below a sibling that already published. - * - The event publish is `writeExclusive`, which is the authority. The book is - * a hint: when it turns out to be stale (another process wrote the slot), - * the publish fails and the caller is told so, rather than a duplicate being - * written or a slot being skipped. + * - An allocation reads the log's tail from disk and picks the position above + * both it and the highest one the book knows of, written or outstanding. + * Reading disk every time is what makes the order safe across storage + * instances: a book that has not seen another instance's writes would + * otherwise hand out a position *below* them, and that inversion is silent + * because the position is genuinely free (see below for what it costs). + * - Once the tail is in hand, handing out a slot is a *synchronous* set + * operation, so concurrent callers in one process get distinct slots with no + * lock: each bumps the ceiling before the next one reads it. + * - The event publish is `writeExclusive`, which is the authority. The tail + * read is a hint: when it turns out to be stale (another process wrote the + * slot between the read and the publish), the publish fails and the caller is + * told so, rather than a duplicate being written or a slot being skipped. * * The book is per storage instance, and two instances may share a data * directory (the cross-process convergence tests rely on exactly that). Their @@ -70,9 +78,9 @@ export interface SlotBook { */ usesSlots(runId: string): Promise; /** - * Reserves the position above every one this book knows of for `runId`, and at - * or above `minSlot`. Distinct for every concurrent caller; the publish still - * has to prove the position was actually free. + * Reserves the position above the run's tail on disk and above every one this + * book knows of, and at or above `minSlot`. Distinct for every concurrent + * caller; the publish still has to prove the position was actually free. * * `minSlot` defaults to the position above the run's first slot, which is * reserved for its own `run_created`: that event needs no allocation, and it @@ -88,8 +96,8 @@ export interface SlotBook { */ claim(runId: string, slot: number): void; /** - * The highest position this run has published, seeding from disk if the run - * has not been read yet, or `FIRST_SLOT - 1` for a log with no events. + * The highest position this run has published, read from the log, or + * `FIRST_SLOT - 1` for a log with no events. * * This is the tail a claim has to clear. "Free" is not the property a claim * needs: allocation is append-only, so a position left unwritten by an @@ -113,9 +121,12 @@ export interface SlotBook { highestWritten(runId: string): Promise; /** * Forgets a reserved or claimed slot whose publish is never going to happen, - * so nothing waits on it. The position itself is not handed out again: it may - * already sit below a sibling that published, and recycling it there would put - * a later event below an earlier one. + * so nothing waits on it. + * + * The position is handed out again only if it is still the highest one this + * book knows of, where it provably sits below nothing. Anywhere else it stays + * empty: it may already sit below a sibling that published, and recycling it + * there would put a later event below an earlier one. */ release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ @@ -142,6 +153,8 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { const books = new Map(); /** runId → in-flight seed scan, so concurrent first callers share one scan. */ const seeds = new Map>(); + /** runId → in-flight tail read, so a burst of writers shares one scan. */ + const syncs = new Map>(); /** * runId → slots claimed while the run had no book yet, so the book the next * allocation seeds starts out holding them. A claim is synchronous and a seed @@ -181,6 +194,47 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return book; } + /** Folds the run's published positions on disk into the book kept for it. */ + async function merge(runId: string, book: RunSlots): Promise { + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + } + return book; + } + + /** + * The run's book with the log's current tail folded in. + * + * Every allocation and every tail read goes through this, because the book on + * its own only knows what *this* instance did: another instance sharing the + * data directory publishes without touching it. One scan is shared by all + * callers waiting on it, so a burst of concurrent writers pays for one readdir + * and still leaves with distinct positions (the scan resolves first, the + * synchronous takes follow). + */ + function synced(runId: string): Promise { + const inFlight = syncs.get(runId); + if (inFlight) { + return inFlight; + } + // Seeding reads the log itself, so a book being opened for the first time is + // already current and a second scan would buy nothing. + const seeding = !books.has(runId); + const opened = open(runId); + const scan = + seeding && opened instanceof Promise + ? opened + : Promise.resolve(opened).then((book) => merge(runId, book)); + const pending = scan.finally(() => syncs.delete(runId)); + syncs.set(runId, pending); + return pending; + } + /** The run's book, seeding it from disk once for all concurrent callers. */ function open(runId: string): RunSlots | Promise { const known = books.get(runId); @@ -234,10 +288,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { - const opened = open(runId); - // Awaiting a book that is already in hand would yield to the microtask - // queue and let a concurrent caller take the same slot. - return take(opened instanceof Promise ? await opened : opened, minSlot); + // The take is synchronous once the scan resolves, so two concurrent + // callers sharing one scan still leave with different positions. + return take(await synced(runId), minSlot); }, claim(runId, slot) { @@ -255,7 +308,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, async highestWritten(runId) { - const book = await open(runId); + const book = await synced(runId); return Math.max(FIRST_SLOT - 1, ...book.written); }, @@ -265,10 +318,18 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { if (!book) { return; } - // The ceiling stays where it is: this position may already sit below one a - // sibling published, and handing it out again would order a later event - // before an earlier one. book.outstanding.delete(slot); + // The position is handed out again only while it is still the top of the + // book: nothing was handed out above it and nothing was read from disk + // above it, so it cannot land below an event that already exists. Below + // the top the ceiling stays where it is, and the position stays a hole. + if (book.ceiling === slot) { + book.ceiling = Math.max( + FIRST_SLOT - 1, + ...book.written, + ...book.outstanding + ); + } }, observe(runId, eventId) { @@ -294,14 +355,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { // Nothing cached to correct; the next reservation seeds from disk. return; } - for (const eventId of await listRunEventIds(basedir, runId, tag)) { - const slot = slotFromId(eventId); - if (slot !== undefined) { - book.written.add(slot); - book.outstanding.delete(slot); - book.ceiling = Math.max(book.ceiling, slot); - } - } + await merge(runId, book); }, forget(runId) { From 20e2752e1dfda8db2a66bdce403c38d6855166ba Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 15:31:15 -0700 Subject: [PATCH 03/14] Make slot event identity unconditional and squash the slot changesets Removes WORKFLOW_SLOT_IDENTITY: every world stamps SPEC_VERSION_SLOT_IDENTITY on the runs it creates, so mintedSpecVersion() and the env var it read are gone. Runs created before the switch keep their own scheme, which is what SPEC_VERSION_MAX_SUPPORTED is for. Co-Authored-By: Claude Opus 5 --- .changeset/inline-claim-reclaim.md | 5 --- .changeset/local-resume-one-event.md | 5 --- .changeset/local-slot-order.md | 5 --- .changeset/slimy-weeks-act.md | 2 - .changeset/slot-claims-tail-tight.md | 7 ---- .changeset/slot-event-identity-client.md | 9 ---- .changeset/slot-event-identity-worlds.md | 8 ---- .changeset/slot-event-identity.md | 11 +++++ .changeset/slot-identity-default-on.md | 8 ---- .changeset/slot-restart-budget.md | 6 --- .changeset/slot-restart-cursor-top-up.md | 6 --- .../docs/v5/configuration/runtime-tuning.mdx | 11 +---- packages/world-local/src/index.ts | 12 +++--- packages/world-postgres/src/index.ts | 12 +++--- packages/world-vercel/src/index.ts | 12 +++--- packages/world/src/index.ts | 2 - packages/world/src/spec-version.test.ts | 42 +++++-------------- packages/world/src/spec-version.ts | 40 +++--------------- 18 files changed, 47 insertions(+), 156 deletions(-) delete mode 100644 .changeset/inline-claim-reclaim.md delete mode 100644 .changeset/local-resume-one-event.md delete mode 100644 .changeset/local-slot-order.md delete mode 100644 .changeset/slimy-weeks-act.md delete mode 100644 .changeset/slot-claims-tail-tight.md delete mode 100644 .changeset/slot-event-identity-client.md delete mode 100644 .changeset/slot-event-identity-worlds.md create mode 100644 .changeset/slot-event-identity.md delete mode 100644 .changeset/slot-identity-default-on.md delete mode 100644 .changeset/slot-restart-budget.md delete mode 100644 .changeset/slot-restart-cursor-top-up.md diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md deleted file mode 100644 index 9d57541efc..0000000000 --- a/.changeset/inline-claim-reclaim.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@workflow/core': patch ---- - -Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/.changeset/local-resume-one-event.md b/.changeset/local-resume-one-event.md deleted file mode 100644 index e3e888b290..0000000000 --- a/.changeset/local-resume-one-event.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@workflow/world-local': patch ---- - -Record one `hook_received` per hook delivery when the delivery is written twice concurrently, and keep the position-numbered log dense across it diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md deleted file mode 100644 index 148f7df032..0000000000 --- a/.changeset/local-slot-order.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@workflow/world-local': patch ---- - -Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/.changeset/slimy-weeks-act.md b/.changeset/slimy-weeks-act.md deleted file mode 100644 index a845151cc8..0000000000 --- a/.changeset/slimy-weeks-act.md +++ /dev/null @@ -1,2 +0,0 @@ ---- ---- diff --git a/.changeset/slot-claims-tail-tight.md b/.changeset/slot-claims-tail-tight.md deleted file mode 100644 index 346b2c92de..0000000000 --- a/.changeset/slot-claims-tail-tight.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@workflow/core': patch -'@workflow/world-local': patch -'@workflow/world-postgres': patch ---- - -Take event slot claims one at a time against the event log's tail, so a replay that decided from a log missing an event is rejected instead of committing. diff --git a/.changeset/slot-event-identity-client.md b/.changeset/slot-event-identity-client.md deleted file mode 100644 index e6d2c5feee..0000000000 --- a/.changeset/slot-event-identity-client.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -'@workflow/world-vercel': minor -'@workflow/core': minor -'@workflow/errors': minor -'@workflow/world': minor -'workflow': minor ---- - -Event creations on runs that number events by slot now claim their own event id and merge, replay and re-claim when a `SlotConflictError` shows another writer took it first. diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md deleted file mode 100644 index 863a2f860b..0000000000 --- a/.changeset/slot-event-identity-worlds.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@workflow/world': minor -'@workflow/world-local': minor -'@workflow/world-postgres': minor -'@workflow/core': minor ---- - -Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/.changeset/slot-event-identity.md b/.changeset/slot-event-identity.md new file mode 100644 index 0000000000..9e1e6717fa --- /dev/null +++ b/.changeset/slot-event-identity.md @@ -0,0 +1,11 @@ +--- +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/world': minor +'@workflow/errors': minor +'@workflow/core': minor +'workflow': minor +--- + +Number new runs' events by position instead of by ULID, in every World, so a reader can prove its copy of an event log is complete. A creation whose position another writer already took is rejected with a `SlotConflictError`, and the runtime merges the events it was missing, replays, and claims a free position. diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md deleted file mode 100644 index 55e3973586..0000000000 --- a/.changeset/slot-identity-default-on.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@workflow/world': minor -'@workflow/world-vercel': minor -'@workflow/world-local': minor -'@workflow/world-postgres': minor ---- - -Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/.changeset/slot-restart-budget.md b/.changeset/slot-restart-budget.md deleted file mode 100644 index 6a4c52c909..0000000000 --- a/.changeset/slot-restart-budget.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Let a run that numbers its events by position absorb more concurrent-write rejections in one invocation, instead of falling back to a delayed re-invocation diff --git a/.changeset/slot-restart-cursor-top-up.md b/.changeset/slot-restart-cursor-top-up.md deleted file mode 100644 index 18752ca81f..0000000000 --- a/.changeset/slot-restart-cursor-top-up.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Recover faster from a concurrent write on runs that number their events by slot, by topping the event log up from its cursor instead of reloading it in full diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index d094be19e6..f3d439d213 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -59,7 +59,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` -- Default: `3`, or `12` on a run numbering its events by position (see [`WORKFLOW_SLOT_IDENTITY`](#workflow_slot_identity)) +- Default: `3`, or `12` on a run numbering its events by position (see [`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) - How many times a single invocation restarts its replay in-process after a rejected event creation before it falls back to a re-invocation. - A restart reloads the event log and rebuilds the workflow from scratch, so it costs a replay but no queue round trip. A World may attach the missing events to its rejection, in which case the first restart needs no event-log request at all. - A run numbering its events by position reads only the page after its cursor instead of reloading the log, since positions are allocated in order and every event it was missing sorts above what it already has. Restarts are cheap enough there that the higher default is worth taking before a re-invocation and its delay. @@ -89,15 +89,6 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: `400` - Ceiling on the wait described above. -### `WORKFLOW_SLOT_IDENTITY` - -- Default: enabled -- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. -- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the run replays from the top against a log that now includes them. -- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. -- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. -- Set `0` or `false` to disable, which numbers new runs by ULID as before. - ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 3c7c0beefe..8acfc52751 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,10 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; +import { + reenqueueActiveRuns, + SPEC_VERSION_SLOT_IDENTITY, +} from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,10 +75,9 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - // What this world stamps on new runs: slot identity, unless - // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes - // whatever this says. - specVersion: mintedSpecVersion(), + // What this world stamps on new runs: slot identity. Every world reads + // both schemes whatever this says. + specVersion: SPEC_VERSION_SLOT_IDENTITY, capabilities: { // world-local deduplicates concurrent `hook_received` writes sharing a // `(runId, resumeId)` via a filesystem sidecar claim (see diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 926d752d3b..071229cbf0 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,8 @@ import type { Storage, World } from '@workflow/world'; -import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; +import { + reenqueueActiveRuns, + SPEC_VERSION_SLOT_IDENTITY, +} from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,10 +66,9 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - // What this world stamps on new runs: slot identity, unless - // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes - // whatever this says. - specVersion: mintedSpecVersion(), + // What this world stamps on new runs: slot identity. Every world reads + // both schemes whatever this says. + specVersion: SPEC_VERSION_SLOT_IDENTITY, ...storage, ...streamer, ...queue, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 78f4412822..00b5aeee99 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { mintedSpecVersion } from '@workflow/world'; +import { SPEC_VERSION_SLOT_IDENTITY } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,12 +29,10 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // What this world stamps on new runs: slot identity (spec v6) unless - // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side - // zstd/gzip payload compression over a superset of the v4 attributes. - // Either way this world reads both, so the stamp only decides how the runs - // it creates from here on are numbered. - specVersion: mintedSpecVersion(), + // What this world stamps on new runs: slot identity (spec v6). It reads + // every earlier version too, so this only decides how the runs it creates + // from here on are numbered. + specVersion: SPEC_VERSION_SLOT_IDENTITY, capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index f0d74f6454..49205702d9 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -122,9 +122,7 @@ export { export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, - mintedSpecVersion, requiresNewerWorld, - SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_MAX_SUPPORTED, diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 0125e2fb1c..3ec2152838 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,7 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, - mintedSpecVersion, requiresNewerWorld, - SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_MAX_SUPPORTED, @@ -18,10 +16,9 @@ describe('spec version constants', () => { expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); - it('can read a newer spec version than it mints', () => { - // Slot identity is readable by every world before any world mints it, so - // that turning it on for new runs cannot make those same worlds reject - // them. Once slots are the default the two constants coincide again. + it('can read a newer spec version than the floor it requires', () => { + // Worlds stamp slot identity on new runs while still reading everything + // back to the compression version, so the ceiling sits above the floor. expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( SPEC_VERSION_CURRENT @@ -74,31 +71,12 @@ describe('isLegacySpecVersion', () => { }); }); -describe('mintedSpecVersion', () => { - it('mints slot identity by default', () => { - expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); - }); - - it('mints the previous version when the flag is switched off', () => { - for (const value of ['0', 'false']) { - expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_CURRENT - ); - } - }); - - it('treats any other value as on', () => { - // An unset-but-present variable is the shape a shell leaves behind, and it - // must not silently switch a deployment's event identity scheme. Opting - // out takes an explicit `0`/`false`. - for (const value of ['', '1', 'true', 'yes']) { - expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_SLOT_IDENTITY - ); - } - }); - - it('mints nothing a world cannot read', () => { - expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); +describe('the version worlds stamp on new runs', () => { + it('is slot identity, and is readable by the worlds that stamp it', () => { + // Every world's `specVersion` is SPEC_VERSION_SLOT_IDENTITY. A world that + // rejected spec-6 would reject the runs it had just stamped spec-6 itself, + // at their first event after run_created. + expect(SPEC_VERSION_SLOT_IDENTITY).toBeGreaterThan(SPEC_VERSION_CURRENT); + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); }); }); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index 874bda0e24..2d8f4bed5b 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -51,10 +51,9 @@ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; * Current spec version (event-sourced architecture with native attributes * and compressed payloads). * - * The floor a world stamps on new runs, and a *lower* bar than the newest - * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a - * world actually stamps comes from {@link mintedSpecVersion}; this is what it - * falls back to when slot identity is switched off. + * The floor a world has to support, and a *lower* bar than the version worlds + * stamp on the runs they create ({@link SPEC_VERSION_SLOT_IDENTITY}) or the + * newest one this build can read ({@link SPEC_VERSION_MAX_SUPPORTED}). */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; @@ -63,40 +62,13 @@ export const SPEC_VERSION_CURRENT = * Newest spec version this build can read. Runs above it are rejected outright * by {@link requiresNewerWorld} rather than misread. * - * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to - * read a version before anything may mint it, and because a world that mints - * slot identity still has to read the spec-5 runs it created before the switch. - * Worlds opt into minting individually, via the `specVersion` they declare. + * Distinct from {@link SPEC_VERSION_CURRENT} because a world reads further back + * than the floor it requires, and because a world that stamps slot identity + * still has to read the spec-5 runs it created before the switch. */ export const SPEC_VERSION_MAX_SUPPORTED = SPEC_VERSION_SLOT_IDENTITY as SpecVersion; -/** - * Environment variable that opts new runs out of slot identity. - * - * Read per `createWorld()` call rather than at module load, so a test or a - * single process can create worlds in both modes. - */ -export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; - -/** - * The spec version a world should stamp on the runs it creates: slot identity - * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case - * {@link SPEC_VERSION_CURRENT}. - * - * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this - * returns, so turning the flag off in one place does not make the runs another - * process created unreadable here. - */ -export function mintedSpecVersion( - env: Record = process.env -): SpecVersion { - const value = env[SLOT_IDENTITY_ENV_VAR]; - return value === '0' || value === 'false' - ? SPEC_VERSION_CURRENT - : SPEC_VERSION_SLOT_IDENTITY; -} - /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation From b00b729e73152d177c32ece29c7e1039e990a7ff Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 16:16:45 -0700 Subject: [PATCH 04/14] Drop two changesets folded into the slot identity entry --- .changeset/precondition-restart-backoff.md | 6 ------ .changeset/tidy-moons-observe.md | 5 ----- 2 files changed, 11 deletions(-) delete mode 100644 .changeset/precondition-restart-backoff.md delete mode 100644 .changeset/tidy-moons-observe.md diff --git a/.changeset/precondition-restart-backoff.md b/.changeset/precondition-restart-backoff.md deleted file mode 100644 index 8e739cd8f0..0000000000 --- a/.changeset/precondition-restart-backoff.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Space out in-process replay restarts with a randomized backoff so concurrent replays of one run stop contending in lockstep diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md deleted file mode 100644 index 9c6c5b7692..0000000000 --- a/.changeset/tidy-moons-observe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@workflow/core': patch ---- - -Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs From ee83cb4a111fab70426588e09fd0529e7899d67a Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 17:29:23 -0700 Subject: [PATCH 05/14] Make per-kind correlation ids the only scheme Correlation ids always draw from a sequence per entity family. The WORKFLOW_PER_KIND_CORRELATION_IDS opt-out is gone, along with the shared run-wide sequence it selected and the test helper that pinned suites to it. Test fixtures that hardcoded ids the shared sequence minted are regenerated against their kind's own sequence, and the two sites in workflow.test.ts that derived expected ids from a raw monotonicFactory now derive them from the run's generator. Folds the per-kind changeset into the slot-event-identity one. --- .changeset/per-kind-correlation-ids.md | 6 -- .changeset/slot-event-identity.md | 2 +- packages/core/src/abort-consistency.test.ts | 9 +- packages/core/src/abort-controller.test.ts | 9 +- .../core/src/abort-replay-ordering.test.ts | 9 +- .../async-deserialization-ordering.test.ts | 70 +++++++------- .../core/src/correlation-id-replay.test.ts | 57 +++++------- packages/core/src/correlation-id.test.ts | 64 +++---------- packages/core/src/correlation-id.ts | 40 +------- .../src/delivery-barrier-coverage.test.ts | 7 -- .../core/src/hook-sleep-interaction.test.ts | 6 -- .../runtime/precondition-guard-replay.test.ts | 3 - .../resume-hook.consumer-preload.test.ts | 3 - .../runtime/wait-completion-replay.test.ts | 3 - .../core/src/step-delivery-hop-count.test.ts | 6 -- .../core/src/step-delivery-ordering.test.ts | 6 -- .../src/step-hydration-memoization.test.ts | 12 +-- packages/core/src/step.test.ts | 44 ++++----- .../src/test-support/correlation-id-scheme.ts | 26 ------ packages/core/src/workflow.test.ts | 40 ++++---- packages/core/src/workflow.ts | 20 ++-- packages/core/src/workflow/hook.test.ts | 92 +++++++++---------- packages/core/src/workflow/sleep.test.ts | 52 +++++------ workbench/nextjs-turbopack/vercel.json | 3 +- 24 files changed, 184 insertions(+), 405 deletions(-) delete mode 100644 .changeset/per-kind-correlation-ids.md delete mode 100644 packages/core/src/test-support/correlation-id-scheme.ts diff --git a/.changeset/per-kind-correlation-ids.md b/.changeset/per-kind-correlation-ids.md deleted file mode 100644 index 2e13a31a73..0000000000 --- a/.changeset/per-kind-correlation-ids.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Add env option to split correlation ID derivation into per-entity-type sequential ULIDs, instead of sharing one derivation source diff --git a/.changeset/slot-event-identity.md b/.changeset/slot-event-identity.md index 9e1e6717fa..3d1fda51cb 100644 --- a/.changeset/slot-event-identity.md +++ b/.changeset/slot-event-identity.md @@ -8,4 +8,4 @@ 'workflow': minor --- -Number new runs' events by position instead of by ULID, in every World, so a reader can prove its copy of an event log is complete. A creation whose position another writer already took is rejected with a `SlotConflictError`, and the runtime merges the events it was missing, replays, and claims a free position. +Number new runs' events by position instead of by ULID, in every World, so a reader can prove its copy of an event log is complete. A creation whose position another writer already took is rejected with a `SlotConflictError`, and the runtime merges the events it was missing, replays, and claims a free position. Correlation IDs now come from a sequence per entity type rather than one shared across all of them, so creating a hook or a sleep no longer renames the steps after it. diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index d99b67de1d..69b9c98075 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -9,12 +9,8 @@ import type { Event, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -36,7 +32,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test-abort-consistency', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -51,8 +46,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index 2ced0b4aca..4b300dd414 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -10,12 +10,8 @@ import { ReplayDivergenceError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS, EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -34,7 +30,6 @@ function setupWorkflowContext( seed: 'test-abort', fixedTimestamp: 1714857600000, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -49,8 +44,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8e7a1a9e98..b87dd7bfee 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -25,12 +25,8 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { scheduleWhenIdle, @@ -69,7 +65,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test-abort-replay-ordering', fixedTimestamp: 1714857600000, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -84,8 +79,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 4ad65f662c..618cad1655 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -1,7 +1,6 @@ import { FatalError } from '@workflow/errors'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; import { createCorrelationIdGenerator } from './correlation-id.js'; @@ -46,7 +45,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { if (hostRegistry) { (context.globalThis as any)[WORKFLOW_CLASS_REGISTRY] = hostRegistry; } - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -61,10 +59,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -100,7 +94,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -111,7 +105,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', result: resultB, @@ -176,7 +170,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step1', result: results[0], @@ -187,7 +181,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'step2', result: results[1], @@ -198,7 +192,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9X', eventData: { stepName: 'step3', result: results[2], @@ -271,7 +265,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: payloadA, @@ -282,7 +276,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: payloadB, @@ -293,7 +287,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -363,7 +357,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -374,7 +368,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', error: errorB, @@ -385,7 +379,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9X', eventData: { stepName: 'stepC', result: resultC, @@ -450,16 +444,16 @@ describe('async deserialization ordering', () => { // Correlation IDs from the deterministic ULID generator const correlationIds = [ - 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCY', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCZ', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD0', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD1', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD2', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD3', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD4', + 'step_01K11TFZ62CHHYKN8SS4KKNC9V', + 'step_01K11TFZ62CHHYKN8SS4KKNC9W', + 'step_01K11TFZ62CHHYKN8SS4KKNC9X', + 'step_01K11TFZ62CHHYKN8SS4KKNC9Y', + 'step_01K11TFZ62CHHYKN8SS4KKNC9Z', + 'step_01K11TFZ62CHHYKN8SS4KKNCA0', + 'step_01K11TFZ62CHHYKN8SS4KKNCA1', + 'step_01K11TFZ62CHHYKN8SS4KKNCA2', + 'step_01K11TFZ62CHHYKN8SS4KKNCA3', + 'step_01K11TFZ62CHHYKN8SS4KKNCA4', ]; const events: Event[] = results.map((result, i) => ({ @@ -524,7 +518,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -535,7 +529,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z') }, createdAt: new Date(), }, @@ -543,7 +537,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -553,7 +547,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepC', result: resultC, @@ -619,7 +613,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', }, @@ -629,7 +623,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', }, @@ -639,7 +633,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -650,7 +644,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', result: resultB, @@ -712,7 +706,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload: new Uint8Array([101, 110, 99, 114]), // "encr" without a key }, @@ -749,7 +743,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload }, createdAt: new Date(), }, @@ -757,7 +751,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt }, createdAt: new Date(), }, diff --git a/packages/core/src/correlation-id-replay.test.ts b/packages/core/src/correlation-id-replay.test.ts index 92b1db81b6..f07bf10a7f 100644 --- a/packages/core/src/correlation-id-replay.test.ts +++ b/packages/core/src/correlation-id-replay.test.ts @@ -1,6 +1,5 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -19,22 +18,18 @@ import { createSleep } from './workflow/sleep.js'; * log carrying the ids a same-seeded replay derives. * * The rest of the replay suites author their event logs with literal correlation - * ids from the shared sequence and pin themselves to it. These fixtures derive - * their ids instead, so they hold under either scheme. + * ids. These fixtures derive theirs instead, so they say something about the + * minting primitives rather than about a captured snapshot of them. */ const SEED = 'test'; const FIXED_TIMESTAMP = 1753481739458; -function setupWorkflowContext( - events: Event[], - perKind: boolean -): WorkflowOrchestratorContext { +function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const context = createContext({ seed: SEED, fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); return { runId: 'wrun_test', encryptionKey: undefined, @@ -48,8 +43,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: SEED, fixedTimestamp: FIXED_TIMESTAMP, - positional: () => ulid(FIXED_TIMESTAMP), - perKind, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -67,10 +60,9 @@ function setupWorkflowContext( * we are after. */ function probeStepId( - perKind: boolean, before?: (ctx: WorkflowOrchestratorContext) => void ): string { - const ctx = setupWorkflowContext([], perKind); + const ctx = setupWorkflowContext([]); before?.(ctx); void createUseStep(ctx)('add')(1, 2).catch(() => {}); const item = [...ctx.invocationsQueue.values()].find( @@ -89,34 +81,27 @@ function createHookAndSleep(ctx: WorkflowOrchestratorContext): void { describe('correlation ids through the replay primitives', () => { it('keeps a step id when a hook and a sleep are created before it', () => { - expect(probeStepId(true, createHookAndSleep)).toBe(probeStepId(true)); - }); - - it('renumbers that step under one sequence shared by every kind', () => { - // The failure this PR removes, and the reason the assertion above is worth - // making: with a shared sequence the hook and the sleep consume the two - // ordinals the step would otherwise have drawn from. - expect(probeStepId(false, createHookAndSleep)).not.toBe(probeStepId(false)); + // Under one sequence shared by every kind, the hook and the sleep consume + // the two ordinals the step would otherwise have drawn, so a replay that + // disagreed about either renamed the step. + expect(probeStepId(createHookAndSleep)).toBe(probeStepId()); }); it('consumes a step_completed authored with the derived id', async () => { - const correlationId = probeStepId(true, createHookAndSleep); - const ctx = setupWorkflowContext( - [ - { - eventId: 'evnt_0', - runId: 'wrun_test', - eventType: 'step_completed', - correlationId, - eventData: { - stepName: 'add', - result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), - }, - createdAt: new Date(), + const correlationId = probeStepId(createHookAndSleep); + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_test', + eventType: 'step_completed', + correlationId, + eventData: { + stepName: 'add', + result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), }, - ], - true - ); + createdAt: new Date(), + }, + ]); createHookAndSleep(ctx); await expect(createUseStep(ctx)('add')(1, 2)).resolves.toBe(3); expect(ctx.onWorkflowError).not.toHaveBeenCalled(); diff --git a/packages/core/src/correlation-id.test.ts b/packages/core/src/correlation-id.test.ts index db95c65df4..07d63ffec0 100644 --- a/packages/core/src/correlation-id.test.ts +++ b/packages/core/src/correlation-id.test.ts @@ -1,31 +1,20 @@ -import { decodeTime, monotonicFactory } from 'ulid'; +import { decodeTime } from 'ulid'; import { describe, expect, it } from 'vitest'; import { CORRELATION_ID_LENGTH, type CorrelationIdKind, createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, } from './correlation-id.js'; const SEED = 'wrun_abc:myWorkflow:dpl_123'; const FIXED_TIMESTAMP = 1753481739458; function makeGenerator( - overrides: { seed?: string; fixedTimestamp?: number; perKind?: boolean } = {} + overrides: { seed?: string; fixedTimestamp?: number } = {} ) { - // A stand-in for the run's shared sequence. Seeded so the positional mode is - // reproducible across the two generators a replay-stability test builds. - let counter = 0; - const ulid = monotonicFactory(() => { - counter = (counter * 1103515245 + 12345) % 2147483648; - return counter / 2147483648; - }); - const fixedTimestamp = overrides.fixedTimestamp ?? FIXED_TIMESTAMP; return createCorrelationIdGenerator({ seed: overrides.seed ?? SEED, - fixedTimestamp, - positional: () => ulid(fixedTimestamp), - perKind: overrides.perKind ?? true, + fixedTimestamp: overrides.fixedTimestamp ?? FIXED_TIMESTAMP, }); } @@ -110,45 +99,14 @@ describe('createCorrelationIdGenerator', () => { expect(withController('hook')).toBe(withoutController('hook')); }); - it('keeps every id on fixedTimestamp in both modes', () => { - // `monotonicFactory` returns `encodeTime(lastTime)` on its increment branch, - // so a single draw that omits the seed time latches the host wall clock and - // every later id in the run carries a timestamp that differs per replay. - // Stream ids used to be drawn that way. - for (const perKind of [true, false]) { - const generate = makeGenerator({ perKind }); - for (const kind of ['stream', 'stream', 'step', 'hook'] as const) { - expect(decodeTime(generate(kind))).toBe(FIXED_TIMESTAMP); - } - } - }); - - it('ignores the kind when per-kind sources are disabled', () => { - const generate = makeGenerator({ perKind: false }); - const shared = makeGenerator({ perKind: false }); - // Positional mode is one sequence for the whole run, so drawing `wait` - // consumes the ordinal the next `step` would otherwise have had. - expect(generate('step')).toBe(shared('step')); - expect(generate('wait')).toBe(shared('step')); - }); -}); - -describe('isPerKindCorrelationIdsEnabled', () => { - it('reads WORKFLOW_PER_KIND_CORRELATION_IDS, defaulting to disabled', () => { - const original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - try { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '1'; - expect(isPerKindCorrelationIdsEnabled()).toBe(true); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - } finally { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } + it('keeps every id on fixedTimestamp', () => { + // Stream ids are drawn without a seed time of their own. Drawn from a raw + // `monotonicFactory` they would latch the host wall clock into `lastTime` + // and every later id in the run would carry a timestamp that differs per + // replay, so they go through this generator like every other kind. + const generate = makeGenerator(); + for (const kind of ['stream', 'stream', 'step', 'hook'] as const) { + expect(decodeTime(generate(kind))).toBe(FIXED_TIMESTAMP); } }); }); diff --git a/packages/core/src/correlation-id.ts b/packages/core/src/correlation-id.ts index dbed6f8f5c..b9aa8fbef3 100644 --- a/packages/core/src/correlation-id.ts +++ b/packages/core/src/correlation-id.ts @@ -169,13 +169,7 @@ function deriveBody(seed: string, kind: CorrelationIdKind): string { return body; } -/** - * Builds a replay's correlation-id generator. - * - * `perKind: false` returns the run's single shared monotonic sequence and - * ignores the kind entirely, so both schemes go through one call path and the - * flag is the only difference between them. - */ +/** Builds a replay's correlation-id generator. */ export function createCorrelationIdGenerator(options: { /** * The run's replay-stable seed. Must not vary between replays of one run, and @@ -183,15 +177,8 @@ export function createCorrelationIdGenerator(options: { */ seed: string; fixedTimestamp: number; - /** The run's shared monotonic sequence, used as-is when `perKind` is false. */ - positional: () => string; - perKind: boolean; }): CorrelationIdGenerator { - const { seed, fixedTimestamp, positional, perKind } = options; - - if (!perKind) { - return positional; - } + const { seed, fixedTimestamp } = options; const time = encodeTime(fixedTimestamp, TIME_CHARS); const bodies = new Map(); @@ -209,26 +196,3 @@ export function createCorrelationIdGenerator(options: { /** Length of a ULID, exported so tests need not restate it. */ export const CORRELATION_ID_LENGTH = TIME_CHARS + BODY_CHARS; - -/** - * Whether each entity family draws correlation ids from its own sequence rather - * than from one sequence shared by the whole run. Off unless opted in, so an SDK - * upgrade alone never moves a run between schemes. - * - * The invariant either way: a run must replay under the scheme that minted its - * ids. A replay under the other scheme mints ids its own earlier events do not - * carry, so it can consume none of them and fails the run. Two things can break - * it, and both are about turning the flag on rather than about upgrading: - * - * - Enabling it while runs are in flight. On Vercel, skew protection keeps a run - * on the deployment that started it, so a run only ever sees the value baked - * into its own deployment. Elsewhere (world-postgres, world-local, a - * self-hosted process) nothing pins a run to the code that started it, so - * enable it during a quiet window. - * - A rolling deploy that leaves both values live, which puts two schemes on one - * run concurrently — the side-by-side append this whole mechanism exists to - * avoid. Roll the value out to the whole fleet at once. - */ -export function isPerKindCorrelationIdsEnabled(): boolean { - return process.env.WORKFLOW_PER_KIND_CORRELATION_IDS === '1'; -} diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index ef6232c8a7..83375a0cee 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -43,7 +43,6 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -68,7 +67,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test', fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -90,10 +88,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -121,7 +115,6 @@ function deterministicUlids(count: number): string[] { seed: 'test', fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return Array.from({ length: count }, () => ulid(workflowStartedAt)); } diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 754ee1a138..4960e7871e 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -2,7 +2,6 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -34,7 +33,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; // Forward onUnconsumedEvent through ctx.onWorkflowError so tests that wire @@ -62,10 +60,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index b245180b28..3a879bc8d1 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -48,7 +48,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { getPreconditionMaxInProcessRestarts, @@ -1032,8 +1031,6 @@ async function inlineClaimRejectionScenario() { }; } -pinSharedCorrelationIds(); - describe('precondition guard through the real replay loop', () => { let originalGuard: string | undefined; let originalRestartBound: string | undefined; diff --git a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts index 3e37a92f2e..c7897553ee 100644 --- a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts +++ b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts @@ -27,7 +27,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; @@ -253,8 +252,6 @@ async function runResumeConsumerScenario(options: { }; } -pinSharedCorrelationIds(); - describe('lazy hook resume consumer preload (Perf Option A)', () => { afterEach(() => { setWorld(undefined); diff --git a/packages/core/src/runtime/wait-completion-replay.test.ts b/packages/core/src/runtime/wait-completion-replay.test.ts index 27ca033d37..c4ca46982a 100644 --- a/packages/core/src/runtime/wait-completion-replay.test.ts +++ b/packages/core/src/runtime/wait-completion-replay.test.ts @@ -14,7 +14,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; @@ -435,8 +434,6 @@ function expectHookBranchQueued( ); } -pinSharedCorrelationIds(); - describe('workflow handler wait completion replay', () => { afterEach(() => { setWorld(undefined); diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index ff511bbcf9..96250021cc 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -27,7 +27,6 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -51,7 +50,6 @@ function setupWorkflowContext( seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -73,10 +71,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 64ffeaa276..ec08f72ef9 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -2,7 +2,6 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -98,7 +97,6 @@ function setupWorkflowContext( seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -122,10 +120,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index 8708c02e5d..df554d5455 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -1,6 +1,5 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -32,7 +31,6 @@ function setupWorkflowContext( seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -46,10 +44,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -68,7 +62,7 @@ async function makeStepEvents(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step1', result: await dehydrateStepReturnValue('one', 'wrun_test', undefined), @@ -79,7 +73,7 @@ async function makeStepEvents(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'step2', result: await dehydrateStepReturnValue('two', 'wrun_test', undefined), @@ -162,7 +156,7 @@ describe('step hydration memoization through the step consumer', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'obj', result: await dehydrateStepReturnValue( diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index faf852a54b..9f61e11e68 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -6,7 +6,6 @@ import { import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; import { createCorrelationIdGenerator } from './correlation-id.js'; @@ -50,7 +49,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { if (hostRegistry) { (context.globalThis as any)[WORKFLOW_CLASS_REGISTRY] = hostRegistry; } - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -65,10 +63,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -87,7 +81,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), @@ -113,7 +107,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -167,7 +161,7 @@ describe('createUseStep', () => { 1, 2, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCV", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9V", "stepName": "add", "type": "step", }, @@ -213,7 +207,7 @@ describe('createUseStep', () => { 1, 2, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCV", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9V", "stepName": "add", "type": "step", }, @@ -222,7 +216,7 @@ describe('createUseStep', () => { 3, 4, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCW", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9W", "stepName": "add", "type": "step", }, @@ -231,7 +225,7 @@ describe('createUseStep', () => { 5, 6, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCX", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9X", "stepName": "add", "type": "step", }, @@ -245,7 +239,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step//input.js//my_step_function', result: await dehydrateStepReturnValue( @@ -411,7 +405,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_created', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', input: new Uint8Array(), @@ -457,7 +451,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_created', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', input: new Uint8Array(), @@ -488,7 +482,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', }, @@ -528,7 +522,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_retrying', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: new Uint8Array(), @@ -566,7 +560,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', result: await dehydrateStepReturnValue(42, 'wrun_test', undefined), @@ -597,7 +591,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', result: await dehydrateStepReturnValue(42, 'wrun_test', undefined), @@ -627,7 +621,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', error: serializedError, @@ -663,7 +657,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -704,7 +698,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -745,7 +739,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -776,7 +770,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_completed', // Wrong event type for a step! - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { resumeAt: new Date(), }, @@ -796,7 +790,7 @@ describe('createUseStep', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for step'); - expect(workflowError?.message).toContain('step_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('step_01K11TFZ62CHHYKN8SS4KKNC9V'); expect(workflowError?.message).toContain('add'); expect(workflowError?.message).toContain('wait_completed'); }); diff --git a/packages/core/src/test-support/correlation-id-scheme.ts b/packages/core/src/test-support/correlation-id-scheme.ts deleted file mode 100644 index 32a000dfdd..0000000000 --- a/packages/core/src/test-support/correlation-id-scheme.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { afterAll, beforeAll } from 'vitest'; - -/** - * Pins a test file to the run-wide shared correlation-id sequence. - * - * Replay tests that drive the real `workflowEntrypoint` against an event log - * with hardcoded correlation ids can only match under the scheme those ids were - * minted by, and the fixtures in this repo predate per-kind sequences. Files - * whose fixture ids are derived rather than written out run under whichever - * scheme `WORKFLOW_PER_KIND_CORRELATION_IDS` selects; per-kind minting itself is - * covered by `correlation-id.test.ts`. - */ -export function pinSharedCorrelationIds(): void { - let original: string | undefined; - beforeAll(() => { - original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - }); - afterAll(() => { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } - }); -} diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fdeb2bd97a..e916a01bf6 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -2,8 +2,9 @@ import { types } from 'node:util'; import { HookConflictError, WorkflowRuntimeError } from '@workflow/errors'; import type { Event, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; -import { decodeTime, monotonicFactory } from 'ulid'; +import { decodeTime } from 'ulid'; import { afterEach, assert, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -13,15 +14,11 @@ import { dehydrateWorkflowArguments, hydrateWorkflowReturnValue, } from './serialization.js'; -import { pinSharedCorrelationIds } from './test-support/correlation-id-scheme.js'; -import { createContext } from './vm/index.js'; import { replayWorkflow, resumeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; -pinSharedCorrelationIds(); - describe('runWorkflow', () => { const getWorkflowTransformCode = (workflowName?: string) => `;globalThis.__private_workflows = new Map(); @@ -509,16 +506,15 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - // Derive the IDs the run should mint, in draw order, with the same seeded - // factory runWorkflow uses: the stream draw first, then the step that - // follows it. The step's recorded event only matches if the stream draw left - // the sequence on `fixedTimestamp`. + // Derive the IDs the run should mint with the same generator runWorkflow + // uses. The stream draw is what this test is about: it carries no seed time + // of its own, so it only lands on `fixedTimestamp` if the binding routes it + // through the run's generator. const seed = `${workflowRunId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`; const fixedTimestamp = +startedAt; - const vm = createContext({ seed, fixedTimestamp }); - const ulid = monotonicFactory(() => vm.globalThis.Math.random()); - const expectedStreamId = ulid(fixedTimestamp); - const stepCorr = `step_${ulid(fixedTimestamp)}`; + const generate = createCorrelationIdGenerator({ seed, fixedTimestamp }); + const expectedStreamId = generate('stream'); + const stepCorr = `step_${generate('step')}`; const events: Event[] = [ { @@ -5646,15 +5642,17 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - // Derive deterministic correlation IDs using the same seeded ULID - // factory runWorkflow uses internally, so events match what the runtime - // expects. The seed mirrors runWorkflow's `runId:workflowName:deploymentId`. + // Derive deterministic correlation IDs using the same generator runWorkflow + // uses internally, so events match what the runtime expects. The seed + // mirrors runWorkflow's `runId:workflowName:deploymentId`. const seed = `${workflowRunId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`; - const vm = createContext({ seed, fixedTimestamp: +startedAt }); - const ulid = monotonicFactory(() => vm.globalThis.Math.random()); - const hookCorr = `hook_${ulid(+startedAt)}`; - const waitCorr = `wait_${ulid(+startedAt)}`; - const stepCorr = `step_${ulid(+startedAt)}`; + const generate = createCorrelationIdGenerator({ + seed, + fixedTimestamp: +startedAt, + }); + const hookCorr = `hook_${generate('hook')}`; + const waitCorr = `wait_${generate('wait')}`; + const stepCorr = `step_${generate('step')}`; const payload1 = await dehydrateStepReturnValue( { done: false }, diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index f91a8b94fc..ec7f49bc1c 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -14,11 +14,7 @@ import { parseWorkflowName } from '@workflow/utils/parse-name'; import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; @@ -406,14 +402,11 @@ async function createWorkflowSession({ state satisfies never; }; - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); const generateCorrelationId = createCorrelationIdGenerator({ seed, - fixedTimestamp, // Correlation IDs must be replay-stable. `startedAt` differs between a // turbo delivery and a later server-backed replay, so use fixedTimestamp. - positional: () => ulid(fixedTimestamp), - perKind: isPerKindCorrelationIdsEnabled(), + fixedTimestamp, }); const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) @@ -535,11 +528,10 @@ async function createWorkflowSession({ // @ts-expect-error - `@types/node` says symbol is not valid, but it does work vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; // Serialization mints stream ids through this symbol, and calls it with no - // seed time. `monotonicFactory` returns `encodeTime(lastTime)` on its - // increment branch, so one such call latches the *host* wall clock into - // `lastTime` and every id the run mints afterwards carries that timestamp - // instead of `fixedTimestamp` — a value that differs on every replay. - // Binding the seed time here keeps the whole run on one replay-stable clock. + // seed time of its own. Routing it through the run's generator keeps those + // ids on `fixedTimestamp` like every other id the run mints, and puts them + // in their own sequence so serializing a stream does not renumber the + // entities created after it. // @ts-expect-error - `@types/node` says symbol is not valid, but it does work vmGlobalThis[STABLE_ULID] = () => generateCorrelationId('stream'); diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index b0f35ab2be..25bae24caf 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -6,7 +6,6 @@ import { import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { aliasSerializationClass, @@ -37,7 +36,6 @@ function setupWorkflowContext( // mirror that here so `hook.getConflict()` can construct the // conflicting run through the registry. aliasSerializationClass(RUN_CLASS_ID, Run, context.globalThis); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -53,10 +51,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -75,7 +69,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -101,7 +95,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', }, @@ -129,7 +123,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', payload: await dehydrateStepReturnValue( @@ -163,7 +157,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', }, @@ -190,7 +184,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', conflictingRunId: 'wrun_conflicting', @@ -236,7 +230,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type for a hook! - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -257,7 +251,7 @@ describe('createCreateHook', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for hook'); - expect(workflowError?.message).toContain('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(workflowError?.message).toContain('step_completed'); }); @@ -268,7 +262,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -278,7 +272,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -315,7 +309,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: {}, createdAt: new Date(), }, @@ -363,7 +357,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', conflictingRunId: 'wrun_conflicting_owner', @@ -395,7 +389,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -418,7 +412,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: {}, createdAt: new Date(), }, @@ -426,7 +420,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload: await dehydrateStepReturnValue( { data: 'after-ready' }, @@ -452,7 +446,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -491,7 +485,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token' }, createdAt: new Date(), }, @@ -499,7 +493,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token' }, createdAt: new Date(), }, @@ -507,7 +501,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -543,7 +537,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -553,7 +547,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -569,7 +563,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -585,7 +579,7 @@ describe('createCreateHook', () => { eventId: 'evnt_3', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -614,7 +608,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -644,7 +638,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', conflictingRunId: 'wrun_conflicting', @@ -676,7 +670,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -702,7 +696,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -712,7 +706,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -728,7 +722,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -795,7 +789,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -813,7 +807,7 @@ describe('createCreateHook', () => { // Wait for events to process (hook_created sets hasCreatedEvent on queue item) await vi.waitFor(() => { - const item = ctx.invocationsQueue.get('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + const item = ctx.invocationsQueue.get('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(item?.type === 'hook' && item.hasCreatedEvent).toBe(true); }); @@ -821,7 +815,7 @@ describe('createCreateHook', () => { hook.dispose(); const queueItem = ctx.invocationsQueue.get( - 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'hook_01K11TFZ62C752Z96G9MRSN85J' ); expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { @@ -837,7 +831,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -847,7 +841,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -863,7 +857,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -879,7 +873,7 @@ describe('createCreateHook', () => { eventId: 'evnt_3', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -915,7 +909,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -946,7 +940,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -1049,7 +1043,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -1112,7 +1106,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -1122,7 +1116,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -1161,7 +1155,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -1177,7 +1171,7 @@ describe('createCreateHook', () => { // Wait for events to process (hook_created sets hasCreatedEvent on queue item) await vi.waitFor(() => { - const item = ctx.invocationsQueue.get('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + const item = ctx.invocationsQueue.get('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(item?.type === 'hook' && item.hasCreatedEvent).toBe(true); }); @@ -1297,7 +1291,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData, createdAt: new Date(), }, diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index 5b2d2e6edf..816b2b9ec8 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -2,7 +2,6 @@ import { ReplayDivergenceError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from '../correlation-id.js'; import { EventsConsumer } from '../events-consumer.js'; @@ -18,7 +17,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const ctx: WorkflowOrchestratorContext = { suspensionGeneration: 0, @@ -42,10 +40,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -65,7 +59,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -75,7 +69,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -96,7 +90,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -106,7 +100,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', createdAt: new Date(), }, ]); @@ -124,7 +118,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -134,7 +128,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -152,7 +146,7 @@ describe('createSleep', () => { expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('wait_completed'); expect(workflowError?.message).toContain('resumeAt'); - expect(workflowError?.message).toContain('wait_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('wait_01K11TFZ62FEAJPFZ0JMCV2A5V'); }); it('should invoke workflow error handler when wait_completed resumeAt is invalid', async () => { @@ -161,7 +155,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -171,7 +165,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date(Number.NaN), }, @@ -214,7 +208,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type for a wait! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -234,7 +228,7 @@ describe('createSleep', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for wait'); - expect(workflowError?.message).toContain('wait_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('wait_01K11TFZ62FEAJPFZ0JMCV2A5V'); expect(workflowError?.message).toContain('step_completed'); }); @@ -244,7 +238,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -264,7 +258,7 @@ describe('createSleep', () => { // Check that the wait item has been updated with hasCreatedEvent const waitItem = ctx.invocationsQueue.get( - 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V' ); expect(waitItem).toBeDefined(); expect(waitItem?.type).toBe('wait'); @@ -283,7 +277,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', // Wrong event type for a wait! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { token: 'test-token', payload: { data: 'test' }, @@ -310,7 +304,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -329,7 +323,7 @@ describe('createSleep', () => { // Queue item should still exist (wait_created is not terminal) expect(ctx.invocationsQueue.size).toBe(1); const waitItem = ctx.invocationsQueue.get( - 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V' ); expect(waitItem).toBeDefined(); expect(waitItem?.type).toBe('wait'); @@ -344,7 +338,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -354,7 +348,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -384,7 +378,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -394,7 +388,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -404,7 +398,7 @@ describe('createSleep', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'wait_completed', // Duplicate! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -431,7 +425,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt, }, @@ -441,7 +435,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt, }, diff --git a/workbench/nextjs-turbopack/vercel.json b/workbench/nextjs-turbopack/vercel.json index 22da2a3327..ff2e7944b0 100644 --- a/workbench/nextjs-turbopack/vercel.json +++ b/workbench/nextjs-turbopack/vercel.json @@ -5,8 +5,7 @@ } }, "env": { - "WORKFLOW_PUBLIC_MANIFEST": "1", - "WORKFLOW_PER_KIND_CORRELATION_IDS": "1" + "WORKFLOW_PUBLIC_MANIFEST": "1" }, "regions": [ "iad1", From 5cb2cd0881f04875115c78a004af8dc580e18751 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 18:30:19 -0700 Subject: [PATCH 06/14] Regenerate correlation-id fixtures against per-kind sequences Fixtures that pinned correlation ids to the shared run-wide sequence now derive them from the kind's own sequence, and the two places in workflow.test.ts that built expected ids from a raw monotonicFactory derive them from the run's generator instead. Also documents hooks.list() ordering: a hook the runtime creates on your behalf draws from a separate sequence, so it sorts at an arbitrary position among the ones a workflow created. --- .../workflow-runtime/world/storage.mdx | 2 + .../src/delivery-barrier-coverage.test.ts | 121 ++++---- .../core/src/hook-sleep-interaction.test.ts | 198 +++++++------ .../runtime/precondition-guard-replay.test.ts | 16 +- .../resume-hook.consumer-preload.test.ts | 13 +- .../runtime/wait-completion-replay.test.ts | 12 +- .../core/src/step-delivery-hop-count.test.ts | 73 +++-- .../core/src/step-delivery-ordering.test.ts | 78 +++-- packages/core/src/workflow.test.ts | 279 ++++++++++-------- 9 files changed, 435 insertions(+), 357 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx b/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx index af7248571f..7787378f9a 100644 --- a/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx +++ b/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx @@ -289,6 +289,8 @@ const result = await world.hooks.list({ // [!code highlight] **Returns:** `{ data: Hook[], cursor?: string }` +Hooks are ordered by hook ID. The hooks a workflow creates are listed in the order it created them. A hook the runtime creates on your behalf, such as the one backing an abort controller, draws its ID from a separate sequence, so it sorts at an arbitrary position among yours rather than at its creation position. + ### Hook Type | Field | Type | Description | diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index 83375a0cee..10a6726877 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -44,7 +44,10 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import { @@ -107,19 +110,24 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { } /** - * The ULIDs the seeded generator hands out, in invocation order. Correlation - * IDs in the fixtures below have to match what the replayed workflow draws. + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. IDs in the fixtures below have to match what the replayed workflow + * draws, and each kind draws from its own sequence, so a fixture indexes into + * the kind it is naming rather than into one run-wide sequence. */ -function deterministicUlids(count: number): string[] { - const context = createContext({ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: FIXED_TIMESTAMP, }); - const workflowStartedAt = context.globalThis.Date.now(); - return Array.from({ length: count }, () => ulid(workflowStartedAt)); + return Array.from({ length: count }, () => generate(kind)); } -const ULIDS = deterministicUlids(8); +const STEP_IDS = correlationIds('step', 8); +const WAIT_IDS = correlationIds('wait', 8); +const HOOK_IDS = correlationIds('hook', 8); +const ABORT_IDS = correlationIds('abort', 8); +const ABORT_HOOK_IDS = correlationIds('abortHook', 8); async function replay( ctx: WorkflowOrchestratorContext, @@ -204,32 +212,32 @@ describe('step result delivery ordering against an earlier step result', () => { ]); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'wait_created', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_1', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'wait_completed', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_4', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'wait_completed', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_4', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'stepB', }), - event('evnt_6', 'step_started', `step_${ULIDS[2]}`, { + event('evnt_6', 'step_started', `step_${STEP_IDS[1]}`, { stepName: 'stepB', }), - event('evnt_7', 'step_completed', `step_${ULIDS[2]}`, { + event('evnt_7', 'step_completed', `step_${STEP_IDS[1]}`, { stepName: 'stepB', result: stepBResult, }), - event('evnt_8', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_8', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterA', }), - event('evnt_9', 'step_created', `step_${ULIDS[4]}`, { + event('evnt_9', 'step_created', `step_${STEP_IDS[3]}`, { stepName: 'afterB', }), ]; @@ -283,22 +291,22 @@ describe('wait completion delivery ordering against an earlier step result', () ); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'wait_created', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_1', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_4', 'wait_completed', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_4', 'wait_completed', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_6', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterSleep', }), ]; @@ -342,28 +350,28 @@ describe('hook payload delivery ordering against an earlier step result', () => ]); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'hook_created', `hook_${ULIDS[1]}`, { + event('evnt_1', 'hook_created', `hook_${HOOK_IDS[0]}`, { token: 'tok', isWebhook: false, }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_4', 'hook_received', `hook_${ULIDS[1]}`, { + event('evnt_4', 'hook_received', `hook_${HOOK_IDS[0]}`, { token: 'tok', payload: hookPayload, }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_6', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterHook', }), ]; @@ -405,12 +413,12 @@ describe('hook payload delivery ordering against an earlier step result', () => // evnt_6 hook_received (abort) ← must NOT overtake evnt_5 describe('abort delivery ordering against an earlier step result', () => { it('delivers the earlier step_completed before the abort', async () => { - // The controller draws two ULIDs on construction (stream id, then hook - // correlation id), so the sleep and stepA take the next two. - const abortHookToken = `abrt_${ULIDS[0]}`; - const abortCorrelationId = `hook_${ULIDS[1]}`; - const waitCorrelationId = `wait_${ULIDS[2]}`; - const stepACorrelationId = `step_${ULIDS[3]}`; + // The controller draws two ids on construction, each from its own kind: + // the abort token, then the hook correlation id backing it. + const abortHookToken = `abrt_${ABORT_IDS[0]}`; + const abortCorrelationId = `hook_${ABORT_HOOK_IDS[0]}`; + const waitCorrelationId = `wait_${WAIT_IDS[0]}`; + const stepACorrelationId = `step_${STEP_IDS[0]}`; const resumeAt = new Date(FIXED_TIMESTAMP + 5_000); const ops: Promise[] = []; @@ -445,10 +453,10 @@ describe('abort delivery ordering against an earlier step result', () => { token: abortHookToken, payload: abortPayload, }), - event('evnt_7', 'step_created', `step_${ULIDS[4]}`, { + event('evnt_7', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_8', 'step_created', `step_${ULIDS[5]}`, { + event('evnt_8', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterAbort', }), ]; @@ -557,8 +565,9 @@ function expectSuspensionSnapshotSteps(error: unknown, expected: string[]) { } describe('suspension timing against parked step deliveries', () => { - // Draw order: sleep -> ULIDS[0]; the three parallel steps -> 1..3; the - // follow-up step (never run, so only allocated) -> 4. + // Draw order within each kind: the sleep is the run's only wait; the three + // parallel steps are its first three steps, and the follow-up step (never + // run, so only allocated) is its fourth. const RESUME_AT = new Date(FIXED_TIMESTAMP + 25 * 60_000); it('a parallel batch with a pending sleep suspends carrying the follow-up step', async () => { @@ -570,36 +579,36 @@ describe('suspension timing against parked step deliveries', () => { ); const events: Event[] = [ - event('evnt_0', 'wait_created', `wait_${ULIDS[0]}`, { + event('evnt_0', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt: RESUME_AT, }), - event('evnt_1', 'step_created', `step_${ULIDS[1]}`, { + event('evnt_1', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', }), - event('evnt_2', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_2', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', }), - event('evnt_3', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_3', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', }), - event('evnt_4', 'step_started', `step_${ULIDS[1]}`, { + event('evnt_4', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', }), - event('evnt_5', 'step_started', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_started', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', }), - event('evnt_6', 'step_started', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_started', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', }), - event('evnt_7', 'step_completed', `step_${ULIDS[1]}`, { + event('evnt_7', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', result: results[0], }), - event('evnt_8', 'step_completed', `step_${ULIDS[2]}`, { + event('evnt_8', 'step_completed', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', result: results[1], }), - event('evnt_9', 'step_completed', `step_${ULIDS[3]}`, { + event('evnt_9', 'step_completed', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', result: results[2], }), @@ -639,16 +648,16 @@ describe('suspension timing against parked step deliveries', () => { ); const events: Event[] = [ - event('evnt_0', 'wait_created', `wait_${ULIDS[0]}`, { + event('evnt_0', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt: RESUME_AT, }), - event('evnt_1', 'step_created', `step_${ULIDS[1]}`, { + event('evnt_1', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'only', }), - event('evnt_2', 'step_started', `step_${ULIDS[1]}`, { + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'only', }), - event('evnt_3', 'step_completed', `step_${ULIDS[1]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'only', result, }), diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 4960e7871e..acc05efd20 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -3,7 +3,10 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -28,10 +31,12 @@ import { createSleep } from './workflow/sleep.js'; * so suspensions wait for both async deserialization AND microtask deliveries. */ +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; @@ -78,15 +83,22 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return ctx; } -// Deterministic correlation IDs from the ULID generator with seed 'test' -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', - '01K11TFZ62YS0YYFDQ3E8B9YCZ', - '01K11TFZ62YS0YYFDQ3E8B9YD0', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 6); +const WAIT_IDS = correlationIds('wait', 6); +const HOOK_IDS = correlationIds('hook', 6); // ─── Helpers ─────────────────────────────────────────── @@ -170,7 +182,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -181,7 +193,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -189,7 +201,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -200,7 +212,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -211,7 +223,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload3, @@ -265,7 +277,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -276,7 +288,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -284,7 +296,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -295,7 +307,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -341,7 +353,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -349,7 +361,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -357,7 +369,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload }, createdAt: new Date(), }, @@ -365,7 +377,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -373,7 +385,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -381,7 +393,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep', result: setupResult }, createdAt: new Date(), }, @@ -389,7 +401,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -397,7 +409,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -467,7 +479,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -475,7 +487,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -483,7 +495,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload }, createdAt: new Date(), }, @@ -491,7 +503,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -499,7 +511,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -507,7 +519,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep', result: setupResult }, createdAt: new Date(), }, @@ -515,7 +527,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -523,7 +535,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -614,7 +626,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_8', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1 }, createdAt: new Date(), }; @@ -622,7 +634,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_10', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[2]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }; @@ -641,7 +653,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -649,7 +661,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -657,7 +669,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -665,7 +677,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep', result: progress0Result }, createdAt: new Date(), }, @@ -673,7 +685,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[2]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -681,7 +693,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload0 }, createdAt: new Date(), }, @@ -689,7 +701,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -697,7 +709,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -706,7 +718,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_9', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep', result: drain0Result }, createdAt: new Date(), }, @@ -714,7 +726,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_11', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -722,7 +734,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_12', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -730,7 +742,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_13', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep', result: progress1Result }, createdAt: new Date(), }, @@ -738,7 +750,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_14', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[5]}`, + correlationId: `step_${STEP_IDS[3]}`, eventData: { stepName: finalStepName }, createdAt: new Date(), }, @@ -850,7 +862,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -861,7 +873,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep' }, createdAt: new Date(), }, @@ -869,7 +881,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep', }, @@ -879,7 +891,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -890,7 +902,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -933,7 +945,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[0]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -941,7 +953,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -949,7 +961,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', }, @@ -959,7 +971,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: resultA, @@ -970,7 +982,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB' }, createdAt: new Date(), }, @@ -978,7 +990,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', }, @@ -988,7 +1000,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', result: resultB, @@ -1027,7 +1039,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[0]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1035,7 +1047,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -1043,7 +1055,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', }, @@ -1053,7 +1065,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: resultA, @@ -1101,7 +1113,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep' }, createdAt: new Date(), }, @@ -1109,7 +1121,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep', }, @@ -1119,7 +1131,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB' }, createdAt: new Date(), }, @@ -1127,7 +1139,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', }, @@ -1137,7 +1149,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', result: resultB, @@ -1148,7 +1160,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC' }, createdAt: new Date(), }, @@ -1156,7 +1168,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC', }, @@ -1166,7 +1178,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC', result: resultC, @@ -1235,7 +1247,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1246,7 +1258,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1255,7 +1267,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -1266,7 +1278,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', input: payload1 }, createdAt: new Date(), }, @@ -1274,7 +1286,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', }, @@ -1284,7 +1296,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', result: stepResult1, @@ -1296,7 +1308,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -1307,7 +1319,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', input: payload2 }, createdAt: new Date(), }, @@ -1315,7 +1327,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_8', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', }, @@ -1325,7 +1337,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_9', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', result: stepResult2, @@ -1376,7 +1388,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1387,7 +1399,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -1398,7 +1410,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -1472,7 +1484,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1483,7 +1495,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1491,7 +1503,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payloadA, @@ -1502,7 +1514,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1510,7 +1522,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payloadB, diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index 3a879bc8d1..0fc8b40ebc 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -40,6 +40,7 @@ import { } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { runtimeLogger } from '../logger.js'; import { registerStepFunction } from '../private.js'; import { workflowEntrypoint } from '../runtime.js'; @@ -48,7 +49,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { createContext } from '../vm/index.js'; import { getPreconditionMaxInProcessRestarts, getPreconditionMaxReinvocations, @@ -141,16 +141,16 @@ async function runPreconditionScenario(options: { ? SPEC_VERSION_SLOT_IDENTITY : SPEC_VERSION_CURRENT; - const { globalThis: vmGlobalThis } = createContext({ + // Correlation ids are ULIDs in both modes: slot identity numbers event ids + // only. Each kind draws from its own sequence, so these are the first id of + // three separate sequences rather than three draws of one. + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); - // Correlation ids are ULIDs in both modes: slot identity numbers event ids - // only. - const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; - const syncStep0CorrelationId = `step_${vmUlid(+startedAt)}`; - const waitCorrelationId = `wait_${vmUlid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; + const syncStep0CorrelationId = `step_${generate('step')}`; + const waitCorrelationId = `wait_${generate('wait')}`; const workflowRun: WorkflowRun = { runId, diff --git a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts index c7897553ee..5853a0ce36 100644 --- a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts +++ b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts @@ -22,12 +22,12 @@ import { } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { workflowEntrypoint } from '../runtime.js'; import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; vi.mock('@vercel/functions', () => ({ waitUntil: vi.fn() })); @@ -81,15 +81,14 @@ async function runResumeConsumerScenario(options: { undefined ); - // Derive the hook correlation id the seeded VM will compute during replay, - // so the preloaded / re-ensured hook_received matches the workflow's own - // createHook call (id assignment order: the hook is the first id derived). - const { globalThis: vmGlobalThis } = createContext({ + // Derive the hook correlation id the replay will compute, so the preloaded / + // re-ensured hook_received matches the workflow's own createHook call. The + // workflow creates one hook, so it draws its kind's first id. + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); - const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; const workflowRun: WorkflowRun = { runId, diff --git a/packages/core/src/runtime/wait-completion-replay.test.ts b/packages/core/src/runtime/wait-completion-replay.test.ts index c4ca46982a..9a35ac110d 100644 --- a/packages/core/src/runtime/wait-completion-replay.test.ts +++ b/packages/core/src/runtime/wait-completion-replay.test.ts @@ -5,8 +5,8 @@ import { type WorkflowRun, type World, } from '@workflow/world'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { registerStepFunction } from '../private.js'; import { workflowEntrypoint } from '../runtime.js'; import { @@ -14,7 +14,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; vi.mock('@vercel/functions', () => ({ @@ -85,14 +84,13 @@ async function runStaleWaitReplayScenario(options: { undefined ); - const { globalThis: vmGlobalThis } = createContext({ + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); - const hookCorrelationId = `hook_${ulid(+startedAt)}`; - const syncStep0CorrelationId = `step_${ulid(+startedAt)}`; - const waitCorrelationId = `wait_${ulid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; + const syncStep0CorrelationId = `step_${generate('step')}`; + const waitCorrelationId = `wait_${generate('wait')}`; const workflowRun: WorkflowRun = { runId, diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index 96250021cc..225ced8337 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -28,7 +28,10 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -42,13 +45,15 @@ import { createContext } from './vm/index.js'; import { createCreateHook } from './workflow/hook.js'; import { createSleep } from './workflow/sleep.js'; +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext( events: Event[], replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; @@ -89,12 +94,22 @@ function setupWorkflowContext( return ctx; } -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 4); +const WAIT_IDS = correlationIds('wait', 4); +const HOOK_IDS = correlationIds('hook', 4); async function runWithDiscontinuation( ctx: WorkflowOrchestratorContext, @@ -136,7 +151,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -144,7 +159,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -152,7 +167,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -160,7 +175,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: hookPayload }, createdAt: new Date(), }, @@ -168,7 +183,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -176,7 +191,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterHook' }, createdAt: new Date(), }, @@ -184,7 +199,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -263,7 +278,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -271,7 +286,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -279,7 +294,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -287,7 +302,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -295,7 +310,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -303,7 +318,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -311,7 +326,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -386,7 +401,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -394,7 +409,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -402,7 +417,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -410,7 +425,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -418,7 +433,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_failed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', error: stepAError }, createdAt: new Date(), }, @@ -426,7 +441,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -434,7 +449,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterFailure' }, createdAt: new Date(), }, diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index ec08f72ef9..a624eba172 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -3,7 +3,10 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -89,13 +92,15 @@ import { createSleep } from './workflow/sleep.js'; * one cache exactly like the replay loop inside a single production queue * delivery does (see the `ReplayPayloadCache` class docstring). */ +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext( events: Event[], replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; @@ -138,13 +143,22 @@ function setupWorkflowContext( return ctx; } -// Deterministic correlation IDs from the ULID generator with seed 'test' -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 4); +const WAIT_IDS = correlationIds('wait', 4); +const HOOK_IDS = correlationIds('hook', 4); async function runWithDiscontinuation( ctx: WorkflowOrchestratorContext, @@ -222,7 +236,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -230,7 +244,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -238,7 +252,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -249,7 +263,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -257,7 +271,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -265,7 +279,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -273,16 +287,16 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, ]; } - // ULID draw order in this body: `stepA()` takes CORR_IDS[0], `sleep()` - // takes CORR_IDS[1], and then whichever branch is resumed FIRST takes - // CORR_IDS[2] while the other takes CORR_IDS[3]. + // Correlation-id draw order in this body: `stepA()` takes STEP_IDS[0], `sleep()` + // takes WAIT_IDS[0], and then whichever branch is resumed FIRST takes + // STEP_IDS[1] while the other takes STEP_IDS[2]. function workflowBody(ctx: WorkflowOrchestratorContext) { const useStep = createUseStep(ctx); const sleep = createSleep(ctx); @@ -325,7 +339,7 @@ describe('step result delivery ordering across replays', () => { } // The log's ordering was reproduced: the sleep branch resumed first and - // drew CORR_IDS[2] for `afterSleep`, so both `step_created` events at + // drew STEP_IDS[1] for `afterSleep`, so both `step_created` events at // the tail matched their consumers and the run suspends with both // follow-up steps pending. expect(pendingStepNames(ctx).sort()).toEqual(['afterSleep', 'afterStep']); @@ -363,7 +377,7 @@ describe('step result delivery ordering across replays', () => { expect(error).toBeDefined(); // FAILS on `main`: the step result now wins, `afterStep` draws - // CORR_IDS[2], and replay diverges at evnt_5 with the production error + // STEP_IDS[1], and replay diverges at evnt_5 with the production error // shape ("... belongs to \"afterSleep\", but the current step consumer // is \"afterStep\""). if (!WorkflowSuspension.is(error)) { @@ -390,7 +404,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -398,7 +412,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -406,7 +420,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -416,7 +430,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: hookPayload }, createdAt: new Date(), }, @@ -424,7 +438,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -432,7 +446,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterHook' }, createdAt: new Date(), }, @@ -440,7 +454,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -448,8 +462,8 @@ describe('step result delivery ordering across replays', () => { } /** - * ULID draw order: `createHook()` takes CORR_IDS[0], `stepA()` takes - * CORR_IDS[1], then the branch resumed FIRST takes CORR_IDS[2]. + * Correlation-id draw order: `createHook()` takes HOOK_IDS[0], `stepA()` takes + * STEP_IDS[0], then the branch resumed FIRST takes STEP_IDS[1]. * * Two ways of consuming the hook. Both subscribe before `hook_received` is * consumed, so both take the same delivery path in `workflow/hook.ts` (the @@ -465,7 +479,7 @@ describe('step result delivery ordering across replays', () => { * settles the promise returned by `next()`, and only then does the loop * body run. Those extra hops are enough for a memo-warm `step_completed` * — which on an unfixed runtime resolves inside its own queue slot with - * no detached chain at all — to draw CORR_IDS[2] first. + * no detached chain at all — to draw STEP_IDS[1] first. * - `await hook` (control): resolution resumes the branch's continuation * directly, so it reaches `afterHook()` in the first microtask and stays * ahead of the step result even on a warm cache. @@ -560,7 +574,7 @@ describe('step result delivery ordering across replays', () => { expect(error).toBeDefined(); // FAILS on `main`: the step result overtakes the hook payload, - // `afterStep` draws CORR_IDS[2], and replay diverges at evnt_5 with the + // `afterStep` draws STEP_IDS[1], and replay diverges at evnt_5 with the // production error shape. if (!WorkflowSuspension.is(error)) { throw error; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index e916a01bf6..17d450b3ed 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -4,10 +4,14 @@ import type { Event, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { decodeTime } from 'ulid'; import { afterEach, assert, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { runIdCreatedAt } from './runtime/run-id-time.js'; import { setWorld } from './runtime/world.js'; import { dehydrateStepReturnValue, @@ -19,6 +23,31 @@ import { replayWorkflow, resumeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; +/** + * The correlation IDs the seeded generator hands out for a run, per kind, in + * draw order. Each kind draws from its own sequence, so a fixture indexes into + * the kind it is naming. The seed and clock mirror what `runWorkflow` derives + * from the run. + */ +function correlationIds(runId: string) { + const generate = createCorrelationIdGenerator({ + seed: `${runId}:workflow:test-deployment`, + fixedTimestamp: + runIdCreatedAt(runId) ?? +new Date('2024-01-01T00:00:00.000Z'), + }); + const take = (kind: CorrelationIdKind, count: number) => + Array.from({ length: count }, () => generate(kind)); + return { + step: take('step', 6), + wait: take('wait', 6), + hook: take('hook', 6), + }; +} + +const IDS_WRUN_123 = correlationIds('wrun_123'); +const IDS_TEST_RUN_123 = correlationIds('test-run-123'); +const IDS_WRUN_ULID = correlationIds('wrun_01K75533W56DAE35VY3082DN3P'); + describe('runWorkflow', () => { const getWorkflowTransformCode = (workflowName?: string) => `;globalThis.__private_workflows = new Map(); @@ -177,7 +206,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -187,7 +216,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -433,7 +462,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -443,7 +472,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -608,7 +637,7 @@ describe('runWorkflow', () => { eventId: 'event-step1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -618,7 +647,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -628,7 +657,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -644,7 +673,7 @@ describe('runWorkflow', () => { eventId: 'event-step2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -654,7 +683,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -664,7 +693,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -680,7 +709,7 @@ describe('runWorkflow', () => { eventId: 'event-step3-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -690,7 +719,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -700,7 +729,7 @@ describe('runWorkflow', () => { eventId: 'event-5', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -776,8 +805,8 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - const startStepId = 'step_01HK153X00SFW49DWMQP3J810S'; - const branchStepId = 'step_01HK153X00SFW49DWMQP3J810T'; + const startStepId = `step_${IDS_WRUN_123.step[0]}`; + const branchStepId = `step_${IDS_WRUN_123.step[1]}`; const events: Event[] = [ { eventId: 'event-run-created', @@ -946,7 +975,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -956,7 +985,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -966,7 +995,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -999,7 +1028,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -1050,7 +1079,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1060,7 +1089,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1070,7 +1099,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1086,7 +1115,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1144,7 +1173,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1154,7 +1183,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1164,7 +1193,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1180,7 +1209,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1238,7 +1267,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1248,7 +1277,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1258,7 +1287,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1274,7 +1303,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1329,7 +1358,7 @@ describe('runWorkflow', () => { const events: Event[] = [ { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7M', + correlationId: `step_${IDS_WRUN_ULID.step[0]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K755385N02MMWXYHFCQSP9P0', eventData: { @@ -1339,7 +1368,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7N', + correlationId: `step_${IDS_WRUN_ULID.step[1]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K755386GHGAFYYDC58V17E3T', eventData: { @@ -1349,7 +1378,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7P', + correlationId: `step_${IDS_WRUN_ULID.step[2]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75538D4Q4X8PJ1ZNDZD5R0W', eventData: { @@ -1359,7 +1388,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7Q', + correlationId: `step_${IDS_WRUN_ULID.step[3]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75538Y9GEHXJQXT3JB89M4C', eventData: { @@ -1369,7 +1398,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7R', + correlationId: `step_${IDS_WRUN_ULID.step[4]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75539CD2PAH419SKJ2X5V5T', eventData: { @@ -1379,7 +1408,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7R', + correlationId: `step_${IDS_WRUN_ULID.step[4]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1395,7 +1424,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7Q', + correlationId: `step_${IDS_WRUN_ULID.step[3]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1411,7 +1440,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7P', + correlationId: `step_${IDS_WRUN_ULID.step[2]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1427,7 +1456,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7N', + correlationId: `step_${IDS_WRUN_ULID.step[1]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1443,7 +1472,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7M', + correlationId: `step_${IDS_WRUN_ULID.step[0]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1700,7 +1729,7 @@ describe('runWorkflow', () => { { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, args: [1, 2], }, ]); @@ -1731,7 +1760,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1802,13 +1831,13 @@ describe('runWorkflow', () => { { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, args: [1, 2], }, { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, args: [3, 4], }, ]); @@ -2187,7 +2216,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2245,7 +2274,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'wrong-token', payload: await dehydrateStepReturnValue( @@ -2297,7 +2326,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2313,7 +2342,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2392,7 +2421,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMA', eventData: { token: 'test-token', @@ -2406,7 +2435,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMA', eventData: { token: 'test-token', @@ -2420,7 +2449,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMB', eventData: { token: 'test-token', @@ -2480,7 +2509,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2496,7 +2525,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2563,7 +2592,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2579,7 +2608,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2637,7 +2666,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2653,7 +2682,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2669,7 +2698,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -2679,7 +2708,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRun.runId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -2748,7 +2777,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2764,7 +2793,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -2774,7 +2803,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -2836,7 +2865,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'my-custom-token', payload: await dehydrateStepReturnValue( @@ -2939,7 +2968,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_created', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: {}, createdAt: new Date(), }, @@ -2990,7 +3019,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'claim-only-token', conflictingRunId: 'wrun_conflicting_owner', @@ -3057,7 +3086,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'my-duplicate-token', conflictingRunId: 'wrun_conflicting', @@ -3114,7 +3143,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'conflicting-token', }, @@ -3929,7 +3958,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -3939,7 +3968,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -3992,7 +4021,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -4002,7 +4031,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:06.000Z'), }, @@ -4091,7 +4120,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4101,7 +4130,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4111,7 +4140,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4121,7 +4150,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4175,7 +4204,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4185,7 +4214,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4195,7 +4224,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4246,7 +4275,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -4256,7 +4285,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -4272,7 +4301,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:03.000Z'), }, @@ -4282,7 +4311,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:03.000Z'), }, @@ -4340,7 +4369,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -4350,7 +4379,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: resumeAt, }, @@ -4403,7 +4432,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4413,7 +4442,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4428,7 +4457,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4438,7 +4467,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4448,7 +4477,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('step done', ops), @@ -4496,7 +4525,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', }, @@ -4506,7 +4535,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', result: await dehydrateStepReturnValue('first done', ops), @@ -4518,7 +4547,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', result: await dehydrateStepReturnValue('duplicate', ops), @@ -4529,7 +4558,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'doWork2', }, @@ -4539,7 +4568,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'doWork2', result: await dehydrateStepReturnValue('second done', ops), @@ -4598,7 +4627,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4608,7 +4637,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('done', ops), @@ -4664,7 +4693,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4674,7 +4703,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('done', ops), @@ -4928,7 +4957,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_created' as const, - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: {}, createdAt: new Date(), }, @@ -4936,7 +4965,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -4952,7 +4981,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'hook_disposed', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', }, @@ -5058,9 +5087,9 @@ describe('runWorkflow', () => { // Correlation IDs match the deterministic ULID generator for the seed // `${runId}:${workflowName}:${+startedAt}` - const stepA = 'step_01HK153X00SFW49DWMQP3J810S'; - const stepB = 'step_01HK153X00SFW49DWMQP3J810T'; - const stepC = 'step_01HK153X00SFW49DWMQP3J810V'; + const stepA = `step_${IDS_WRUN_123.step[0]}`; + const stepB = `step_${IDS_WRUN_123.step[1]}`; + const stepC = `step_${IDS_WRUN_123.step[2]}`; const events: Event[] = [ { @@ -5288,7 +5317,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-created', runId: workflowRunId, eventType: 'hook_created', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token' }, createdAt: new Date('2024-01-01T00:00:00.200Z'), }, @@ -5296,7 +5325,7 @@ describe('runWorkflow', () => { eventId: 'evnt-wait-created', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00SFW49DWMQP3J810T', + correlationId: `wait_${IDS_WRUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-02T00:00:00.000Z') }, createdAt: new Date('2024-01-01T00:00:00.300Z'), }, @@ -5304,7 +5333,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-1', runId: workflowRunId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -5315,7 +5344,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', input: payload1 }, createdAt: new Date('2024-01-01T00:00:01.100Z'), }, @@ -5323,7 +5352,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', }, @@ -5333,7 +5362,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', result: stepResult1, @@ -5344,7 +5373,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-2', runId: workflowRunId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -5355,7 +5384,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', input: payload2 }, createdAt: new Date('2024-01-01T00:00:02.100Z'), }, @@ -5363,7 +5392,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', }, @@ -5373,7 +5402,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', result: stepResult2, @@ -5476,7 +5505,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -5486,7 +5515,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -5496,7 +5525,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -5513,7 +5542,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -5523,7 +5552,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -5533,7 +5562,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -5550,7 +5579,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -5560,7 +5589,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -5570,7 +5599,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( From 4505f6941f83e507167880495d2935bb791f8e9d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 19:42:46 -0700 Subject: [PATCH 07/14] fix(world-local): keep a hook retry pinned to the canonical event position A second replay's hook_created adopts the canonical position recorded in the token claim. Under slot identity that write had also picked a position of its own, so losing the canonical one was read as ordinary slot contention and the writer reallocated around it, publishing a second hook_created for one hook and leaving its first pick as a hole. Pin a converged write to the canonical position: losing it means the hook exists, which the runtime already swallows as a benign concurrent replay. Release the position the write gave up so the log stays dense. --- .../world-local/src/storage/events-storage.ts | 28 +++++++- .../src/storage/slot-identity.test.ts | 66 ++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index a85a5b45d3..a325cb977d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -896,6 +896,12 @@ export function createEventsStorage( // claim's owner is the only writer allowed to move off the pinned // position while the resume is still unrecorded. let ownsResumeClaim = false; + // Whether the position this write ends up publishing to was chosen by + // this world for idempotency rather than claimed by the caller. A hook + // retry adopts the canonical `hook_created` position recorded in the + // token claim, so losing that position says the hook is already + // created, not that the caller's log is short an event. + let convergedOnCanonicalPosition = false; const now = new Date(); // For run_created events, use client-provided runId or generate one server-side @@ -2385,6 +2391,17 @@ export function createEventsStorage( canonicalEventId = pinned; } + if (canonicalEventId !== eventId) { + // Moving to the canonical position leaves the position this + // write was going to take unused. Hand it back so the allocator + // can fill it rather than leaving a hole the log can never close. + const abandoned = slotFromId(eventId); + if (abandoned !== undefined && reserved.delete(abandoned)) { + slots.release(effectiveRunId, abandoned); + } + } + convergedOnCanonicalPosition = true; + // A canonical ULID also makes converging writes byte-identical. // A slot id carries no time, and reading one as a ULID yields the // epoch, so a slot-numbered run keeps the wall clock and the two @@ -2801,7 +2818,14 @@ export function createEventsStorage( // the run already has a creation event, and appending a second one above // it would be worse than the duplicate the publish is reporting. const reallocatesSlot = - slotMode && params?.eventId === undefined && !ownsFirstSlot; + slotMode && + params?.eventId === undefined && + !ownsFirstSlot && + // A write that adopted the canonical position of an existing hook is + // pinned to it for the same reason `run_created` is pinned to the + // first slot: moving somewhere free would publish a second + // `hook_created` for one hook. + !convergedOnCanonicalPosition; const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; const pinnedDeadline = Date.now() + PINNED_EVENT_WAIT_MS; let compositeKey = ''; @@ -2931,7 +2955,7 @@ export function createEventsStorage( { status: 503 } ); } - if (slotMode) { + if (slotMode && !convergedOnCanonicalPosition) { // Losing a claimed slot means someone else's event occupies this // position, so the log this event was derived from is missing at // least that event — the whole proposed event is stale, not just diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 983cda0079..2c13904ed0 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -1,7 +1,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { SlotConflictError } from '@workflow/errors'; +import { EntityConflictError, SlotConflictError } from '@workflow/errors'; import type { Storage } from '@workflow/world'; import { FIRST_SLOT, @@ -573,3 +573,67 @@ describe('a resume with two writers', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5, 6, 7]); }); }); + +async function hookCreatedCount(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.filter((event) => event.eventType === 'hook_created').length; +} + +describe('a hook created twice', () => { + it('commits one event when the retry allocated its own position', async () => { + // Two replays of the same run both reach `hook.create`. Neither holds an + // event log, so each takes a position of its own choosing; the token claim + // then points the second at the first's event. Losing that position says + // the hook exists, so the retry must stop there. Reallocating around it — + // which is what a writer that picked its own position normally does — would + // publish a second `hook_created` for one hook. + const runId = await newSlotRun(); + const hook = { hookId: 'hook_a', token: 'tok:1' }; + + await createHook(runId, hook.hookId, hook.token); + await expect(createHook(runId, hook.hookId, hook.token)).rejects.toThrow( + EntityConflictError + ); + + await expect(hookCreatedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('leaves no hole behind the retry it turned away', async () => { + // The retry reserved a position before the claim redirected it. Handing it + // back is what keeps the run's log dense, and a dense log is the only thing + // that lets a client prove it holds every event. + const runId = await newSlotRun(); + await createHook(runId, 'hook_a', 'tok:1'); + await createHook(runId, 'hook_a', 'tok:1').catch(() => {}); + + await createHook(runId, 'hook_b', 'tok:2'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('commits one event when both replays race across instances', async () => { + const runId = await newSlotRun(); + const other = createStorage(testDir); + + const outcomes = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'hook_a', + eventData: { token: 'tok:1', hookId: 'hook_a' }, + }), + other.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'hook_a', + eventData: { token: 'tok:1', hookId: 'hook_a' }, + }), + ]); + + expect( + outcomes.filter((outcome) => outcome.status === 'fulfilled') + ).toHaveLength(1); + await expect(hookCreatedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); +}); From d28a8b8c7903b2a81f04d02150ecfb96dec2f7a2 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Mon, 3 Aug 2026 21:35:02 -0700 Subject: [PATCH 08/14] test(world-postgres): issue a batch of slot claims the way the client does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A claim asserts the log is complete up to the position it names, so the client takes the batch's claims one at a time, each against the tail its writer has actually seen. The test issued all ten at once, and when they arrived out of order the tail fence rejected a claim that was genuinely free — intermittently failing on a rejection the client cannot provoke. Serializing costs the test nothing: numbering the second event off the log rather than off the claim still leaves the batch's first reserved slot a hole, which the density assertion still catches. Also covers the fence itself, which had no direct test here. --- .../world-postgres/test/slot-identity.test.ts | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index a6cf498651..3a96d033a3 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -138,6 +138,36 @@ describe('Slot identity (Postgres integration)', () => { await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); }); + test('rejects a claim on a free position below the log’s tail', async () => { + // The undercut that a "is the position free?" check cannot catch. A + // position claimed by a write that then failed is never filled, so a log + // carries holes below its tail — and a caller numbering from a snapshot + // that predates the events above one of those holes aims straight at it. + // Let it land and the event sits below events another replay has already + // consumed: the log stays internally consistent while its order silently + // changes. A claim asserts a complete log, so a claim that does not clear + // the tail is a conflict, exactly as a taken one is. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await expect( + createStep(runId, 'step_early', slotEventId(2)) + ).rejects.toThrow(SlotConflictError); + // The hole stays a hole, and the log stays in slot order. + expect(ascending(await slotsOf(runId))).toEqual([1, 3]); + }); + + test('accepts the claim immediately above a tail with a hole below it', async () => { + // The fence rejects at-or-below, so the first position above the tail has + // to stay writable — otherwise every write following a hole would + // conflict forever and the run could never make progress again. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + expect(await createStep(runId, 'step_next', slotEventId(4))).toBe( + slotEventId(4) + ); + expect(ascending(await slotsOf(runId))).toEqual([1, 3, 4]); + }); + test('numbers a ULID-mode run the way it always did', async () => { const created = await events.create(null, { eventType: 'run_created', @@ -195,18 +225,24 @@ describe('Slot identity (Postgres integration)', () => { ).toEqual(['1 run_created', '2 step_created', '3 step_started']); }); - test('keeps every claim in a burst of lazy starts', async () => { - // The suspension flush issues its lazy starts at once, each having - // reserved two positions. A second event numbered off the log as this - // world sees it would take the slot the next start in the batch claimed, - // costing every start after the first its claim. + test('keeps every claim in a batch of lazy starts', async () => { + // The suspension flush's lazy starts each reserve two positions and claim + // the top one, so the batch's claims are every second slot. A second + // event numbered off the log rather than off the claim would take the + // position below the *next* claim instead of below its own, leaving the + // batch's first reserved slot a hole the log can never close. + // + // The batch is issued one claim at a time because that is how the client + // issues it: a claim asserts the log is complete up to that position, so + // each one is taken against the tail its writer has actually seen. const runId = await newSlotRun(); const claims = Array.from({ length: 10 }, (_, index) => slotEventId(FIRST_SLOT + 2 * (index + 1)) ); - const started = await Promise.all( - claims.map((eventId, index) => - events.create( + const started = []; + for (const [index, eventId] of claims.entries()) { + started.push( + await events.create( runId, { eventType: 'step_started', @@ -216,8 +252,8 @@ describe('Slot identity (Postgres integration)', () => { }, { eventId } ) - ) - ); + ); + } expect(started.map((result) => result.event?.eventId)).toEqual(claims); expect(ascending(await slotsOf(runId))).toEqual( denseFrom(2 * claims.length + 1) From 98692e4ba16c1ea827432593a1df74084d1ad999 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:39:20 -0700 Subject: [PATCH 09/14] fix(core): preserve QuickJS hook retention --- .changeset/quickjs-hook-retention.md | 5 ++++ .../core/src/runtime/quickjs-entrypoint.ts | 4 +++ .../core/src/runtime/quickjs-runtime.test.ts | 28 +++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 23 +++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 .changeset/quickjs-hook-retention.md diff --git a/.changeset/quickjs-hook-retention.md b/.changeset/quickjs-hook-retention.md new file mode 100644 index 0000000000..7085b51d3a --- /dev/null +++ b/.changeset/quickjs-hook-retention.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Preserve Hook minimum-retention deadlines in QuickJS workflows. diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 416199a243..ba02395b8a 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -182,6 +182,10 @@ async function dispatchPendingOps(params: { correlationId: hook.correlationId, eventData: { token: hook.token, + tokenRetentionUntil: + hook.tokenRetentionUntil === undefined + ? undefined + : new Date(hook.tokenRetentionUntil), metadata: encryptedMetadata, // Always include isWebhook explicitly. Worlds default it to // `true` when absent, which would break the public webhook diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index ab70943e02..55770983ea 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -87,6 +87,34 @@ describe('runQuickJSWorkflow', () => { ); }); + it('preserves a Hook minimum-retention deadline across the VM boundary', async () => { + const result = await runQuickJSWorkflow({ + workflowCode: ` + async function workflow() { + var hook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]({ + token: "retained", + experimental_minRetention: 60000, + }); + await hook.getConflict(); + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + }); + + expect(result.suspended?.pendingOperations).toContainEqual( + expect.objectContaining({ + type: 'hook', + token: 'retained', + tokenRetentionUntil: + new Date('2025-01-01T00:00:00Z').getTime() + 60_000, + }) + ); + }); + it('should complete after step resolves via full event replay', async () => { const code = ` var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index fea5314bec..fb2281873d 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -86,6 +86,8 @@ export interface PendingHook { type: 'hook'; correlationId: string; token: string; + /** Earliest token reuse time, as milliseconds since the Unix epoch. */ + tokenRetentionUntil?: number; isWebhook: boolean; metadata?: unknown; hasCreatedEvent: boolean; @@ -568,6 +570,26 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { var correlationId = "hook_" + globalThis.__generateUlid(); var isDisposed = false; var hasCreatedEvent = false; + var tokenRetentionUntil; + if (options.experimental_minRetention !== undefined) { + var minRetention = options.experimental_minRetention; + if (typeof minRetention === "number") { + if (minRetention < 0 || !isFinite(minRetention)) { + throw new Error("Invalid duration: " + minRetention + ". Expected a non-negative finite number of milliseconds."); + } + tokenRetentionUntil = Date.now() + minRetention; + } else if (typeof minRetention === "string") { + var retentionMs = globalThis.__parseDurationMs(minRetention); + if (typeof retentionMs !== "number" || retentionMs < 0 || !isFinite(retentionMs)) { + throw new Error('Invalid duration: "' + minRetention + '". Expected a valid duration string like "1s", "1m", "1h", etc.'); + } + tokenRetentionUntil = Date.now() + retentionMs; + } else if (minRetention instanceof Date) { + tokenRetentionUntil = minRetention.getTime(); + } else { + throw new Error("Invalid duration parameter. Expected a duration string, number (milliseconds), or Date object."); + } + } // Register in pending operations. // Serialize metadata inside the VM so Response/Request objects are @@ -576,6 +598,7 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { type: "hook", correlationId: correlationId, token: token, + tokenRetentionUntil: tokenRetentionUntil, isWebhook: !!options.isWebhook, metadata: options.metadata ? globalThis[Symbol.for("workflow-serialize")](options.metadata) : undefined, hasCreatedEvent: false, From 342c64cfd9a037930ff8c6d4216877b1e369d4ab Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:03:07 -0700 Subject: [PATCH 10/14] fix(core): reject webhook hook retention in QuickJS --- .../core/src/runtime/quickjs-runtime.test.ts | 22 +++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 3 +++ 2 files changed, 25 insertions(+) diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 55770983ea..14d1ed6fe4 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -115,6 +115,28 @@ describe('runQuickJSWorkflow', () => { ); }); + it('rejects minimum retention for webhook Hooks', async () => { + const result = await runQuickJSWorkflow({ + workflowCode: ` + async function workflow() { + globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]({ + isWebhook: true, + experimental_minRetention: 60000, + }); + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + }); + + expect(result.failed?.message).toBe( + 'Webhook hooks do not support `experimental_minRetention`. Use a non-webhook `createHook()` with `resumeHook()`.' + ); + }); + it('should complete after step resolves via full event replay', async () => { const code = ` var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index fb2281873d..a492eba734 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -566,6 +566,9 @@ if (typeof Request === "undefined") { // The promise is resolved when a hook_received event arrives. globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { options = options || {}; + if (options.isWebhook === true && options.experimental_minRetention !== undefined) { + throw new Error('Webhook hooks do not support \`experimental_minRetention\`. Use a non-webhook \`createHook()\` with \`resumeHook()\`.'); + } var token = options.token || globalThis.__generateNanoid(); var correlationId = "hook_" + globalThis.__generateUlid(); var isDisposed = false; From 1f33f512d76a51ae57e238b8fdf2515aafa29adb Mon Sep 17 00:00:00 2001 From: "vercel[bot]" <35613825+vercel[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:57:31 +0000 Subject: [PATCH 11/14] Fix: The QuickJS workflow engine accepts `experimental_minRetention` for Hooks on Worlds that don't support hook retention, silently dropping the requested retention instead of failing closed like the node:vm engine. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes the issue reported at packages/core/src/runtime/quickjs-runtime.ts:569 ## Bug The node:vm workflow engine gates Hook retention on a World capability in `packages/core/src/workflow/hook.ts` (~L89): ```ts if ( options.experimental_minRetention !== undefined && ctx.worldCapabilities?.hookRetention?.active !== true ) { throw new FatalError( 'The configured World does not support `experimental_minRetention` for Hooks.' ); } ``` `ctx.worldCapabilities` is populated from `world.capabilities` (`packages/core/src/runtime.ts:2399`). The `WorldCapabilities.hookRetention` contract (`packages/world/src/interfaces.ts:315`) explicitly states this must **fail closed**: "Missing or inactive means the runtime rejects retained Hooks before registration." The QuickJS engine (opt-in `WORKFLOW_VM=quickjs`) replicated the webhook rejection (commit `342c64c`) and the retention-deadline computation (commit `98692e4`) inside `WORKFLOW_CREATE_HOOK`, but **not** the world-capability gate. It simply computed `tokenRetentionUntil = Date.now() + retentionMs` and passed it through to `world.events.create` in the entrypoint. ### Concrete trigger - Capability declarations: `world-local` and `world-postgres` declare `hookRetention: { active: true }`; **`world-vercel` does not** (verified in `packages/world-vercel/src/index.ts:36` capabilities block). - A workflow running under the QuickJS engine on the Vercel world calling `createHook({ experimental_minRetention: '1h' })` would throw a `FatalError` up front on the node engine, but the QuickJS engine silently accepted it. The requested minimum token retention was therefore never enforced by the backend — the token could be reused earlier than requested — a silent divergence from the documented fail-closed contract. ## Fix Added the missing capability gate at the same synchronous point as the existing webhook check, so it fails closed by default: 1. `WORKFLOW_CREATE_HOOK` (in `VM_BOOTSTRAP`) now throws `'The configured World does not support `experimental_minRetention` for Hooks.'` (same message as the node engine) when `experimental_minRetention` is set but `globalThis.__worldSupportsHookRetention !== true`. 2. Added `worldSupportsHookRetention?: boolean` to `QuickJSRuntimeOptions`, injected per-run into the VM as `globalThis.__worldSupportsHookRetention` (defaults to `false` → unsupported when omitted, i.e. fail closed). 3. The QuickJS entrypoint passes `world.capabilities?.hookRetention?.active === true`, mirroring the node engine's `ctx.worldCapabilities` source. 4. Updated the existing test `preserves a Hook minimum-retention deadline across the VM boundary` to pass `worldSupportsHookRetention: true` so the supported path stays green; the webhook-rejection test is unaffected. This preserves behavior for `world-local`/`world-postgres` (which declare the capability) while making QuickJS reject retention on `world-vercel`, matching the node engine. Co-authored-by: Vercel Co-authored-by: VaguelySerious --- .../core/src/runtime/quickjs-entrypoint.ts | 3 +++ .../core/src/runtime/quickjs-runtime.test.ts | 1 + packages/core/src/runtime/quickjs-runtime.ts | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index ba02395b8a..6c859a1952 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -710,6 +710,9 @@ export async function runWorkflowWithQuickJS(params: { encryptionKey, port, runInput, + // Fail closed on Hook retention when the World doesn't declare support, + // matching the node:vm engine's world-capability gate (see hook.ts). + worldSupportsHookRetention: world.capabilities?.hookRetention?.active === true, }); runtimeLogger.debug('QuickJS runtime: VM returned', { diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 14d1ed6fe4..d2a21bdbe6 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -103,6 +103,7 @@ describe('runQuickJSWorkflow', () => { workflowId: 'workflow//test//workflow', workflowRun: makeRun(), events: [], + worldSupportsHookRetention: true, }); expect(result.suspended?.pendingOperations).toContainEqual( diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index a492eba734..84fcbc795a 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -207,6 +207,15 @@ export interface QuickJSRuntimeOptions { * (eventually-consistent read after the parent's start() wrote it). */ runInput?: RunInput; + /** + * Whether the configured World supports `experimental_minRetention` for + * Hooks (`world.capabilities.hookRetention.active === true`). Gates the + * VM's `createHook()` retention path so the QuickJS engine fails closed on + * Worlds without the capability, matching the node:vm engine's + * world-capability gate (see `hook.ts`). Defaults to unsupported when + * omitted. + */ + worldSupportsHookRetention?: boolean; } // ---- VM Bootstrap Code ---- @@ -569,6 +578,13 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { if (options.isWebhook === true && options.experimental_minRetention !== undefined) { throw new Error('Webhook hooks do not support \`experimental_minRetention\`. Use a non-webhook \`createHook()\` with \`resumeHook()\`.'); } + // Fail closed when the configured World does not declare hook-retention + // support (parity with the node:vm engine's world-capability gate in + // hook.ts). The host injects \`__worldSupportsHookRetention\` per run from + // \`world.capabilities.hookRetention.active\`; absence means unsupported. + if (options.experimental_minRetention !== undefined && globalThis.__worldSupportsHookRetention !== true) { + throw new Error('The configured World does not support \`experimental_minRetention\` for Hooks.'); + } var token = options.token || globalThis.__generateNanoid(); var correlationId = "hook_" + globalThis.__generateUlid(); var isDisposed = false; @@ -968,6 +984,7 @@ export async function runQuickJSWorkflow( options: QuickJSRuntimeOptions ): Promise { const { workflowCode, workflowId, workflowRun, events } = options; + const worldSupportsHookRetention = options.worldSupportsHookRetention === true; const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); @@ -1062,6 +1079,14 @@ export async function runQuickJSWorkflow( `globalThis.__ulidTimestamp = ${runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt)};` ).dispose(); + // Whether the configured World supports Hook retention. Gates the VM's + // `createHook({ experimental_minRetention })` path so QuickJS fails + // closed on Worlds without the capability, matching the node:vm engine + // (see hook.ts's world-capability gate). + vm.evalCode( + `globalThis.__worldSupportsHookRetention = ${worldSupportsHookRetention};` + ).dispose(); + // `process.env` — parity with the node:vm engine, which exposes a frozen // copy of the host env (vm/index.ts). Injected per run so the snapshot of // the env is taken at invocation time, same as node. From 29c3515c85d105cb9487810426a9040fb0fef1e5 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Tue, 4 Aug 2026 09:02:51 -0700 Subject: [PATCH 12/14] style: wrap two long lines biome would reformat --- packages/core/src/runtime/quickjs-entrypoint.ts | 3 ++- packages/core/src/runtime/quickjs-runtime.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 6c859a1952..b2136d2fbf 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -712,7 +712,8 @@ export async function runWorkflowWithQuickJS(params: { runInput, // Fail closed on Hook retention when the World doesn't declare support, // matching the node:vm engine's world-capability gate (see hook.ts). - worldSupportsHookRetention: world.capabilities?.hookRetention?.active === true, + worldSupportsHookRetention: + world.capabilities?.hookRetention?.active === true, }); runtimeLogger.debug('QuickJS runtime: VM returned', { diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 84fcbc795a..b457756df1 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -984,7 +984,8 @@ export async function runQuickJSWorkflow( options: QuickJSRuntimeOptions ): Promise { const { workflowCode, workflowId, workflowRun, events } = options; - const worldSupportsHookRetention = options.worldSupportsHookRetention === true; + const worldSupportsHookRetention = + options.worldSupportsHookRetention === true; const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); From bd175d492fbb1116c10b6fb45f6ec098a64e5475 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Tue, 4 Aug 2026 10:55:16 -0700 Subject: [PATCH 13/14] fix(core): commit QuickJS hook writes before dispatching steps The QuickJS engine dispatched hook_created concurrently with step_created and the step's queue message, so a step that calls abort() on a signal it received could reach the hook resume before the hook row existed. The resume throws HookNotFoundError and is swallowed as best-effort, so the abort was lost with no error. The node:vm engine already settles its hook phase before its step phase; this restores parity. --- .changeset/quickjs-hook-ordering.md | 5 ++++ .../core/src/runtime/quickjs-entrypoint.ts | 27 ++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 .changeset/quickjs-hook-ordering.md diff --git a/.changeset/quickjs-hook-ordering.md b/.changeset/quickjs-hook-ordering.md new file mode 100644 index 0000000000..f61188d2fc --- /dev/null +++ b/.changeset/quickjs-hook-ordering.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Fix aborts raised inside a step being dropped on the QuickJS engine when the abort controller's hook was created in the same suspension. diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index b2136d2fbf..59b3640f0c 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -301,8 +301,9 @@ async function dispatchPendingOps(params: { // validated by the world — parallel dispatch would otherwise record a // spurious hook_conflict against the run's own disposed hook (e.g. a // dispose→recreate loop reusing one token). Different tokens have no - // claim interaction, so token groups run in parallel with each other - // and with the non-hook ops below. + // claim interaction, so token groups run in parallel with each other, + // but the whole hook phase is awaited before any step/wait op is issued + // (see the barrier below). const hookOpsByToken = new Map< string, (PendingHook | PendingHookDispose)[] @@ -327,8 +328,9 @@ async function dispatchPendingOps(params: { hookOpsByToken.set(key, [op as PendingHook | PendingHookDispose]); } } + const hookPhasePromises: Promise[] = []; for (const group of hookOpsByToken.values()) { - opsPromises.push( + hookPhasePromises.push( (async () => { for (const op of group) { if (op.type === 'hook') { @@ -340,6 +342,25 @@ async function dispatchPendingOps(params: { })() ); } + // Barrier: every hook_created is durable before any step is dispatched. + // This mirrors the node:vm suspension handler, which settles its hook + // phase before the step phase. A step that receives an AbortSignal can + // call controller.abort() as soon as it starts running, and that resume + // targets the abort controller's hook by token: if the hook row does not + // exist yet the resume throws HookNotFoundError and the abort is lost + // (serialization.ts treats the resume as best-effort). Dispatching steps + // concurrently with hook creation makes that a race decided by write + // latency. + if (hookPhasePromises.length > 0) { + // Settle rather than Promise.all so a rejecting group cannot leave its + // siblings' in-flight writes as unhandled rejections. + const rejections = (await Promise.allSettled(hookPhasePromises)) + .filter((r): r is PromiseRejectedResult => r.status === 'rejected') + .map((r) => r.reason); + if (rejections.length > 0) { + throw rejections[0]; + } + } for (const op of pendingOperations) { if (op.type === 'step' && !op.hasCreatedEvent) { From 36a789b8cac161e4a3a40f4a5b778a6bae8c6c49 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Tue, 4 Aug 2026 12:24:03 -0700 Subject: [PATCH 14/14] Latch a rejected slot claim for the rest of the batch A suspension flushes its creates concurrently, so once one of them loses its slot every sibling behind it is proposing into the same taken range. The rejection is recorded on the log and the siblings rethrow it without issuing a create, which turns an N-way fan-out's N round-trips after a conflict into one. The merged delta is also trusted on every restart of a run that numbers its events, since a dense log holds exactly `maxSlot` events and a delta that left a hole is caught by that count instead of by a reload. --- .../workflow-errors/slot-conflict-error.mdx | 6 +- packages/core/src/runtime.ts | 94 ++++++++++--------- packages/core/src/runtime/helpers.test.ts | 37 ++++++-- packages/core/src/runtime/helpers.ts | 85 ++++++++++------- .../runtime/precondition-guard-replay.test.ts | 33 ++++++- packages/core/src/runtime/step-executor.ts | 22 ++--- .../core/src/runtime/suspension-handler.ts | 5 +- packages/errors/src/index.ts | 19 ++-- packages/world-local/src/fs.ts | 8 +- .../world-local/src/storage/events-storage.ts | 41 ++++---- packages/world-local/src/storage/helpers.ts | 2 +- packages/world-local/src/storage/slots.ts | 14 +-- packages/world-postgres/src/drizzle/schema.ts | 8 +- packages/world-postgres/src/slots.ts | 13 ++- packages/world-postgres/src/storage.ts | 12 +-- packages/world-vercel/src/events-v4.ts | 13 ++- packages/world/src/events.ts | 7 +- packages/world/src/slot-identity.ts | 2 +- packages/world/src/spec-version.ts | 4 +- packages/world/src/ulid.ts | 2 +- 20 files changed, 241 insertions(+), 186 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx index 96a4f67f17..5efc190e8c 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx @@ -10,7 +10,7 @@ related: `SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics. -On a run that numbers its events by slot, each event's id encodes its position in the log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. +Each event's id encodes its position in the run's log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. The rejection carries the missing events inline so that merge usually costs no extra round-trip: @@ -18,7 +18,7 @@ The rejection carries the missing events inline so that merge usually costs no e - `cursor` — cursor to continue the delta from. - `hasMore` — whether events beyond `events` remain to be fetched. -This is the slot-numbering counterpart to [`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error), which is how the same staleness is reported for runs guarded by an event-log snapshot watermark instead. A run uses one scheme or the other for its whole life, decided when it is created. +[`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error) reports the same staleness for a run guarded by an event-log snapshot watermark. A run uses one scheme or the other for its whole life, decided when it is created. The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly. @@ -46,7 +46,7 @@ try { { const slot = eventId ? slotFromId(eventId) : undefined; @@ -1093,32 +1094,35 @@ export function workflowEntrypoint( // completed session must not be resumed again. retainedSession = null; // A World MAY return the events we were missing on the - // rejection. - // Trust it only on the FIRST restart: its completeness proof - // leans on the backend's own bookkeeping, so if that - // under-counts, a "complete" delta can still leave a hole. - // Bounding it to one attempt caps that at a single wasted - // restart; every later restart does the authoritative load. + // rejection. A merged log is checked for density before the + // replay trusts it, so a delta that left a hole costs the + // reload it would have done anyway and every restart can + // take one. A run fenced by the watermark has no such check + // — its delta's completeness rests on the backend's own + // bookkeeping — so it trusts one delta and loads + // authoritatively after that. + const slotNumbered = usesSlotIdentity( + workflowRun?.specVersion + ); const delta = - allowDelta && preconditionRestarts === 1 + allowDelta && (slotNumbered || preconditionRestarts === 1) ? preconditionEventDelta(error, runId) : null; // A delta is only usable as a delta if there is a cached log // to merge it into; with no base log the restart has to load // the whole thing anyway. const usedDelta = Boolean(delta && cachedEvents); - // Without a delta, a slot-numbered log still heals from its - // cursor rather than from a full reload. Slot ids sort in - // write order, so every event this replay was missing is - // strictly above the cursor and one incremental page brings - // it in — and density (a dense log from slot 1 holds - // exactly `maxSlot` events) proves afterwards that it did, - // so nothing is being trusted here that is not checked. - // Neither property holds under ULID ids, which is why those - // restarts reload whole. + // Without a delta, the log heals from its cursor rather + // than from a full reload. Slot ids sort in write order, so + // every event this replay was missing is strictly above the + // cursor and one incremental page brings it in, and density + // (a dense log from slot 1 holds exactly `maxSlot` events) + // proves afterwards that it did, so nothing is being trusted + // here that is not checked. Neither property holds for a + // ULID-numbered log, so those restarts reload whole. const topsUpFromCursor = !usedDelta && - usesSlotIdentity(workflowRun?.specVersion) && + slotNumbered && cachedEvents !== null && eventsCursor !== null; const restartSource = usedDelta @@ -1143,6 +1147,7 @@ export function workflowEntrypoint( // (`pendingInlineDelta && cachedEvents`) with no // events.list round trip at all. pendingInlineDelta = delta; + slotDensityCheckPending = slotNumbered; } else if (topsUpFromCursor) { // Keep the cached log and its cursor: the loop's // incremental branch fetches the page above the cursor @@ -1151,7 +1156,7 @@ export function workflowEntrypoint( // close the gap. Appends land above everything already // scanned for payload prewarming, so no rescan is needed // unless that fallback fires. - slotTopUpPending = true; + slotDensityCheckPending = true; preloadedEvents = undefined; preloadedEventsCursor = undefined; pendingInlineDelta = null; @@ -1169,6 +1174,7 @@ export function workflowEntrypoint( preloadedEvents = undefined; preloadedEventsCursor = undefined; pendingInlineDelta = null; + slotDensityCheckPending = false; // The corrected log inserts the missing events BELOW the // length already scanned for payload prewarming, shifting // every later position. Only a full rescan sees them. @@ -1618,8 +1624,8 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; - // A slot-numbered run's first two positions are the run's - // own: `run_created` from start(), then the `run_started` + // The run's first two positions are its own: + // `run_created` from start(), then the `run_started` // in flight above. Both are certain before any write of // this invocation, and turbo replays against the empty // snapshot skipped just above — so seed the floor with @@ -2159,15 +2165,15 @@ export function workflowEntrypoint( // the wait pass, which may swap in a freshly loaded array. cachedEvents = events; - if (slotTopUpPending) { - slotTopUpPending = false; - // A slot-numbered log is dense from slot 1, so a - // complete one holds exactly `maxSlot` events. A short - // count means the page above the cursor did not bring - // in everything the restart was missing — the only - // other reading, a permanent hole from a write that - // took a slot and then failed, is equally unrecoverable - // from here — so fall back to the authoritative load. + if (slotDensityCheckPending) { + slotDensityCheckPending = false; + // The log is dense from slot 1, so a complete one + // holds exactly `maxSlot` events. A short + // count means the merge did not bring in everything the + // restart was missing — the only other reading, a + // permanent hole from a write that took a slot and then + // failed, is equally unrecoverable from here — so fall + // back to the authoritative load. if (maxSlotOf(events) !== events.length) { const loaded = await loadWorkflowRunEvents(runId); events = loaded.events; @@ -3057,10 +3063,10 @@ export function workflowEntrypoint( // (threaded below via `claimFenceFor`; on rejection // the batch is abandoned and re-invoked for a fresh // replay, so a stale view can never commit a step). - // A slot-numbered run gets there differently — the - // claim merges the missed events and retries in - // place, so the same events are observed without - // discarding the batch. See claimFenceFor. + // A slot claim gets there differently: it merges the + // missed events and retries in place, so the same + // events are observed without discarding the batch. + // See claimFenceFor. // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 78b11985a2..5d16a9b9f8 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -879,18 +879,37 @@ describe('claimFenceFor', () => { expect(eventsListMock).not.toHaveBeenCalled(); }); - it('leaves the rest of the batch claiming into the occupied range', async () => { - // The tail stops advancing for this log while the backend's moves on, so - // the siblings behind a rejected claim propose slots the backend has - // already filled and are rejected with it. + it('fails the rest of the batch without a round-trip', async () => { + // The siblings behind a rejected claim were decided from the same log, so + // they carry the fence it just proved wrong. They rethrow that rejection + // instead of spending a create each to be told the same thing. const log = toMutableEventLog([slotEvent(1)], 'c0'); const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); - const loser = claim(async (fence) => { - throw new SlotConflictError('taken', { - eventId: fence?.eventId as string, - }); + const rejection = new SlotConflictError('taken', { + eventId: slotEventId(2), + }); + const loser = claim(async () => { + throw rejection; }); - await expect(loser).rejects.toBeInstanceOf(SlotConflictError); + await expect(loser).rejects.toBe(rejection); + + const sibling = vi.fn( + async (fence?: { eventId?: string }) => fence?.eventId + ); + await expect(claim(sibling)).rejects.toBe(rejection); + expect(sibling).not.toHaveBeenCalled(); + }); + + it('lets a write that failed without taking its slot claim it again', async () => { + // An entity conflict means the event never landed, so the slot is still + // free. Only a stale-write rejection stops the log. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + await expect( + claim(async () => { + throw new EntityConflictError('wait already completed'); + }) + ).rejects.toBeInstanceOf(EntityConflictError); await expect(claim(async (fence) => fence?.eventId)).resolves.toBe( slotEventId(2) diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index d9e367cf05..56788e4472 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -487,8 +487,8 @@ function recordRequestedEventCursor( } /** - * Appends events whose IDs are not already present in `target`, keeping a - * slot-numbered log in slot order. + * Appends events whose IDs are not already present in `target`, keeping the log + * in slot order. * * Arrival order is not log order under slot identity. A slot is reserved when * its event is issued and written when that issue resolves, so a lower slot can @@ -501,8 +501,7 @@ function recordRequestedEventCursor( * next event it reads belongs to a step it never started. * * Sorting by event id *is* sorting by slot: ids are zero-padded to a fixed - * width, and a slot-numbered run's log carries no ULID ids to interleave with - * them. + * width, and one log never mixes them with ULID ids. * * Pass the IDs currently present in `target` when appending repeatedly to the * same array. The set is updated alongside `target`. @@ -750,11 +749,18 @@ export interface MutableEventLog extends LoadedEventLog { maxSlot: number; /** * Next slot `reserveSlot` will hand out. Only a writer holding the log's - * write chain may draw from it, and it is rewound to `maxSlot + 1` when a - * claim is rejected so the rest of the batch claims into the occupied range - * and is rejected with it. + * write chain may draw from it. */ nextSlot: number; + /** + * The stale-write rejection this log's tail stopped at, if any. + * + * A rejected claim means the log is missing an event, so nothing else decided + * from it may land either. Recording the rejection here fails the rest of the + * batch locally, instead of spending a round-trip each to be told the same + * thing by the backend. + */ + claimRejection?: unknown; /** * Tail of the chain of creates numbered off this log, or `undefined` when * none is in flight. @@ -794,6 +800,9 @@ export function toMutableEventLog( /** * Merges loaded events into `log` in place, keeping `maxSlot` current and * advancing the reservation pointer past the events merged in. + * + * The merged events are the ones a rejected claim was missing, so the log is + * usable again and its recorded rejection is cleared. */ export function mergeLoadedEvents( log: MutableEventLog, @@ -802,6 +811,7 @@ export function mergeLoadedEvents( appendUniqueEvents(log.events, events); log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); + log.claimRejection = undefined; } /** @@ -936,10 +946,9 @@ export function preconditionSnapshotParams( * Whether a World rejected an event creation because the replay that produced it * had not seen the whole event log. * - * The two schemes reject differently and stay separately countable — 412 for the - * event-log watermark, 409 for a lost slot claim, both live at once while runs on - * the older numbering drain — but they prove the same thing and are recovered the - * same way, by restarting the replay over the corrected log. + * A lost slot claim rejects with 409 and a failed watermark comparison with 412, + * so the two stay separately countable. They prove the same thing and are + * recovered the same way, by restarting the replay over the corrected log. */ export function isStaleWriteRejection(error: unknown): boolean { return PreconditionFailedError.is(error) || SlotConflictError.is(error); @@ -1028,15 +1037,14 @@ export interface EventCreateFence extends PreconditionSnapshotParams { * * Neither scheme is retried in place. A rejection under either one proves the * replay derived this event from a log that was missing another, and correlation - * ids are positional ordinals of one sequence — so a replay over the corrected - * log mints different ids and re-posting this write would persist an event no - * correct replay produces. Recovery is a restarted replay + * ids are drawn from one seeded sequence in mint order — so a replay over the + * corrected log mints different ids and re-posting this write would persist an + * event no correct replay produces. Recovery is a restarted replay * ({@link isStaleWriteRejection}), never a re-send. * - * Claims a slot off `log` for a slot-numbered run — which counts as a - * reservation, so a caller that fences several creates from one log gets a - * distinct slot per create. Empty when the run is fenced neither way, leaving - * the create exactly as unfenced as it was before either mechanism existed. + * Claiming a slot counts as reserving it, so a caller that fences several + * creates from one log gets a distinct slot per create. Empty when the run is + * fenced neither way, which leaves the create unfenced. * * `extraEvents` is how many events *besides* the one being created this write * publishes: a lazy inline `step_started` also materializes the `step_created` @@ -1056,7 +1064,7 @@ export function eventCreateFenceFor( options?: { extraEvents?: number } ): EventCreateFence { if (usesSlotIdentity(specVersion)) { - return reserveSlotFence(log, options?.extraEvents ?? 0); + return reserveSlotFence(log, options?.extraEvents ?? 0).fence; } return preconditionSnapshotParams(log.events, log.cursor); } @@ -1073,17 +1081,18 @@ export function eventCreateFenceFor( function reserveSlotFence( log: MutableEventLog, extraEvents: number -): EventCreateFence { +): { fence: EventCreateFence; slot: number } { const maxSlot = log.maxSlot; for (let i = 0; i < extraEvents; i++) { reserveSlot(log); } - return { eventId: slotEventId(reserveSlot(log)), maxSlot }; + const slot = reserveSlot(log); + return { fence: { eventId: slotEventId(slot), maxSlot }, slot }; } /** - * Runs one slot-numbered create with the log's claim to itself, taking its slot - * only once every create ahead of it on the log has settled. + * Runs one create with the log's claim to itself, taking its slot only once + * every create ahead of it on the log has settled. * * A slot claim is an assertion about the tail: "nothing has been published * since the view I decided from". Claims handed out up front to a concurrent @@ -1093,11 +1102,11 @@ function reserveSlotFence( * it. Taking claims one at a time keeps every write's fence tight against the * tail the writer actually saw. * - * A rejection therefore stops the whole batch rather than only its own write: - * the log's tail stops advancing while the backend's moves on, so the claims - * behind it fall inside the occupied range and are rejected in turn. That is - * the intent — the batch was decided from a log missing an event, so none of it - * should land. + * A stale-write rejection therefore stops the whole batch rather than only its + * own write: the batch was decided from a log missing an event, so none of it + * should land. The rejection is recorded on the log and rethrown for the claims + * behind it without a round-trip, since their fences all name the tail it just + * proved wrong. */ async function withSerializedClaim( log: MutableEventLog, @@ -1113,19 +1122,23 @@ async function withSerializedClaim( await ahead; } try { - const fence = reserveSlotFence(log, extraEvents); + if (log.claimRejection !== undefined) { + throw log.claimRejection; + } + const { fence, slot } = reserveSlotFence(log, extraEvents); const result = await op(fence); - log.maxSlot = Math.max( - log.maxSlot, - maxSlotOf([{ eventId: fence.eventId ?? '' }]) - ); + log.maxSlot = Math.max(log.maxSlot, slot); log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); return result; } catch (error) { - // The slots this attempt drew are not the writer's, and the tail is at - // least as high as the claim that lost. Rewinding onto the occupied range - // is what makes the rest of the batch fail with it. + // The slots this attempt drew are not the writer's, and the tail is no + // lower than the claim that lost. Rewinding leaves the next claim naming + // the same slot, which is correct for a write that failed without taking it + // (an entity conflict, say) and is retried on its own. log.nextSlot = log.maxSlot + 1; + if (isStaleWriteRejection(error)) { + log.claimRejection = error; + } throw error; } finally { done(); diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index 0fc8b40ebc..a6baf45da0 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -10,10 +10,10 @@ * 2. The restart reloads the whole event log with no cursor, because a hole is * defined by ULID time while a cursor filters lexicographically — unless * the World attached the missing events to the 412, which the runtime - * consumes with no events.list round trip at all (first restart only). A run - * numbering its events by slot instead heals from its cursor, since slot ids - * sort in write order and density proves afterwards that the page closed the - * gap; a short count sends it back to the full reload. + * consumes with no events.list round trip at all. A run numbering its events + * by slot heals from its cursor when nothing was attached, since slot ids + * sort in write order and density proves afterwards that the merge closed + * the gap; a short count sends it back to the full reload. * 3. Restarts are bounded; once the bound is spent the runtime schedules a * delayed re-invocation instead of failing the run — and that escalation is * itself counted on the queue message, so a run that can never observe its @@ -1263,6 +1263,31 @@ describe('precondition guard through the real replay loop', () => { expect(cursorlessLoads(result.listEvents)).toBe(1); }); + it('trusts an attached delta on every restart of a slot-numbered run', async () => { + const warn = vi.spyOn(runtimeLogger, 'warn'); + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 2, + attachDelta: 'complete', + }); + await result.handlerInvocation; + + // Density checks the merged log here, so a delta that left a hole would be + // caught rather than replayed over. Nothing rests on the World's own + // bookkeeping, and no restart has to spend a load of any kind to be sure. + expect(result.waitCompletedRejectionCount()).toBe(2); + expect(cursorlessLoads(result.listEvents)).toBe(0); + expect( + warn.mock.calls + .filter( + ([message]) => + message === + 'Event creation rejected as stale; restarting replay in-process' + ) + .map(([, fields]) => (fields as { source?: string }).source) + ).toEqual(['inline-delta', 'inline-delta']); + }); + it('reports whether a restarted replay reloaded the events it was missing', async () => { const warn = vi.spyOn(runtimeLogger, 'warn'); const result = await runPreconditionScenario({ diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index dd8a36d2d6..8fbb78c13b 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -211,9 +211,7 @@ export interface StepExecutorParams { inlineDeltaSinceCursor?: string; /** * Runs this step's `step_started` claim under its run's concurrency fence: - * the event slot the claim occupies, or the caller's replay snapshot - * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the - * older numbering. + * the event slot the claim occupies. * * On the lazy inline path the claim is the step's FIRST durable write (its * `step_created` is deferred), so without a fence it would be unguarded @@ -222,13 +220,13 @@ export interface StepExecutorParams { * rejects such a claim with `SlotConflictError` (409) or * `PreconditionFailedError` (412). * - * Whether a rejection is retried in place is the caller's decision, made per - * scheme — see `claimFenceFor`. Either way executeStep does NOT translate a - * rejection that reaches it, so an unretried one propagates for the caller to - * abandon the batch and force a fresh replay. + * Whether a rejection is retried in place is the caller's decision, see + * `claimFenceFor`. Either way executeStep does NOT translate a rejection that + * reaches it, so an unretried one propagates for the caller to abandon the + * batch and force a fresh replay. * - * Undefined when the caller has no snapshot, or when the watermark guard is - * disabled on a run that uses it; Worlds that fence neither way ignore it. + * Undefined when the caller has no snapshot; a World that does not fence + * ignores it. */ claimFence?: FencedCreate; /** @@ -344,9 +342,9 @@ export async function executeStep( // put us on the Vercel branch with nothing to build a host from, making // `https://` the base URL of every step. const isVercel = Boolean(process.env.VERCEL_URL); - // Unfenced when the caller passes no fence — every World that fences - // ignores the field it does not understand, so this is the same create it - // was before either mechanism existed. + // Unfenced when the caller passes no fence, which runs the create plainly. A + // World that does not fence also ignores a fence it is handed, so passing one + // is never wrong. const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index ecb9cf4ed3..02b1eb815f 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -45,9 +45,8 @@ export interface SuspensionHandlerParams { requestId?: string; /** * The runtime's loaded event log. Every event creation this suspension makes - * carries a fence derived from it — its own event slot, or the snapshot's - * watermark for a run on the older numbering — so a backend that has recorded - * an event the replay did not see rejects the write (409/412) instead of + * carries a fence derived from it, the event slot it claims, so a backend that + * has recorded an event the replay did not see rejects the write instead of * accepting a divergent event. The rejection is not retried here: the event's * correlation id was minted by *this* replay's seeded sequence, so * re-committing it against a corrected log would persist an event no correct diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 0c547c00aa..35467bbcce 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -825,21 +825,20 @@ export class PreconditionFailedError extends WorkflowWorldError { * Thrown when the backend rejects an event creation because the event slot the * client named was already taken by another writer (HTTP 409). * - * On a run that numbers its events by slot, whoever writes a slot first owns - * it, and the loser has by definition been replaying against an event log - * missing at least one event. Retrying the same write can therefore never - * succeed: the client has to merge the events it was missing, replay, and - * propose whatever slot that replay lands on. The rejection carries those - * events inline so the common case costs no extra round-trip. + * Runs name their events by position in the log, and whoever writes a position + * first owns it. The loser has by definition been replaying against an event log + * missing at least one event, so retrying the same write can never succeed: the + * client has to merge the events it was missing, replay, and propose whatever + * slot that replay lands on. The rejection carries those events inline so the + * common case costs no extra round-trip. * - * Distinct from `PreconditionFailedError` (412), which is the equivalent - * rejection for a run guarded by the `stateUpdatedAt` watermark instead. Both - * mechanisms are live at once while runs on the older numbering drain. + * `PreconditionFailedError` (412) is the equivalent rejection for a run whose + * events are guarded by the `stateUpdatedAt` watermark. * * The workflow runtime handles this automatically. Users interacting with world * storage backends directly may encounter it. * - * @property eventId - The slot-numbered event id that was already taken. + * @property eventId - The event id, naming a slot, that was already taken. * @property events - The events recorded after the client's snapshot, in * ascending slot order. Empty when the backend could not read them, in which * case the client reloads the log itself. diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index c01d0926f0..17b2ec2089 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -587,10 +587,10 @@ interface PaginatedFileSystemQueryConfig { getId?(item: T): string; /** * The time an item sorts and paginates by, when that is not its `createdAt`. - * A slot-numbered event log orders by slot — the position is the order — and - * a writer that loses a slot re-proposes above the winner while keeping the - * stamp it started with, so `createdAt` there disagrees with the log. Such an - * item reports one shared time and lets the `getId` tie-break order it. + * An event log orders by slot — the position is the order — and a writer that + * loses a slot re-proposes above the winner while keeping the stamp it started + * with, so `createdAt` there disagrees with the log. Such an item reports one + * shared time and lets the `getId` tie-break order it. */ getOrderTime?: (item: NoInfer) => number; } diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index a325cb977d..9a809bd2ed 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -148,15 +148,14 @@ function getHookRetentionLimitMs(): number { // cross-process Hook token handoffs use `withHookTokenClaimLock`. /** - * The time an event orders and paginates by. A slot-numbered run's order is its - * slot order — the position *is* the order, the way the sort key is for the - * other backends — so every such event reports the same time and lets the - * event-id tie-break do the ordering. Ordering those by `createdAt` reads the - * log in an order no replay produced: a writer that loses a slot re-proposes - * above the winner while keeping the stamp it started with, and a caller that - * reserves slots for a whole flush commits them in whatever order the network - * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree - * with anyway. + * The time an event orders and paginates by. Slot order is the log's order — + * the position *is* the order, the way the sort key is for the other backends — + * so every slot-named event reports the same time and lets the event-id + * tie-break do the ordering. Ordering those by `createdAt` reads the log in an + * order no replay produced: a writer that loses a slot re-proposes above the + * winner while keeping the stamp it started with, and a caller that reserves + * slots for a whole flush commits them in whatever order the network returns. A + * ULID-numbered run keeps its wall-clock order, which its ids agree with anyway. */ const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; @@ -308,8 +307,8 @@ async function findExistingHookCreatedEventId( /** * The run's committed `hook_received` for one resume, if it has one. * - * The resume claim records where the event is going to be published, and in a - * slot-numbered run that is only a hint: another instance can allocate the same + * The resume claim records where the event is going to be published, and that + * is only a hint: another instance can allocate the same * position for an unrelated event from its own book, and then the claim points * at a stranger. The key the event itself carries is the durable identity, so * this scan is what decides whether a resume has already been recorded. @@ -771,8 +770,8 @@ export function createEventsStorage( assertSafeEntityId('eventId', params.eventId); } - // A slot-numbered create reserves its position before running the - // validation and materialization that may still reject it. Handing the + // A create reserves its position before running the validation and + // materialization that may still reject it. Handing the // reservation back on the way out — whether the create throws or returns // another writer's event — is what keeps the log dense: an abandoned slot // below a sibling's published one is a hole that can never be filled, and @@ -1135,11 +1134,11 @@ export function createEventsStorage( // free one: a `run_started` racing it (start() issues the creation and // the queue send in parallel) may already have moved the book past it. const ownsFirstSlot = data.eventType === 'run_created'; - // A slot-numbered run's ids name positions in its log, so an id is - // either claimed by a caller that holds the log (and is therefore - // asserting the log is complete up to that position) or allocated here - // for a caller that has no log — a step completion reporting in, a - // cancellation from an API call. + // An event id names a position in the run's log, so it is either claimed + // by a caller that holds the log (and is therefore asserting the log is + // complete up to that position) or allocated here for a caller that has + // no log — a step completion reporting in, a cancellation from an API + // call. // // The position of the second event a lazy start publishes, when this // one publishes two. Consumed by the materialization below; released @@ -1453,9 +1452,9 @@ export function createEventsStorage( tag ); // The event the claim points at answers for this resume only if it - // is this resume's. In a slot-numbered run the pinned position can - // hold an unrelated event another instance allocated it for, and - // returning that would report a step's event as the resume's. + // is this resume's. The pinned position can hold an unrelated + // event another instance allocated it for, and returning that + // would report a step's event as the resume's. if (pinned?.resumeId === resumeId) { return { event: pinned }; } diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index f8e264988e..4d18306042 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -276,7 +276,7 @@ export async function listRunEventIds( * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. * - * A slot-numbered run takes the slot above the highest visible one, which + * In slot mode the event takes the slot above the highest visible one, which * dominates by construction, paired with the wall clock — `createdAt` needs * only to be >= every visible one, by the same argument as above. This is * the one allocation that deliberately does *not* fill a hole below the max: diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 2b5c3948b4..8e4a777307 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -1,8 +1,8 @@ /** * Slot allocation for the Local World. * - * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second. A replay reads the log in slot + * Runs name their events by position: `evnt_…001` is the first event of the run, + * `evnt_…002` the second. A replay reads the log in slot * order, so the order slots are handed out in has to be an order some execution * could have produced — which makes allocation strictly *append-only*: a slot is * only ever handed out above every position this book has seen. @@ -71,10 +71,10 @@ export interface SlotBook { /** * Whether `runId`'s events are numbered by slot, read from the run's * persisted `specVersion` — never from the build, so a run stays in the mode - * it was created in for life. A run that does not exist yet is not - * slot-numbered, and that answer is not cached: the resilient-start path - * creates the run moments later, and caching "no" would strand it on ULIDs - * for the rest of this process's life. + * it was created in for life. A run that does not exist yet answers `false`, + * and that answer is not cached: the resilient-start path creates the run + * moments later, and caching "no" would strand it on ULID ids for the rest of + * this process's life. */ usesSlots(runId: string): Promise; /** @@ -148,7 +148,7 @@ export interface SlotBook { } export function createSlotBook(basedir: string, tag?: string): SlotBook { - /** runId → whether the run is slot-numbered, memoized once it exists. */ + /** runId → whether the run numbers by slot, memoized once it exists. */ const modes = new Map(); const books = new Map(); /** runId → in-flight seed scan, so concurrent first callers share one scan. */ diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 31b098a8c0..a17748c2df 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -153,10 +153,10 @@ export const events = schema.table( > >, (tb) => [ - // Event ids are only unique within their run: under slot identity every run - // numbers its own log from 1, so `evnt_0…001` exists once per run. The run - // leads the key so the range scans in `list` stay a single index seek, and - // it subsumes the plain `run_id` index the table used to carry. + // Event ids are only unique within their run: every run numbers its own log + // from 1, so `evnt_0…001` exists once per run. The run leads the key so the + // range scans in `list` stay a single index seek, which also makes a + // separate `run_id` index redundant. primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 6a0366a636..ede2a0efa5 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -1,8 +1,8 @@ /** * Slot identity for the postgres world. * - * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second. Contention on a position is the + * Runs name their events by position: `evnt_…001` is the first event of the + * run, `evnt_…002` the second. Contention on a position is the * point — it is what makes a concurrent write detectable rather than silent — * so the two things this module has to get right are that a position is written * at most once and that a position this allocator loses is retried rather than @@ -60,9 +60,9 @@ export function isEventKeyViolation(error: unknown): boolean { * The highest event id in a run's log, or undefined when the log is empty. * * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width - * within a scheme, so for a slot-numbered run the highest id names the highest - * written position — and because a log holds ids of exactly one scheme, that id - * also reports which scheme the run was created with. + * within a scheme, so the highest id names the highest written position. A log + * holds ids of exactly one scheme, so that id also reports which scheme the run + * was created with. */ export async function highestEventId( drizzle: Drizzle, @@ -140,8 +140,7 @@ export interface PlaceEventOptions { } /** - * Writes an event of a slot-numbered run, at the position the caller claimed or - * at the next free one. + * Writes an event at the position the caller claimed, or at the next free one. * * Every round re-probes rather than incrementing a local counter: each round at * least one writer wins, so re-probing guarantees progress under any amount of diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 7d3c8727c6..4e5bea1ceb 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -811,8 +811,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (slotMode === undefined) { // step_completed and step_retrying skip the run read above. The log's // own highest id reports the scheme, since a log holds ids of exactly - // one, and it is the probe the allocator needs anyway — so a - // slot-numbered run pays nothing extra for this query. + // one, and it is the probe the allocator needs anyway, so the query + // costs a slot-mode write nothing extra. seedHighestEventId = await highestEventId(drizzle, effectiveRunId); slotMode = isSlotId(seedHighestEventId ?? ''); } @@ -820,10 +820,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ - // A slot-numbered run's ids name positions in its log, so an id is either - // claimed by a caller that holds the log (and is therefore asserting the - // log is complete up to that position) or allocated at write time for a - // caller that has no log — a step completion reporting in, a cancellation + // An event id names a position in the run's log, so it is either claimed + // by a caller that holds the log (and is therefore asserting the log is + // complete up to that position) or allocated at write time for a caller + // that has no log — a step completion reporting in, a cancellation // from an API call. let claimedSlot: number | undefined; if (params?.eventId !== undefined) { diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 170dc3d567..0015d9de03 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -239,13 +239,12 @@ export interface CreateEventV4Input { */ stateUpdatedAt?: number; /** - * The event's id, claimed by the client instead of minted by the server. - * Sent only for a run that numbers its events by slot, where the id encodes - * the event's position in the log. The server inserts it conditionally and - * answers 409 `slot-conflict` when the slot is already taken; a run on the - * older numbering that sends one is rejected with 400. Older servers ignore - * the field and mint an id as before — which is why only runs stamped with - * slot identity ever send it. + * The event's id, claimed by the client rather than minted by the server. It + * encodes the event's position in the run's log. The server inserts it + * conditionally and answers 409 `slot-conflict` when the slot is already + * taken. Sent only for a run stamped with slot identity: a run guarded by the + * watermark instead is rejected with 400, and a server that predates the field + * ignores it and mints an id of its own. */ eventId?: string; /** diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index f94d707643..9df02e6b8d 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -885,14 +885,13 @@ export interface CreateEventParams { * partly by one and partly by the other and no replay can read it in the order * it was written. * - * A World that ignores this field keeps minting ids itself, which is correct - * only for runs that were never stamped with slot identity in the first - * place. + * A World that ignores this field mints ids itself, which is correct only for + * a run that was never stamped with slot identity. */ eventId?: string; /** * The highest slot the client has observed in the run's event log, or 0 for a - * log with no slot-numbered events. Sent alongside {@link eventId} purely as + * log holding no slot ids. Sent alongside {@link eventId} purely as * an observability signal: because slots are dense, a persisted slot more * than one past this is a hole, which is unrecoverable and worth alerting on. * Worlds MAY ignore it. diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index 286e910c39..256bcf8837 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -103,7 +103,7 @@ export function slotRetryDelay(round: number): number { } /** - * The highest slot named by any of `events`, or 0 when none is slot-numbered. + * The highest slot named by any of `events`, or 0 when none names a slot. * * Scans rather than reading the last element: a log is merged from several * loads and is not necessarily sorted, and callers use this value to pick the diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index 2d8f4bed5b..f74977d93f 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -42,8 +42,8 @@ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; * event ids and is replayed by a slot-capable build would otherwise propose * `evnt_…001`, a position its very first event already occupies. * - * Correlation ids are unaffected: steps, waits, hooks and attributes keep their - * seeded ULIDs in both modes. + * Correlation ids are unrelated: steps, waits, hooks and attributes use seeded + * ULIDs at every spec version. */ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index cc893834c3..92dbbd4d66 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -44,7 +44,7 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = * position, not a time, so it is reported here as having no time at all — * callers must read the object's own `createdAt`. Silently returning 1970 * instead would, among other things, rewind a replaying workflow's clock and - * make cursor pagination skip every slot-numbered event. + * make cursor pagination skip every event named by a slot. */ export function ulidToDate(maybeUlid: string): Date | null { if (isSlotId(maybeUlid)) {