diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/.changeset/per-kind-correlation-ids.md b/.changeset/per-kind-correlation-ids.md deleted file mode 100644 index 2e13a31a73..0000000000 --- a/.changeset/per-kind-correlation-ids.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Add env option to split correlation ID derivation into per-entity-type sequential ULIDs, instead of sharing one derivation source diff --git a/.changeset/quickjs-hook-ordering.md b/.changeset/quickjs-hook-ordering.md new file mode 100644 index 0000000000..f61188d2fc --- /dev/null +++ b/.changeset/quickjs-hook-ordering.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Fix aborts raised inside a step being dropped on the QuickJS engine when the abort controller's hook was created in the same suspension. diff --git a/.changeset/slot-duplicate-start-skip.md b/.changeset/slot-duplicate-start-skip.md new file mode 100644 index 0000000000..fc434c6abf --- /dev/null +++ b/.changeset/slot-duplicate-start-skip.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Skip a step whose concurrent start another handler already wrote, instead of restarting the replay to rediscover it diff --git a/.changeset/slot-event-identity.md b/.changeset/slot-event-identity.md new file mode 100644 index 0000000000..3d1fda51cb --- /dev/null +++ b/.changeset/slot-event-identity.md @@ -0,0 +1,11 @@ +--- +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/world': minor +'@workflow/errors': minor +'@workflow/core': minor +'workflow': minor +--- + +Number new runs' events by position instead of by ULID, in every World, so a reader can prove its copy of an event log is complete. A creation whose position another writer already took is rejected with a `SlotConflictError`, and the runtime merges the events it was missing, replays, and claims a free position. Correlation IDs now come from a sequence per entity type rather than one shared across all of them, so creating a hook or a sleep no longer renames the steps after it. diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/.github/scripts/render-event-log-race-repro-results.js b/.github/scripts/render-event-log-race-repro-results.js index bbead520b2..c0cc1aadfd 100644 --- a/.github/scripts/render-event-log-race-repro-results.js +++ b/.github/scripts/render-event-log-race-repro-results.js @@ -407,6 +407,11 @@ function renderLatestFailures(entry) { `\nShowing 20 of ${entry.failing.length + entry.truncatedFailingCount} non-completed runs.` ); } + // Deliberately not inlined here: the slices are large and this comment has a + // size limit, while the artifact has none. + console.log( + '\nThe `event-log-race-repro-results` artifact carries a window of the committed log around the divergent event, for a sample of the corruptions.' + ); console.log(''); } diff --git a/.gitignore b/.gitignore index a3f813e31a..3241f9667c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ workbench/nextjs-*/public/.well-known/workflow workbench/sveltekit/static/.well-known/workflow +# Local e2e diagnostics dumps +e2e-diagnostics-*.json + # Event log race repro output (written to the repo root by the harness and by # scripts/event-log-race-repro-local.sh) event-log-race-repro-results.json diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index bd4c061ada..540fc5ccef 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -82,6 +82,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when an event creation is rejected because the client's event-log snapshot is stale. + + Thrown when an event creation is rejected because another writer already took the event's slot. + Thrown when a request is made before the system is ready to process it. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/meta.json b/docs/content/docs/v5/api-reference/workflow-errors/meta.json index a84eddd1ad..b2de769e0e 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/meta.json +++ b/docs/content/docs/v5/api-reference/workflow-errors/meta.json @@ -15,6 +15,7 @@ "throttle-error", "entity-conflict-error", "precondition-failed-error", + "slot-conflict-error", "run-expired-error", "run-not-supported-error", "too-early-error" diff --git a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx new file mode 100644 index 0000000000..5efc190e8c --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx @@ -0,0 +1,76 @@ +--- +title: SlotConflictError +description: Thrown when an event creation is rejected because another writer already took the event's slot. +type: reference +summary: Catch SlotConflictError when a world rejects an event creation whose slot in the run's event log was already taken. +related: + - /docs/api-reference/workflow-errors/workflow-world-error + - /docs/api-reference/workflow-errors/precondition-failed-error +--- + +`SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics. + +Each event's id encodes its position in the run's log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. + +The rejection carries the missing events inline so that merge usually costs no extra round-trip: + +- `events` — the events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them, in which case the client reloads the log itself. +- `cursor` — cursor to continue the delta from. +- `hasMore` — whether events beyond `events` remain to be fetched. + +[`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error) reports the same staleness for a run guarded by an event-log snapshot watermark. A run uses one scheme or the other for its whole life, decided when it is created. + + +The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly. + + +```typescript lineNumbers +import { SlotConflictError } from "workflow/errors" +declare const world: { events: { create(...args: any[]): Promise } }; // @setup +declare const runId: string; // @setup +declare const event: any; // @setup + +try { + await world.events.create(runId, event); +} catch (error) { + if (SlotConflictError.is(error)) { // [!code highlight] + console.log(`Slot ${error.eventId} taken; ${error.events.length} event(s) missed`); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `SlotConflictError.is(value)` + +Type-safe check for `SlotConflictError` instances. Preferred over `instanceof` because it works across module boundaries and VM contexts. + +```typescript +import { SlotConflictError } from "workflow/errors" +declare const error: unknown; // @setup + +if (SlotConflictError.is(error)) { + // error is typed as SlotConflictError +} +``` diff --git a/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx b/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx index af7248571f..7787378f9a 100644 --- a/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx +++ b/docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx @@ -289,6 +289,8 @@ const result = await world.hooks.list({ // [!code highlight] **Returns:** `{ data: Hook[], cursor?: string }` +Hooks are ordered by hook ID. The hooks a workflow creates are listed in the order it created them. A hook the runtime creates on your behalf, such as the one backing an abort controller, draws its ID from a separate sequence, so it sorts at an arbitrary position among yours rather than at its creation position. + ### Hook Type | Field | Type | Description | diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 96b0ae936b..dd5ae29eba 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -69,9 +69,11 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` -- Default: `3` +- Default: `3`, or `12` on a run numbering its events by position (see [`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) - How many times a single invocation restarts its replay in-process after a rejected event creation before it falls back to a re-invocation. - A restart reloads the event log and rebuilds the workflow from scratch, so it costs a replay but no queue round trip. A World may attach the missing events to its rejection, in which case the first restart needs no event-log request at all. +- A run numbering its events by position reads only the page after its cursor instead of reloading the log, since positions are allocated in order and every event it was missing sorts above what it already has. Restarts are cheap enough there that the higher default is worth taking before a re-invocation and its delay. +- Setting this overrides both defaults. ### `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS` @@ -85,16 +87,17 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Delay before a re-invocation caused by a rejected event creation. - Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up — so the delay gives the other writers a moment to quiesce. -### `WORKFLOW_PER_KIND_CORRELATION_IDS` +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS` -- Default: disabled -- Experimental. Gives each kind of entity a workflow creates — steps, waits, hooks, attribute writes, abort controllers, stream IDs — its own sequence of correlation IDs. -- With one sequence shared by every kind, an ID is an ordinal over the whole run, so a single extra draw of any kind shifts every ID after it. Two concurrent replays of the same run that disagree about one `sleep()` then assign different IDs to every step that follows, and each writes events the other can neither match nor consume, which fails the run with `CORRUPTED_EVENT_LOG`. Per-kind sequences confine that to the kind that actually differs. -- IDs remain ordered within a kind, so hooks created by your workflow are still listed in creation order. A hook the runtime creates for you, such as the one backing an abort controller, draws from its own kind and so is listed at an arbitrary position relative to your hooks rather than at its creation position. -- A run must replay under the scheme that minted its IDs. A replay that switches schemes mid-run assigns IDs its own earlier events do not carry, so it can consume none of them and the run fails. - - On Vercel, a run keeps replaying on the deployment it started on, so it only ever sees the value baked into that deployment. Changing the setting affects new runs only. - - Elsewhere — `@workflow/world-postgres`, `@workflow/world-local`, any self-hosted process — nothing pins a run to the code that started it. Turn the setting on during a quiet window with no runs in flight, and roll the new value out to your whole fleet at once: a rolling deploy that leaves both values live replays one run under two schemes concurrently, which is the failure the setting exists to reduce. -- Set `1` to enable. +- Default: `15` +- Base for the randomized wait before an in-process replay restart re-derives, in milliseconds. The wait doubles with each restart the invocation has spent, up to [`WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`](#workflow_precondition_restart_backoff_max_ms), and is drawn uniformly from zero to that bound. +- The wait exists for runs with several replays in flight at once — a fan-out of steps completing together, or a burst of hooks. A rejected event creation means another replay got there first; if every loser re-derives immediately they all contend again, and none of them pulls far enough ahead to finish. Drawing each wait from the full range spreads the retries apart. +- Set to `0` to restart without waiting. + +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS` + +- Default: `400` +- Ceiling on the wait described above. ## Inline execution @@ -262,6 +265,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index b577df7b5c..c2895ff944 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -137,6 +137,31 @@ interface ReproRunResult { resumesFailed: number; stragglers?: number; }; + /** + * The committed log around the divergent event, for corruptions only. A + * divergence is a disagreement between the order the log records and the + * order a replay reconstructs, so the log's own ordering is the only + * evidence that distinguishes the candidate causes — and the run is on an + * ephemeral preview deployment, so it has to be captured while the job is + * still running rather than read back afterwards. + */ + logSlice?: LogSliceEntry[]; +} + +/** + * One committed event, projected to the fields that decide replay order: + * its position (`slot`), what it resolves (`eventType`/`correlationId`), and + * both clock domains. `occurredAt` is the client/VM moment and `createdAt` + * the persisted event time the sandbox clock is driven from; a race between a + * `sleep` and a step is decided by that clock, so the two have to be + * comparable side by side. + */ +interface LogSliceEntry { + slot: number | string; + eventType: string; + correlationId?: string; + occurredAt?: string; + createdAt?: string; } function envNumber(name: string, fallback: number) { @@ -392,6 +417,98 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + +/** + * How many events either side of the divergent one to keep. The window has to + * span a whole round of the storm — width branches, each with a step create, + * start and completion, plus the round's waits — or it can miss the very + * event whose position explains the divergence. + */ +const LOG_SLICE_RADIUS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RADIUS', 45); + +/** + * Cap on how many corruptions carry a slice. The results JSON is rendered into + * a PR comment, and a body over GitHub's limit is rejected outright, so the + * slices are a sample rather than a complete record. + */ +const LOG_SLICE_MAX_RUNS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RUNS', 6); + +let logSlicesCaptured = 0; + +/** Worlds hand timestamps back as a Date or as the stored ISO string. */ +function isoOrUndefined(value: unknown): string | undefined { + if (value instanceof Date) return value.toISOString(); + return typeof value === 'string' ? value : undefined; +} + +/** Ordinal of a slot-numbered id, or the raw id when it is a ULID. */ +function idOrdinal(id: string): number | string { + const body = id.slice(id.indexOf('_') + 1); + return /^\d+$/.test(body) ? Number(body) : id; +} + +/** + * Reads the committed log and returns the window around the divergent event + * named in `message`. Best-effort: the report is a measurement, so a failed + * read costs a slice rather than the attempt's classification. + */ +async function readLogSlice( + runId: string, + message: string | undefined +): Promise { + if (logSlicesCaptured >= LOG_SLICE_MAX_RUNS) return undefined; + try { + const world = await getWorld(); + const { data: events } = await world.events.list({ runId }); + const projected: LogSliceEntry[] = events.map((event) => ({ + slot: idOrdinal(event.eventId), + eventType: event.eventType, + correlationId: event.correlationId, + occurredAt: isoOrUndefined(event.occurredAt), + createdAt: isoOrUndefined(event.createdAt), + })); + + // The corruption message names the last divergent event; centre on it when + // it is there, and otherwise keep the tail, where a divergence that ran out + // of recovery replays ends up. + const divergent = message?.match(/evnt_[0-9A-Z]+/)?.[0]; + const at = divergent + ? projected.findIndex((entry) => entry.slot === idOrdinal(divergent)) + : -1; + const centre = at >= 0 ? at : projected.length - 1; + logSlicesCaptured += 1; + return projected.slice( + Math.max(0, centre - LOG_SLICE_RADIUS), + centre + LOG_SLICE_RADIUS + 1 + ); + } catch { + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -431,14 +548,27 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); + const outcome = classifyFailure(failure.errorCode); + const errorMessage = hydrated?.message ?? failure.error?.message; return { ...base, - outcome: classifyFailure(failure.errorCode), + outcome, status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, + logSlice: + outcome === 'CORRUPTED_EVENT_LOG' + ? await readLogSlice(run.runId, errorMessage) + : undefined, }; } diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 910dd1f94b..5d4fc12265 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -659,7 +659,11 @@ async function getRunDiagnostics(tracked: TrackedRun): Promise { const elapsed = baseTime ? ((event.createdAt?.getTime?.() ?? 0) - baseTime) / 1000 : 0; - const prefix = ` +${elapsed.toFixed(1)}s`; + // The event's own id leads the line: on a run numbering its events + // by position it says where in the log the event sits, which is what + // a diagnosis of an out-of-order or gapped log needs and what the + // correlation id below cannot report. + const prefix = ` ${event.eventId} +${elapsed.toFixed(1)}s`; let detail = event.eventType; if ('eventData' in event) { const data = (event as any).eventData; diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index d99b67de1d..69b9c98075 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -9,12 +9,8 @@ import type { Event, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -36,7 +32,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test-abort-consistency', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -51,8 +46,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index 2ced0b4aca..4b300dd414 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -10,12 +10,8 @@ import { ReplayDivergenceError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS, EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -34,7 +30,6 @@ function setupWorkflowContext( seed: 'test-abort', fixedTimestamp: 1714857600000, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -49,8 +44,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8e7a1a9e98..b87dd7bfee 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -25,12 +25,8 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { scheduleWhenIdle, @@ -69,7 +65,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test-abort-replay-ordering', fixedTimestamp: 1714857600000, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -84,8 +79,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 4ad65f662c..618cad1655 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -1,7 +1,6 @@ import { FatalError } from '@workflow/errors'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; import { createCorrelationIdGenerator } from './correlation-id.js'; @@ -46,7 +45,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { if (hostRegistry) { (context.globalThis as any)[WORKFLOW_CLASS_REGISTRY] = hostRegistry; } - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -61,10 +59,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -100,7 +94,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -111,7 +105,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', result: resultB, @@ -176,7 +170,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step1', result: results[0], @@ -187,7 +181,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'step2', result: results[1], @@ -198,7 +192,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9X', eventData: { stepName: 'step3', result: results[2], @@ -271,7 +265,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: payloadA, @@ -282,7 +276,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: payloadB, @@ -293,7 +287,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -363,7 +357,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -374,7 +368,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', error: errorB, @@ -385,7 +379,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9X', eventData: { stepName: 'stepC', result: resultC, @@ -450,16 +444,16 @@ describe('async deserialization ordering', () => { // Correlation IDs from the deterministic ULID generator const correlationIds = [ - 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCY', - 'step_01K11TFZ62YS0YYFDQ3E8B9YCZ', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD0', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD1', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD2', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD3', - 'step_01K11TFZ62YS0YYFDQ3E8B9YD4', + 'step_01K11TFZ62CHHYKN8SS4KKNC9V', + 'step_01K11TFZ62CHHYKN8SS4KKNC9W', + 'step_01K11TFZ62CHHYKN8SS4KKNC9X', + 'step_01K11TFZ62CHHYKN8SS4KKNC9Y', + 'step_01K11TFZ62CHHYKN8SS4KKNC9Z', + 'step_01K11TFZ62CHHYKN8SS4KKNCA0', + 'step_01K11TFZ62CHHYKN8SS4KKNCA1', + 'step_01K11TFZ62CHHYKN8SS4KKNCA2', + 'step_01K11TFZ62CHHYKN8SS4KKNCA3', + 'step_01K11TFZ62CHHYKN8SS4KKNCA4', ]; const events: Event[] = results.map((result, i) => ({ @@ -524,7 +518,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -535,7 +529,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z') }, createdAt: new Date(), }, @@ -543,7 +537,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -553,7 +547,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCX', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepC', result: resultC, @@ -619,7 +613,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', }, @@ -629,7 +623,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', }, @@ -639,7 +633,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'stepA', result: resultA, @@ -650,7 +644,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'stepB', result: resultB, @@ -712,7 +706,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload: new Uint8Array([101, 110, 99, 114]), // "encr" without a key }, @@ -749,7 +743,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload }, createdAt: new Date(), }, @@ -757,7 +751,7 @@ describe('async deserialization ordering', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt }, createdAt: new Date(), }, diff --git a/packages/core/src/correlation-id-replay.test.ts b/packages/core/src/correlation-id-replay.test.ts index 92b1db81b6..f07bf10a7f 100644 --- a/packages/core/src/correlation-id-replay.test.ts +++ b/packages/core/src/correlation-id-replay.test.ts @@ -1,6 +1,5 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -19,22 +18,18 @@ import { createSleep } from './workflow/sleep.js'; * log carrying the ids a same-seeded replay derives. * * The rest of the replay suites author their event logs with literal correlation - * ids from the shared sequence and pin themselves to it. These fixtures derive - * their ids instead, so they hold under either scheme. + * ids. These fixtures derive theirs instead, so they say something about the + * minting primitives rather than about a captured snapshot of them. */ const SEED = 'test'; const FIXED_TIMESTAMP = 1753481739458; -function setupWorkflowContext( - events: Event[], - perKind: boolean -): WorkflowOrchestratorContext { +function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const context = createContext({ seed: SEED, fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); return { runId: 'wrun_test', encryptionKey: undefined, @@ -48,8 +43,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: SEED, fixedTimestamp: FIXED_TIMESTAMP, - positional: () => ulid(FIXED_TIMESTAMP), - perKind, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -67,10 +60,9 @@ function setupWorkflowContext( * we are after. */ function probeStepId( - perKind: boolean, before?: (ctx: WorkflowOrchestratorContext) => void ): string { - const ctx = setupWorkflowContext([], perKind); + const ctx = setupWorkflowContext([]); before?.(ctx); void createUseStep(ctx)('add')(1, 2).catch(() => {}); const item = [...ctx.invocationsQueue.values()].find( @@ -89,34 +81,27 @@ function createHookAndSleep(ctx: WorkflowOrchestratorContext): void { describe('correlation ids through the replay primitives', () => { it('keeps a step id when a hook and a sleep are created before it', () => { - expect(probeStepId(true, createHookAndSleep)).toBe(probeStepId(true)); - }); - - it('renumbers that step under one sequence shared by every kind', () => { - // The failure this PR removes, and the reason the assertion above is worth - // making: with a shared sequence the hook and the sleep consume the two - // ordinals the step would otherwise have drawn from. - expect(probeStepId(false, createHookAndSleep)).not.toBe(probeStepId(false)); + // Under one sequence shared by every kind, the hook and the sleep consume + // the two ordinals the step would otherwise have drawn, so a replay that + // disagreed about either renamed the step. + expect(probeStepId(createHookAndSleep)).toBe(probeStepId()); }); it('consumes a step_completed authored with the derived id', async () => { - const correlationId = probeStepId(true, createHookAndSleep); - const ctx = setupWorkflowContext( - [ - { - eventId: 'evnt_0', - runId: 'wrun_test', - eventType: 'step_completed', - correlationId, - eventData: { - stepName: 'add', - result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), - }, - createdAt: new Date(), + const correlationId = probeStepId(createHookAndSleep); + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_test', + eventType: 'step_completed', + correlationId, + eventData: { + stepName: 'add', + result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), }, - ], - true - ); + createdAt: new Date(), + }, + ]); createHookAndSleep(ctx); await expect(createUseStep(ctx)('add')(1, 2)).resolves.toBe(3); expect(ctx.onWorkflowError).not.toHaveBeenCalled(); diff --git a/packages/core/src/correlation-id.test.ts b/packages/core/src/correlation-id.test.ts index db95c65df4..07d63ffec0 100644 --- a/packages/core/src/correlation-id.test.ts +++ b/packages/core/src/correlation-id.test.ts @@ -1,31 +1,20 @@ -import { decodeTime, monotonicFactory } from 'ulid'; +import { decodeTime } from 'ulid'; import { describe, expect, it } from 'vitest'; import { CORRELATION_ID_LENGTH, type CorrelationIdKind, createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, } from './correlation-id.js'; const SEED = 'wrun_abc:myWorkflow:dpl_123'; const FIXED_TIMESTAMP = 1753481739458; function makeGenerator( - overrides: { seed?: string; fixedTimestamp?: number; perKind?: boolean } = {} + overrides: { seed?: string; fixedTimestamp?: number } = {} ) { - // A stand-in for the run's shared sequence. Seeded so the positional mode is - // reproducible across the two generators a replay-stability test builds. - let counter = 0; - const ulid = monotonicFactory(() => { - counter = (counter * 1103515245 + 12345) % 2147483648; - return counter / 2147483648; - }); - const fixedTimestamp = overrides.fixedTimestamp ?? FIXED_TIMESTAMP; return createCorrelationIdGenerator({ seed: overrides.seed ?? SEED, - fixedTimestamp, - positional: () => ulid(fixedTimestamp), - perKind: overrides.perKind ?? true, + fixedTimestamp: overrides.fixedTimestamp ?? FIXED_TIMESTAMP, }); } @@ -110,45 +99,14 @@ describe('createCorrelationIdGenerator', () => { expect(withController('hook')).toBe(withoutController('hook')); }); - it('keeps every id on fixedTimestamp in both modes', () => { - // `monotonicFactory` returns `encodeTime(lastTime)` on its increment branch, - // so a single draw that omits the seed time latches the host wall clock and - // every later id in the run carries a timestamp that differs per replay. - // Stream ids used to be drawn that way. - for (const perKind of [true, false]) { - const generate = makeGenerator({ perKind }); - for (const kind of ['stream', 'stream', 'step', 'hook'] as const) { - expect(decodeTime(generate(kind))).toBe(FIXED_TIMESTAMP); - } - } - }); - - it('ignores the kind when per-kind sources are disabled', () => { - const generate = makeGenerator({ perKind: false }); - const shared = makeGenerator({ perKind: false }); - // Positional mode is one sequence for the whole run, so drawing `wait` - // consumes the ordinal the next `step` would otherwise have had. - expect(generate('step')).toBe(shared('step')); - expect(generate('wait')).toBe(shared('step')); - }); -}); - -describe('isPerKindCorrelationIdsEnabled', () => { - it('reads WORKFLOW_PER_KIND_CORRELATION_IDS, defaulting to disabled', () => { - const original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - try { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '1'; - expect(isPerKindCorrelationIdsEnabled()).toBe(true); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - } finally { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } + it('keeps every id on fixedTimestamp', () => { + // Stream ids are drawn without a seed time of their own. Drawn from a raw + // `monotonicFactory` they would latch the host wall clock into `lastTime` + // and every later id in the run would carry a timestamp that differs per + // replay, so they go through this generator like every other kind. + const generate = makeGenerator(); + for (const kind of ['stream', 'stream', 'step', 'hook'] as const) { + expect(decodeTime(generate(kind))).toBe(FIXED_TIMESTAMP); } }); }); diff --git a/packages/core/src/correlation-id.ts b/packages/core/src/correlation-id.ts index dbed6f8f5c..b9aa8fbef3 100644 --- a/packages/core/src/correlation-id.ts +++ b/packages/core/src/correlation-id.ts @@ -169,13 +169,7 @@ function deriveBody(seed: string, kind: CorrelationIdKind): string { return body; } -/** - * Builds a replay's correlation-id generator. - * - * `perKind: false` returns the run's single shared monotonic sequence and - * ignores the kind entirely, so both schemes go through one call path and the - * flag is the only difference between them. - */ +/** Builds a replay's correlation-id generator. */ export function createCorrelationIdGenerator(options: { /** * The run's replay-stable seed. Must not vary between replays of one run, and @@ -183,15 +177,8 @@ export function createCorrelationIdGenerator(options: { */ seed: string; fixedTimestamp: number; - /** The run's shared monotonic sequence, used as-is when `perKind` is false. */ - positional: () => string; - perKind: boolean; }): CorrelationIdGenerator { - const { seed, fixedTimestamp, positional, perKind } = options; - - if (!perKind) { - return positional; - } + const { seed, fixedTimestamp } = options; const time = encodeTime(fixedTimestamp, TIME_CHARS); const bodies = new Map(); @@ -209,26 +196,3 @@ export function createCorrelationIdGenerator(options: { /** Length of a ULID, exported so tests need not restate it. */ export const CORRELATION_ID_LENGTH = TIME_CHARS + BODY_CHARS; - -/** - * Whether each entity family draws correlation ids from its own sequence rather - * than from one sequence shared by the whole run. Off unless opted in, so an SDK - * upgrade alone never moves a run between schemes. - * - * The invariant either way: a run must replay under the scheme that minted its - * ids. A replay under the other scheme mints ids its own earlier events do not - * carry, so it can consume none of them and fails the run. Two things can break - * it, and both are about turning the flag on rather than about upgrading: - * - * - Enabling it while runs are in flight. On Vercel, skew protection keeps a run - * on the deployment that started it, so a run only ever sees the value baked - * into its own deployment. Elsewhere (world-postgres, world-local, a - * self-hosted process) nothing pins a run to the code that started it, so - * enable it during a quiet window. - * - A rolling deploy that leaves both values live, which puts two schemes on one - * run concurrently — the side-by-side append this whole mechanism exists to - * avoid. Roll the value out to the whole fleet at once. - */ -export function isPerKindCorrelationIdsEnabled(): boolean { - return process.env.WORKFLOW_PER_KIND_CORRELATION_IDS === '1'; -} diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index ef6232c8a7..10a6726877 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -43,9 +43,11 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import { @@ -68,7 +70,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test', fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -90,10 +91,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -113,20 +110,24 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { } /** - * The ULIDs the seeded generator hands out, in invocation order. Correlation - * IDs in the fixtures below have to match what the replayed workflow draws. + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. IDs in the fixtures below have to match what the replayed workflow + * draws, and each kind draws from its own sequence, so a fixture indexes into + * the kind it is naming rather than into one run-wide sequence. */ -function deterministicUlids(count: number): string[] { - const context = createContext({ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); - const workflowStartedAt = context.globalThis.Date.now(); - return Array.from({ length: count }, () => ulid(workflowStartedAt)); + return Array.from({ length: count }, () => generate(kind)); } -const ULIDS = deterministicUlids(8); +const STEP_IDS = correlationIds('step', 8); +const WAIT_IDS = correlationIds('wait', 8); +const HOOK_IDS = correlationIds('hook', 8); +const ABORT_IDS = correlationIds('abort', 8); +const ABORT_HOOK_IDS = correlationIds('abortHook', 8); async function replay( ctx: WorkflowOrchestratorContext, @@ -211,32 +212,32 @@ describe('step result delivery ordering against an earlier step result', () => { ]); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'wait_created', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_1', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'wait_completed', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_4', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'wait_completed', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_4', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'stepB', }), - event('evnt_6', 'step_started', `step_${ULIDS[2]}`, { + event('evnt_6', 'step_started', `step_${STEP_IDS[1]}`, { stepName: 'stepB', }), - event('evnt_7', 'step_completed', `step_${ULIDS[2]}`, { + event('evnt_7', 'step_completed', `step_${STEP_IDS[1]}`, { stepName: 'stepB', result: stepBResult, }), - event('evnt_8', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_8', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterA', }), - event('evnt_9', 'step_created', `step_${ULIDS[4]}`, { + event('evnt_9', 'step_created', `step_${STEP_IDS[3]}`, { stepName: 'afterB', }), ]; @@ -290,22 +291,22 @@ describe('wait completion delivery ordering against an earlier step result', () ); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'wait_created', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_1', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_4', 'wait_completed', `wait_${ULIDS[1]}`, { resumeAt }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_4', 'wait_completed', `wait_${WAIT_IDS[0]}`, { resumeAt }), + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_6', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterSleep', }), ]; @@ -349,28 +350,28 @@ describe('hook payload delivery ordering against an earlier step result', () => ]); const events: Event[] = [ - event('evnt_0', 'step_created', `step_${ULIDS[0]}`, { + event('evnt_0', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_1', 'hook_created', `hook_${ULIDS[1]}`, { + event('evnt_1', 'hook_created', `hook_${HOOK_IDS[0]}`, { token: 'tok', isWebhook: false, }), - event('evnt_2', 'step_started', `step_${ULIDS[0]}`, { + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'stepA', }), - event('evnt_3', 'step_completed', `step_${ULIDS[0]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'stepA', result: stepAResult, }), - event('evnt_4', 'hook_received', `hook_${ULIDS[1]}`, { + event('evnt_4', 'hook_received', `hook_${HOOK_IDS[0]}`, { token: 'tok', payload: hookPayload, }), - event('evnt_5', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_6', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterHook', }), ]; @@ -412,12 +413,12 @@ describe('hook payload delivery ordering against an earlier step result', () => // evnt_6 hook_received (abort) ← must NOT overtake evnt_5 describe('abort delivery ordering against an earlier step result', () => { it('delivers the earlier step_completed before the abort', async () => { - // The controller draws two ULIDs on construction (stream id, then hook - // correlation id), so the sleep and stepA take the next two. - const abortHookToken = `abrt_${ULIDS[0]}`; - const abortCorrelationId = `hook_${ULIDS[1]}`; - const waitCorrelationId = `wait_${ULIDS[2]}`; - const stepACorrelationId = `step_${ULIDS[3]}`; + // The controller draws two ids on construction, each from its own kind: + // the abort token, then the hook correlation id backing it. + const abortHookToken = `abrt_${ABORT_IDS[0]}`; + const abortCorrelationId = `hook_${ABORT_HOOK_IDS[0]}`; + const waitCorrelationId = `wait_${WAIT_IDS[0]}`; + const stepACorrelationId = `step_${STEP_IDS[0]}`; const resumeAt = new Date(FIXED_TIMESTAMP + 5_000); const ops: Promise[] = []; @@ -452,10 +453,10 @@ describe('abort delivery ordering against an earlier step result', () => { token: abortHookToken, payload: abortPayload, }), - event('evnt_7', 'step_created', `step_${ULIDS[4]}`, { + event('evnt_7', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'afterStep', }), - event('evnt_8', 'step_created', `step_${ULIDS[5]}`, { + event('evnt_8', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'afterAbort', }), ]; @@ -564,8 +565,9 @@ function expectSuspensionSnapshotSteps(error: unknown, expected: string[]) { } describe('suspension timing against parked step deliveries', () => { - // Draw order: sleep -> ULIDS[0]; the three parallel steps -> 1..3; the - // follow-up step (never run, so only allocated) -> 4. + // Draw order within each kind: the sleep is the run's only wait; the three + // parallel steps are its first three steps, and the follow-up step (never + // run, so only allocated) is its fourth. const RESUME_AT = new Date(FIXED_TIMESTAMP + 25 * 60_000); it('a parallel batch with a pending sleep suspends carrying the follow-up step', async () => { @@ -577,36 +579,36 @@ describe('suspension timing against parked step deliveries', () => { ); const events: Event[] = [ - event('evnt_0', 'wait_created', `wait_${ULIDS[0]}`, { + event('evnt_0', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt: RESUME_AT, }), - event('evnt_1', 'step_created', `step_${ULIDS[1]}`, { + event('evnt_1', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', }), - event('evnt_2', 'step_created', `step_${ULIDS[2]}`, { + event('evnt_2', 'step_created', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', }), - event('evnt_3', 'step_created', `step_${ULIDS[3]}`, { + event('evnt_3', 'step_created', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', }), - event('evnt_4', 'step_started', `step_${ULIDS[1]}`, { + event('evnt_4', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', }), - event('evnt_5', 'step_started', `step_${ULIDS[2]}`, { + event('evnt_5', 'step_started', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', }), - event('evnt_6', 'step_started', `step_${ULIDS[3]}`, { + event('evnt_6', 'step_started', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', }), - event('evnt_7', 'step_completed', `step_${ULIDS[1]}`, { + event('evnt_7', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'parallelA', result: results[0], }), - event('evnt_8', 'step_completed', `step_${ULIDS[2]}`, { + event('evnt_8', 'step_completed', `step_${STEP_IDS[1]}`, { stepName: 'parallelB', result: results[1], }), - event('evnt_9', 'step_completed', `step_${ULIDS[3]}`, { + event('evnt_9', 'step_completed', `step_${STEP_IDS[2]}`, { stepName: 'parallelC', result: results[2], }), @@ -646,16 +648,16 @@ describe('suspension timing against parked step deliveries', () => { ); const events: Event[] = [ - event('evnt_0', 'wait_created', `wait_${ULIDS[0]}`, { + event('evnt_0', 'wait_created', `wait_${WAIT_IDS[0]}`, { resumeAt: RESUME_AT, }), - event('evnt_1', 'step_created', `step_${ULIDS[1]}`, { + event('evnt_1', 'step_created', `step_${STEP_IDS[0]}`, { stepName: 'only', }), - event('evnt_2', 'step_started', `step_${ULIDS[1]}`, { + event('evnt_2', 'step_started', `step_${STEP_IDS[0]}`, { stepName: 'only', }), - event('evnt_3', 'step_completed', `step_${ULIDS[1]}`, { + event('evnt_3', 'step_completed', `step_${STEP_IDS[0]}`, { stepName: 'only', result, }), diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index cc05c6f975..08fae7c3ff 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -87,6 +118,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } append(events: Event[]): void { @@ -208,32 +240,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 754ee1a138..acc05efd20 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -2,9 +2,11 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -29,12 +31,13 @@ import { createSleep } from './workflow/sleep.js'; * so suspensions wait for both async deserialization AND microtask deliveries. */ +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; // Forward onUnconsumedEvent through ctx.onWorkflowError so tests that wire @@ -62,10 +65,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -84,15 +83,22 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return ctx; } -// Deterministic correlation IDs from the ULID generator with seed 'test' -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', - '01K11TFZ62YS0YYFDQ3E8B9YCZ', - '01K11TFZ62YS0YYFDQ3E8B9YD0', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 6); +const WAIT_IDS = correlationIds('wait', 6); +const HOOK_IDS = correlationIds('hook', 6); // ─── Helpers ─────────────────────────────────────────── @@ -176,7 +182,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -187,7 +193,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -195,7 +201,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -206,7 +212,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -217,7 +223,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload3, @@ -271,7 +277,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -282,7 +288,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -290,7 +296,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -301,7 +307,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -347,7 +353,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -355,7 +361,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -363,7 +369,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload }, createdAt: new Date(), }, @@ -371,7 +377,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -379,7 +385,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -387,7 +393,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep', result: setupResult }, createdAt: new Date(), }, @@ -395,7 +401,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -403,7 +409,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -473,7 +479,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -481,7 +487,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -489,7 +495,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload }, createdAt: new Date(), }, @@ -497,7 +503,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -505,7 +511,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep' }, createdAt: new Date(), }, @@ -513,7 +519,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'setupStep', result: setupResult }, createdAt: new Date(), }, @@ -521,7 +527,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -529,7 +535,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -620,7 +626,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_8', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1 }, createdAt: new Date(), }; @@ -628,7 +634,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_10', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[2]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }; @@ -647,7 +653,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -655,7 +661,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -663,7 +669,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -671,7 +677,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'progressStep', result: progress0Result }, createdAt: new Date(), }, @@ -679,7 +685,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[2]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -687,7 +693,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload0 }, createdAt: new Date(), }, @@ -695,7 +701,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -703,7 +709,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep' }, createdAt: new Date(), }, @@ -712,7 +718,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_9', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'drainStep', result: drain0Result }, createdAt: new Date(), }, @@ -720,7 +726,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_11', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -728,7 +734,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_12', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep' }, createdAt: new Date(), }, @@ -736,7 +742,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_13', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[4]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'progressStep', result: progress1Result }, createdAt: new Date(), }, @@ -744,7 +750,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_14', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[5]}`, + correlationId: `step_${STEP_IDS[3]}`, eventData: { stepName: finalStepName }, createdAt: new Date(), }, @@ -856,7 +862,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -867,7 +873,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep' }, createdAt: new Date(), }, @@ -875,7 +881,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep', }, @@ -885,7 +891,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -896,7 +902,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -939,7 +945,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[0]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -947,7 +953,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -955,7 +961,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', }, @@ -965,7 +971,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: resultA, @@ -976,7 +982,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB' }, createdAt: new Date(), }, @@ -984,7 +990,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', }, @@ -994,7 +1000,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', result: resultB, @@ -1033,7 +1039,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[0]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1041,7 +1047,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -1049,7 +1055,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', }, @@ -1059,7 +1065,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: resultA, @@ -1107,7 +1113,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep' }, createdAt: new Date(), }, @@ -1115,7 +1121,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'incompleteStep', }, @@ -1125,7 +1131,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB' }, createdAt: new Date(), }, @@ -1133,7 +1139,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', }, @@ -1143,7 +1149,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'stepB', result: resultB, @@ -1154,7 +1160,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC' }, createdAt: new Date(), }, @@ -1162,7 +1168,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC', }, @@ -1172,7 +1178,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'stepC', result: resultC, @@ -1241,7 +1247,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1252,7 +1258,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1261,7 +1267,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -1272,7 +1278,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', input: payload1 }, createdAt: new Date(), }, @@ -1280,7 +1286,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', }, @@ -1290,7 +1296,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'processPayload', result: stepResult1, @@ -1302,7 +1308,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -1313,7 +1319,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_7', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', input: payload2 }, createdAt: new Date(), }, @@ -1321,7 +1327,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_8', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', }, @@ -1331,7 +1337,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_9', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'processPayload', result: stepResult2, @@ -1382,7 +1388,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1393,7 +1399,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -1404,7 +1410,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -1478,7 +1484,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false, @@ -1489,7 +1495,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1497,7 +1503,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payloadA, @@ -1508,7 +1514,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: new Date('2099-01-01') }, createdAt: new Date(), }, @@ -1516,7 +1522,7 @@ function defineTests(mode: 'sync' | 'async') { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: payloadB, diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..c558208a77 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_01ABCDEFGHJKMNPQRSTVWXYZ.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_01ABCDEFGHJKMNPQRSTVWXYZ. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 9c04c2fc3d..fff8e56fd5 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -563,12 +563,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 4f6a4e81a7..093ea433be 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -20,6 +23,7 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from './serialization.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; // Capture every promise handed to `waitUntil` so tests can assert that // progress-critical sends are never registered on a detached, unconsumed @@ -1532,6 +1536,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1544,12 +1559,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1565,8 +1583,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1585,6 +1605,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1630,7 +1651,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1638,7 +1659,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1724,6 +1745,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ @@ -1770,6 +1818,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index c5bf715128..2b40c4e69a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -5,7 +5,6 @@ import { FatalError, HookNotFoundError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -19,13 +18,17 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, + maxSlotOf, ROOT_RUN_ID_ATTRIBUTE, type RunInput, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, + usesSlotIdentity, type WorkflowInvokePayload, WorkflowInvokePayloadSchema, type WorkflowRun, @@ -53,23 +56,26 @@ import { isInlineOwnershipEnabled, isTurboEnabled, isVmRetentionEnabled, + preconditionRestartBackoffMs, } from './runtime/constants.js'; import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, + claimFenceFor, type EventCreator, + eventCreateFenceFor, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, insertEventByEventId, isPreconditionGuardEnabled, - type LoadedEventLog, + isStaleWriteRejection, loadWorkflowRunEvents, memoizeEncryptionKey, parseHealthCheckPayload, preconditionEventDelta, - preconditionSnapshotParams, queueMessage, + toMutableEventLog, withHealthCheck, } from './runtime/helpers.js'; import { @@ -763,6 +769,14 @@ export function workflowEntrypoint( let cachedEvents: Event[] | null = null; let eventsCursor: string | null = null; + // Set when a restarted replay is recovering by merging into + // its cached log rather than reloading it whole, from either + // an inline delta or one page above the cursor. The next load + // verifies the result is dense before the replay trusts it, + // and falls back to the authoritative load if it is not; see + // the consume site below the load. + let slotDensityCheckPending = false; + // Inline-delta optimization: when an inline step's terminal // write returns the event-log delta since the pre-write // cursor (a supporting World only), we stash it here so the @@ -787,6 +801,18 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published, for the writes whose + // snapshot cannot show it: turbo backgrounds `run_started` and + // replays against an empty log, so a claim numbered from that + // log alone would propose a slot `run_started` already holds. + // 0 for a run whose ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -950,6 +976,14 @@ export function workflowEntrypoint( // Precondition (412) recovery: how many times this invocation // has thrown away its replay and started over in-process. let preconditionRestarts = 0; + /** + * Wait the next loop iteration owes before re-deriving, set + * by `restartReplayInProcess`. Held here rather than awaited + * at the restart's call sites because the restart is decided + * from three of them, in synchronous code, while the loop head + * is the single point every restart passes through. + */ + let pendingRestartBackoffMs = 0; /** * Event ids the discarded replay held, kept until the next * load resolves so a restart can report what its reload @@ -965,7 +999,7 @@ export function workflowEntrypoint( ids: Set; restart: number; reason: string; - source: 'inline-delta' | 'full-reload'; + source: 'inline-delta' | 'full-reload' | 'slot-top-up'; } | null = null; /** * Report what a stale-snapshot restart's reload found, once @@ -1015,13 +1049,15 @@ export function workflowEntrypoint( * per-invocation budget is spent (the caller then falls back * to a fresh invocation). * - * A 412 means the log this replay derived its events from was - * missing an event the backend had already recorded. The - * rejected write cannot simply be retried: correlation ids - * are positional ordinals of one seeded sequence, so a replay - * over the corrected log mints a different id for the same - * logical event, and re-posting this one would persist an - * event no correct replay ever produces. The whole replay has + * A rejection — 412 for the event-log watermark, 409 for a + * lost slot claim — means the log this replay derived its + * events from was missing an event the backend had already + * recorded. The rejected write cannot simply be retried: + * correlation ids are positional ordinals of one seeded + * sequence, and a slot event id is a position in the log, so + * a replay over the corrected log mints different ids for the + * same logical event, and re-posting this one would persist + * an event no correct replay ever produces. The whole replay has * to be re-derived — which the loop does by discarding its * cached log, since `runWorkflow` then builds a fresh VM, * seed and correlation-id sequence from the reloaded events. @@ -1042,7 +1078,9 @@ export function workflowEntrypoint( ): boolean => { if ( preconditionRestarts >= - getPreconditionMaxInProcessRestarts() + getPreconditionMaxInProcessRestarts( + usesSlotIdentity(workflowRun?.specVersion) + ) ) { return false; } @@ -1050,25 +1088,48 @@ export function workflowEntrypoint( // Every stale-snapshot restart invalidates the parked VM: // the log it consumed was missing events, so only a fresh // replay over the corrected log is authoritative. This - // also covers the suspension-create 412, which fires + // also covers the suspension-create rejection, which fires // before the retention decision (kill switch + step-input - // gate) ever ran, and the run_completed 412, where a + // gate) ever ran, and the run_completed rejection, where a // completed session must not be resumed again. retainedSession = null; - // A World MAY return the events we were missing on the 412. - // Trust it only on the FIRST restart: its completeness proof - // leans on the backend's own bookkeeping, so if that - // under-counts, a "complete" delta can still leave a hole. - // Bounding it to one attempt caps that at a single wasted - // restart; every later restart does the authoritative load. + // A World MAY return the events we were missing on the + // rejection. A merged log is checked for density before the + // replay trusts it, so a delta that left a hole costs the + // reload it would have done anyway and every restart can + // take one. A run fenced by the watermark has no such check + // — its delta's completeness rests on the backend's own + // bookkeeping — so it trusts one delta and loads + // authoritatively after that. + const slotNumbered = usesSlotIdentity( + workflowRun?.specVersion + ); const delta = - allowDelta && preconditionRestarts === 1 + allowDelta && (slotNumbered || preconditionRestarts === 1) ? preconditionEventDelta(error, runId) : null; // A delta is only usable as a delta if there is a cached log // to merge it into; with no base log the restart has to load // the whole thing anyway. const usedDelta = Boolean(delta && cachedEvents); + // Without a delta, the log heals from its cursor rather + // than from a full reload. Slot ids sort in write order, so + // every event this replay was missing is strictly above the + // cursor and one incremental page brings it in, and density + // (a dense log from slot 1 holds exactly `maxSlot` events) + // proves afterwards that it did, so nothing is being trusted + // here that is not checked. Neither property holds for a + // ULID-numbered log, so those restarts reload whole. + const topsUpFromCursor = + !usedDelta && + slotNumbered && + cachedEvents !== null && + eventsCursor !== null; + const restartSource = usedDelta + ? 'inline-delta' + : topsUpFromCursor + ? 'slot-top-up' + : 'full-reload'; // Snapshot the set being discarded while it is still in // hand; the comparison happens once the next load resolves. preconditionRestartBaseline = cachedEvents @@ -1078,7 +1139,7 @@ export function workflowEntrypoint( ), restart: preconditionRestarts, reason, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } : null; if (usedDelta) { @@ -1086,6 +1147,19 @@ export function workflowEntrypoint( // (`pendingInlineDelta && cachedEvents`) with no // events.list round trip at all. pendingInlineDelta = delta; + slotDensityCheckPending = slotNumbered; + } else if (topsUpFromCursor) { + // Keep the cached log and its cursor: the loop's + // incremental branch fetches the page above the cursor + // and appends it, and the density check below the load + // sends the restart to a full reload if that page did not + // close the gap. Appends land above everything already + // scanned for payload prewarming, so no rescan is needed + // unless that fallback fires. + slotDensityCheckPending = true; + preloadedEvents = undefined; + preloadedEventsCursor = undefined; + pendingInlineDelta = null; } else { // MUST be a full, cursor-less reload. The cursor filters // by lexicographic event id while a hole is defined by @@ -1100,6 +1174,7 @@ export function workflowEntrypoint( preloadedEvents = undefined; preloadedEventsCursor = undefined; pendingInlineDelta = null; + slotDensityCheckPending = false; // The corrected log inserts the missing events BELOW the // length already scanned for payload prewarming, shifting // every later position. Only a full rescan sees them. @@ -1112,12 +1187,14 @@ export function workflowEntrypoint( reason, loopIteration, preconditionRestarts, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } ); span?.setAttributes({ 'workflow.precondition_restarts': preconditionRestarts, }); + pendingRestartBackoffMs = + preconditionRestartBackoffMs(preconditionRestarts); return true; }; @@ -1158,7 +1235,7 @@ export function workflowEntrypoint( return { reinvoked: false, error: new WorkflowRuntimeError( - `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts()} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, + `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts(usesSlotIdentity(workflowRun?.specVersion))} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, { cause: error } ), }; @@ -1495,6 +1572,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = createEvent( runStartedEvent, // We background this purely as a write barrier and @@ -1504,7 +1591,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1517,6 +1604,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1533,7 +1624,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; - const now = new Date(); + // The run's first two positions are its own: + // `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } workflowRun = { runId, status: 'running', @@ -1582,6 +1686,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1841,6 +1946,19 @@ export function workflowEntrypoint( while (true) { loopIteration++; + // A restart lost a slot to a concurrent replay of this same + // run. Pause before re-deriving so the winner gets a clear + // window to extend the log: re-deriving immediately puts + // every loser back in contention at once, and none of them + // pulls ahead. + if (pendingRestartBackoffMs > 0) { + const backoffMs = pendingRestartBackoffMs; + pendingRestartBackoffMs = 0; + await new Promise((resolve) => + setTimeout(resolve, backoffMs) + ); + } + // Replay-budget check: bail out (retry or fail) if // non-step time within this invocation has exceeded // the configured budget. Step bodies are excluded @@ -2047,6 +2165,26 @@ export function workflowEntrypoint( // the wait pass, which may swap in a freshly loaded array. cachedEvents = events; + if (slotDensityCheckPending) { + slotDensityCheckPending = false; + // The log is dense from slot 1, so a complete one + // holds exactly `maxSlot` events. A short + // count means the merge did not bring in everything the + // restart was missing — the only other reading, a + // permanent hole from a write that took a slot and then + // failed, is equally unrecoverable from here — so fall + // back to the authoritative load. + if (maxSlotOf(events) !== events.length) { + const loaded = await loadWorkflowRunEvents(runId); + events = loaded.events; + eventsCursor = loaded.cursor; + cachedEvents = events; + // The reload can insert events below the prefix + // already scanned for payload prewarming. + replayPayloadCache.resetScan(); + } + } + reportPreconditionRestartReload(events); // Detect concurrent completion via the event log: if @@ -2088,12 +2226,24 @@ export function workflowEntrypoint( }, })); + // One log for the whole loop: `events` is appended to in + // place by the guard's reloads, so a per-iteration + // rescan for the slot high-water mark would be wasted + // work on an array that never changes identity here. + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + const waitClaim = claimFenceFor( + waitLog, + workflowRun.specVersion + ); for (const waitEvent of waitsToComplete) { try { - await createEvent(waitEvent, { - requestId, - ...preconditionSnapshotParams(events, eventsCursor), - }); + await waitClaim((fence) => + createEvent(waitEvent, { requestId, ...fence }) + ); } catch (err) { if (EntityConflictError.is(err)) { runtimeLogger.info( @@ -2202,6 +2352,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + runtimeLogger.debug('Starting workflow execution', { workflowRunId: runId, loopIteration, @@ -2219,7 +2383,11 @@ export function workflowEntrypoint( events ); let workflowResult: WorkflowResumeResult = retainedSession - ? await resumeWorkflow(retainedSession, events) + ? await resumeWorkflow( + retainedSession, + events, + replayWriteLog + ) : { type: 'replay' }; if (workflowResult.type === 'replay') { @@ -2235,6 +2403,7 @@ export function workflowEntrypoint( // events before the terminal `awaitRunReady()` below. runReadyBarrier, worldCapabilities: world.capabilities, + eventLog: replayWriteLog, }); } await payloadPrewarm; @@ -2258,10 +2427,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below restarts the replay in-process. + // computed by this replay, so a rejection proving the view + // was incomplete — 412 stale watermark, or 409 taken slot + // — must force a *fresh replay* (which may observe the new + // event and produce a different result), not re-commit the + // stale result. The catch below restarts the replay + // in-process. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -2275,7 +2446,10 @@ export function workflowEntrypoint( }, { requestId, - ...preconditionSnapshotParams(events, eventsCursor), + ...eventCreateFenceFor( + replayWriteLog, + workflowRun.specVersion + ), } ); } catch (err) { @@ -2347,12 +2521,13 @@ export function workflowEntrypoint( } // V2: handle suspension without queuing steps. - // Each event creation inside handleSuspension carries the - // precondition snapshot of the loaded event log, so a - // backend holding an event this replay never saw rejects - // the write (412) instead of accepting a divergent one. - // The rejection is handled here, by restarting the - // replay — never by re-posting the same event. + // Each event creation inside handleSuspension carries + // the run's concurrency fence — its own event slot, or + // the loaded snapshot's watermark — so a backend holding + // an event this replay never saw rejects the write + // (409/412) instead of accepting a divergent one. The + // rejection is handled here, by restarting the replay — + // never by re-posting the same event. const suspensionStart = Date.now(); // The snapshot refresh above always sets cachedEvents // before the replay can suspend. Re-narrow it for this @@ -2365,10 +2540,11 @@ export function workflowEntrypoint( 'Invariant violation: workflow suspended before its event log was loaded' ); } - const suspensionLog: LoadedEventLog = { - events: cachedEvents, - cursor: eventsCursor, - }; + const suspensionLog = toMutableEventLog( + cachedEvents, + eventsCursor, + knownSlotFloor + ); let suspensionResult: Awaited< ReturnType >; @@ -2384,14 +2560,15 @@ export function workflowEntrypoint( replayRecoveryReporter, }); } catch (suspensionError) { - // A suspension create was rejected as stale: re-derive - // the replay from a corrected log in this invocation. - // Once the in-process budget is spent, fall back to an + // A suspension create was rejected as stale (412 + // watermark, or 409 taken slot): re-derive the replay + // from a corrected log in this invocation. Once the + // in-process budget is spent, fall back to an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (isStaleWriteRejection(suspensionError)) { if ( restartReplayInProcess( 'suspension-create', @@ -2882,11 +3059,15 @@ export function workflowEntrypoint( // rejected with 412 — its guarded suspension creates // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started - // claim of its next inline step, which carries the - // snapshot too (threaded below via - // `stateUpdatedAt`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // claim of its next inline step, which is fenced too + // (threaded below via `claimFenceFor`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot claim gets there differently: it merges the + // missed events and retries in place, so the same + // events are observed without discarding the batch. + // See claimFenceFor. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -3026,21 +3207,23 @@ export function workflowEntrypoint( turbo, }); - // Precondition-guard snapshot for the inline - // step_started claims: the lazy claim is the first - // durable write of a hot-path step (its step_created - // is deferred), so without a snapshot it would bypass - // the guard entirely and a stale replay could claim — - // and commit — a step scheduled off a view that misses - // an event it never loaded. - // `preconditionSnapshotParams` returns an empty object - // when the guard env flag is off, so this is a no-op - // outside guarded deployments; Worlds that don't - // enforce the guard ignore it. - const inlineClaimSnapshot = preconditionSnapshotParams( - cachedEvents, - preInlineWriteCursor - ); + // Concurrency fence for the inline step_started claims: + // the lazy claim is the first durable write of a + // hot-path step (its step_created is deferred), so + // without one it would be unguarded and a stale replay + // could claim — and commit — a step scheduled off a view + // that misses an out-of-band event. One log for the + // whole batch so each claim draws its own event slot; + // for a run fenced by the watermark instead, every claim + // in the batch carries the same snapshot. + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -3048,6 +3231,26 @@ export function workflowEntrypoint( >[]; const stepExecutionPromises = inlineExecutions.map( (s, stepIndex) => { + // Drawn here — synchronously, in replay order — + // rather than inside `run`: a slot claim is + // positional, so it has to be assigned before these + // executions start racing each other, and the order + // it is assigned in has to be replay-stable. + const claimFence = claimFenceFor( + inlineClaimLog, + workflowRun.specVersion, + { + // A lazy start publishes two events: the World + // writes the step's deferred `step_created` + // alongside the claim, so the batch has to + // reserve a slot for that one too — otherwise + // it lands on the slot the next start in the + // batch is holding and costs that start its + // claim. + extraEvents: + s.lazyStepInput !== undefined ? 1 : 0, + } + ); const run = () => executeStep({ world, @@ -3119,7 +3322,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - preconditionSnapshot: inlineClaimSnapshot, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -3161,23 +3364,26 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is missing an event the backend already has, - // so the claim was fenced by the guard and no step - // events were written. Abandon the batch — any - // optimistic body result is discarded by executeStep's - // reconciliation — and restart the replay so it - // observes the missing event. Wait for the sibling - // executions to settle first so no owned body is in - // flight when the restart (or the ack path) runs. - if (PreconditionFailedError.is(stepErr)) { + // An incomplete-view rejection of an inline + // step_started claim — 412 for the event-log + // watermark, 409 for a lost slot claim: the loaded + // view this batch was scheduled from is missing an + // event the backend already has, so the claim was + // fenced and no step events were written. Abandon the + // batch — any optimistic body result is discarded by + // executeStep's reconciliation — and restart the + // replay so it observes the missing event. Wait for + // the sibling executions to settle first so no owned + // body is in flight when the restart (or the ack + // path) runs. + if (isStaleWriteRejection(stepErr)) { const settled = await Promise.allSettled( stepExecutionPromises ); // A sibling whose claim was accepted wrote step // events of its own, possibly after the World built - // this 412's delta — so that delta can no longer be + // the rejection's delta — so that delta can no + // longer be // assumed to complete the log, and the restart has // to reload it in full. `skipped` (the step already // existed), `gone` and `throttled` (claim rejected) @@ -3315,7 +3521,7 @@ export function workflowEntrypoint( // correlationId so it dedupes against the // keyed re-dispatch the suspension handler // performs on replay (it also uses - // `idempotencyKey: step.correlationId`). + // the same run-scoped correlationId key). // // Without this, a mixed batch where one step // `completed` with unflushed background ops @@ -3392,24 +3598,25 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a guarded write made - // directly by the replay loop — the result-bearing - // `run_completed`, or the `wait_completed` of the wait - // pass. Both reach this one catch and the rejection - // does not say which, hence the neutral label. - // Neither may be re-posted in place: the correlation id - // and (for run_completed) the result itself came from - // this replay, and a corrected log may produce - // different ones. Don't fail the run — restart the - // replay in this invocation, and only once that budget - // is spent schedule an explicit re-invocation. + // Incomplete-view rejection of a guarded write made + // directly by the replay loop — a stale watermark (412) + // or a taken slot (409), on the result-bearing + // `run_completed` or on the `wait_completed` of the + // wait pass. Both reach this one catch and the + // rejection does not say which, hence the neutral + // label. Neither may be re-posted in place: the + // correlation id and (for run_completed) the result + // itself came from this replay, and a corrected log may + // produce different ones. Don't fail the run — restart + // the replay in this invocation, and only once that + // budget is spent schedule an explicit re-invocation. // Rethrowing instead would rely on redelivery of the // CURRENT message, which the turbo path has already // acked — empirically the run then stalls for the // queue's ~300s default visibility timeout before // completing. let terminalError = err; - if (PreconditionFailedError.is(err)) { + if (isStaleWriteRejection(err)) { if (restartReplayInProcess('replay-write', err)) { continue; } @@ -3552,10 +3759,10 @@ export function workflowEntrypoint( // type identity and custom properties round-trip // through the event log. // - // Precondition-guard asymmetry: unlike `run_completed`, - // this terminal `run_failed` sends no `stateUpdatedAt` - // snapshot, so it is never 412-rejected even if a hook - // landed mid-replay and could have changed the path that + // Fencing asymmetry: unlike `run_completed`, this + // terminal `run_failed` carries no concurrency fence, so + // it is never rejected even if a hook landed mid-replay + // and could have changed the path that // threw. This is intentional and fail-open: a spurious // failure is recoverable (the run can be re-run from the // dashboard), whereas a spurious *completion* commits a diff --git a/packages/core/src/runtime/constants.test.ts b/packages/core/src/runtime/constants.test.ts index 208250e8e6..919c4edd14 100644 --- a/packages/core/src/runtime/constants.test.ts +++ b/packages/core/src/runtime/constants.test.ts @@ -18,6 +18,9 @@ import { MAX_REPLAY_TIMEOUT_MS, MIN_MAX_INLINE_STEPS, MIN_REPLAY_TIMEOUT_MS, + PRECONDITION_RESTART_BACKOFF_BASE_MS, + PRECONDITION_RESTART_BACKOFF_MAX_MS, + preconditionRestartBackoffMs, REPLAY_TIMEOUT_MS, } from './constants.js'; @@ -393,3 +396,60 @@ describe('getInlineOwnershipLeaseSeconds', () => { expect(getInlineOwnershipLeaseSeconds()).toBe(1); }); }); + +describe('preconditionRestartBackoffMs', () => { + const BASE_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS'; + const MAX_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS'; + + beforeEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + afterEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + it('doubles the window on each successive restart', () => { + // Draw the top of the window so the window itself is observable. + const top = () => 0.999_999; + expect(preconditionRestartBackoffMs(1, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS - 1 + ); + expect(preconditionRestartBackoffMs(2, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 2 - 1 + ); + expect(preconditionRestartBackoffMs(3, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 4 - 1 + ); + }); + + it('draws over the whole window, not a fixed delay plus noise', () => { + // Full jitter is the property that decorrelates concurrent replays; a + // floor would keep them in lockstep however long the wait. + expect(preconditionRestartBackoffMs(5, () => 0)).toBe(0); + expect(preconditionRestartBackoffMs(5, () => 0.5)).toBeLessThan( + preconditionRestartBackoffMs(5, () => 0.999_999) + ); + }); + + it('caps the window', () => { + expect(preconditionRestartBackoffMs(40, () => 0.999_999)).toBe( + PRECONDITION_RESTART_BACKOFF_MAX_MS - 1 + ); + }); + + it('is disabled by a zero base', () => { + process.env[BASE_ENV] = '0'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(0); + expect(preconditionRestartBackoffMs(9, () => 0.999_999)).toBe(0); + }); + + it('honours overrides of both the base and the cap', () => { + process.env[BASE_ENV] = '100'; + process.env[MAX_ENV] = '150'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(99); + expect(preconditionRestartBackoffMs(2, () => 0.999_999)).toBe(149); + }); +}); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index dadd81ca5d..a0d09de4c6 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -414,14 +414,32 @@ export function getReplayDivergenceMaxRetries(): number { // run-level budget below then applies. export const PRECONDITION_MAX_INPROCESS_RESTARTS = 3; +// A run that numbers its events by slot recovers by topping its log up from +// its cursor, so a restart costs one incremental page instead of a full +// reload. The tight bound above is priced for the reload; spending it here +// buys a re-invocation — a queue hop plus `PRECONDITION_REINVOKE_DELAY_SECONDS` +// — in place of restarts that are orders of magnitude cheaper than the thing +// the bound was protecting against. Measured on the step-storm repro against +// world-postgres at 6-way concurrency, raising this to 12 took the six runs +// from 196–241s (two of them exceeding the harness timeout) to 119–148s with +// none timing out. +export const PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL = 12; + /** * Effective in-process replay-restart budget for stale-snapshot rejections. * Override via `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`. + * + * @param incremental Whether a restart heals from the log's cursor rather than + * reloading it whole, which holds for runs on slot identity. */ -export function getPreconditionMaxInProcessRestarts(): number { +export function getPreconditionMaxInProcessRestarts( + incremental = false +): number { return envNumber( 'WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS', - PRECONDITION_MAX_INPROCESS_RESTARTS, + incremental + ? PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL + : PRECONDITION_MAX_INPROCESS_RESTARTS, { integer: true } ); } @@ -464,3 +482,66 @@ export function getPreconditionReinvokeDelaySeconds(): number { { integer: true } ); } + +// Concurrent replays of one run contend for the same event slots, and a +// rejected write costs its loser a restart. With no pause the losers re-derive +// at full speed and collide again immediately, so a run under heavy fan-out can +// spend its whole invocation budget with no writer ever pulling far enough +// ahead to finish. A short randomized wait between restarts breaks the lockstep +// by spreading the retries. +export const PRECONDITION_RESTART_BACKOFF_BASE_MS = 15; + +// Ceiling on that wait. The restart is cheap (one incremental page plus a +// re-derive), so the backoff must stay well under the cost of the +// re-invocation it is competing with. +export const PRECONDITION_RESTART_BACKOFF_MAX_MS = 400; + +/** + * Effective base for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS`; `0` disables the wait entirely. + */ +export function getPreconditionRestartBackoffBaseMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS', + PRECONDITION_RESTART_BACKOFF_BASE_MS, + { integer: true } + ); +} + +/** + * Effective ceiling for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`. + */ +export function getPreconditionRestartBackoffMaxMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS', + PRECONDITION_RESTART_BACKOFF_MAX_MS, + { integer: true } + ); +} + +/** + * Full-jitter backoff for the `restarts`-th in-process replay restart, in + * milliseconds. + * + * Full jitter (a uniform draw over the whole window, not a fixed delay plus + * noise) is what actually decorrelates the racers: equal-length waits would + * keep colliding replays in lockstep no matter how long they were. + * + * @param restarts 1-based count of restarts spent so far, including this one. + * @param random Injectable uniform source; defaults to `Math.random`. The delay + * never reaches the event log, so it does not affect replay determinism. + */ +export function preconditionRestartBackoffMs( + restarts: number, + random: () => number = Math.random +): number { + const base = getPreconditionRestartBackoffBaseMs(); + if (base <= 0) return 0; + const exponent = Math.max(0, restarts - 1); + const window = Math.min( + getPreconditionRestartBackoffMaxMs(), + base * 2 ** exponent + ); + return Math.floor(random() * window); +} diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index d71fdfd6a3..5d16a9b9f8 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,5 +1,15 @@ -import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; -import type { Event, World } from '@workflow/world'; +import { + EntityConflictError, + PreconditionFailedError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; +import { + type Event, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + type World, +} from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; @@ -12,15 +22,21 @@ import { } from '../serialization.js'; import { appendUniqueEvents, + claimFenceFor, + eventCreateFenceFor, getWorkflowQueueName, handleHealthCheckMessage, healthCheck, insertEventByEventId, + isStaleWriteRejection, latestEventStateUpdatedAt, loadWorkflowRunEvents, memoizeEncryptionKey, + mergeLoadedEvents, preconditionEventDelta, preconditionSnapshotParams, + reserveSlot, + toMutableEventLog, } from './helpers.js'; // Mock the logger to suppress output during tests @@ -626,6 +642,332 @@ describe('latestEventStateUpdatedAt', () => { }); }); +describe('slot bookkeeping', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + it('reads maxSlot from the highest slot present, not the last element', () => { + // Nothing forces a caller's array into slot order — a World is free to + // hand back a page in whatever order its index produced — so the highest + // slot is a scan, not a peek at the last element. + const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); + expect(log.maxSlot).toBe(3); + expect(log.nextSlot).toBe(4); + }); + + it('reports maxSlot 0 for an empty or ULID-numbered log', () => { + expect(toMutableEventLog([], null).maxSlot).toBe(0); + expect( + toMutableEventLog([makeUlidEvent(1_700_000_000_000)], null).maxSlot + ).toBe(0); + }); + + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + + it('never lowers maxSlot when an older delta is merged in', () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); + mergeLoadedEvents(log, [slotEvent(2)]); + expect(log.maxSlot).toBe(3); + expect(log.events).toHaveLength(3); + }); + + it('raises the reservation pointer past a newer delta', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + reserveSlot(log); + reserveSlot(log); + expect(log.nextSlot).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); + + expect(log.maxSlot).toBe(5); + expect(reserveSlot(log)).toBe(6); + }); + + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + + it('deduplicates merged events by id', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + ]); + }); + + it('restores slot order when a merge brings in a lower slot', () => { + // Arrival order is not log order: a slot is reserved when its event is + // issued and written when the issue resolves, so a lower slot can be + // learned after a higher one. The replay consumes this array positionally, + // so an event left sitting ahead of the one it followed decides races the + // wrong way. + const log = toMutableEventLog([slotEvent(1), slotEvent(4)], 'c0'); + mergeLoadedEvents(log, [slotEvent(3), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + slotEventId(3), + slotEventId(4), + ]); + }); + + it('leaves a ULID log in the order the World returned it', () => { + // ULID ids are minted at write time, so arrival order *is* log order and + // the World's ordering is the authority. + const events = [makeUlidEvent(1_700_000_000_000)]; + const later = makeUlidEvent(1_700_000_001_000); + const earlier = makeUlidEvent(1_699_999_999_000); + const log = toMutableEventLog(events, 'c0'); + mergeLoadedEvents(log, [later, earlier]); + expect(log.events).toEqual([events[0], later, earlier]); + }); + + it('hands out contiguous distinct slots for a synchronous burst', () => { + // The suspension flush issues every operation synchronously and awaits + // them together; without contiguous reservation they would all propose the + // same slot and all but one would conflict. + const log = toMutableEventLog([slotEvent(4)], 'c0'); + const burst = Array.from({ length: 20 }, () => reserveSlot(log)); + expect(burst).toEqual(Array.from({ length: 20 }, (_, i) => 5 + i)); + expect(new Set(burst).size).toBe(burst.length); + // Reservations sit past maxSlot rather than moving it: only merged events + // prove a slot is taken. + expect(log.maxSlot).toBe(4); + }); + + it('proposes a padded event id only for a slot-identity run', () => { + const log = toMutableEventLog([], null); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(1), + maxSlot: 0, + }); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(2), + maxSlot: 0, + }); + }); + + it('reserves a slot per extra event and names the top one', () => { + // A lazy inline `step_started` publishes two events: the World also writes + // the `step_created` it deferred, which takes the slot below the claim. + // Reserving it here is what keeps it off the slot the next write of the + // same batch will claim. + const log = toMutableEventLog([slotEvent(1)], null); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(3), maxSlot: 1 }); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(5), maxSlot: 1 }); + // A single-event write in the same batch still gets the next free slot. + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(6), + maxSlot: 1, + }); + }); + + it('burns no slot on an extra event of a ULID-numbered run', () => { + const log = toMutableEventLog([], null); + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { + extraEvents: 1, + }); + expect(log.nextSlot).toBe(1); + }); + + it('proposes no event id for a ULID-numbered run', () => { + // A run whose ids the backend mints must not burn slots either. + const log = toMutableEventLog([], null); + const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + expect(fence?.eventId).toBeUndefined(); + expect(log.nextSlot).toBe(1); + }); +}); + +describe('claimFenceFor', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('claims the next free slot and passes the observed maxSlot alongside it', async () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(2)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async () => 'ok'); + + await expect(claim(op)).resolves.toBe('ok'); + expect(op).toHaveBeenCalledWith({ eventId: slotEventId(3), maxSlot: 2 }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('takes each claim only once the create ahead of it has committed', async () => { + // A claim fences out a concurrent writer only while it names the slot right + // after the tail the writer saw, so a second create cannot be numbered + // until the first has landed. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const claimed: (string | undefined)[] = []; + let releaseFirst!: () => void; + const firstLanded = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const first = claim(async (fence) => { + claimed.push(fence?.eventId); + await firstLanded; + return 'first'; + }); + const second = claim(async (fence) => { + claimed.push(fence?.eventId); + return 'second'; + }); + + await Promise.resolve(); + expect(claimed).toEqual([slotEventId(2)]); + + releaseFirst(); + await expect(first).resolves.toBe('first'); + await expect(second).resolves.toBe('second'); + expect(claimed).toEqual([slotEventId(2), slotEventId(3)]); + }); + + it('rejects a lost claim rather than re-addressing the write', async () => { + // A 409 says this replay decided from a log missing an event. Moving the + // same write to a free slot would commit that decision anyway, so the + // rejection propagates and the run replays. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async (fence?: { eventId?: string }) => { + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(SlotConflictError); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('fails the rest of the batch without a round-trip', async () => { + // The siblings behind a rejected claim were decided from the same log, so + // they carry the fence it just proved wrong. They rethrow that rejection + // instead of spending a create each to be told the same thing. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const rejection = new SlotConflictError('taken', { + eventId: slotEventId(2), + }); + const loser = claim(async () => { + throw rejection; + }); + await expect(loser).rejects.toBe(rejection); + + const sibling = vi.fn( + async (fence?: { eventId?: string }) => fence?.eventId + ); + await expect(claim(sibling)).rejects.toBe(rejection); + expect(sibling).not.toHaveBeenCalled(); + }); + + it('lets a write that failed without taking its slot claim it again', async () => { + // An entity conflict means the event never landed, so the slot is still + // free. Only a stale-write rejection stops the log. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + await expect( + claim(async () => { + throw new EntityConflictError('wait already completed'); + }) + ).rejects.toBeInstanceOf(EntityConflictError); + + await expect(claim(async (fence) => fence?.eventId)).resolves.toBe( + slotEventId(2) + ); + }); + + it('numbers a batch in the order its claims fire, each one above the last', async () => { + // A step that publishes the `step_created` it deferred takes the two slots + // below its claim, and the next claim starts above both. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const withCreate = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const plain = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + await withCreate(record); + await plain(record); + + expect(claimed).toEqual([slotEventId(3), slotEventId(4)]); + }); + + it('rethrows a non-conflict error immediately, without merging', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unserialized', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = claimFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith( + expect.objectContaining({ stateUpdatedAt: time }) + ); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + describe('preconditionSnapshotParams', () => { let originalGuard: string | undefined; @@ -833,6 +1175,62 @@ describe('preconditionEventDelta', () => { expect(delta({ events: [null] })).toBe(null); expect(delta('not-an-object')).toBe(null); }); + + it('reads the delta a lost slot claim carries as typed fields', () => { + const event = makeUlidEvent(1_700_000_000_000); + + expect( + preconditionEventDelta( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [event], + cursor: 'eid:next', + hasMore: false, + }), + RUN_ID + ) + ).toEqual({ events: [event], cursor: 'eid:next' }); + }); + + it('refuses a truncated slot-conflict delta', () => { + // A partial delta would restart the replay on a log that is still missing + // events; only a full reload can prove it saw all of them. + expect( + preconditionEventDelta( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [makeUlidEvent(1_700_000_000_000)], + cursor: 'eid:next', + hasMore: true, + }), + RUN_ID + ) + ).toBe(null); + }); +}); + +describe('isStaleWriteRejection', () => { + it('accepts both rejections that prove the replay read an incomplete log', () => { + expect(isStaleWriteRejection(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + isStaleWriteRejection( + new SlotConflictError('taken', { + eventId: slotEventId(4), + events: [], + cursor: null, + hasMore: false, + }) + ) + ).toBe(true); + }); + + it('rejects every other failure, which is not recovered by a restart', () => { + expect(isStaleWriteRejection(new WorkflowWorldError('boom'))).toBe(false); + expect(isStaleWriteRejection(new Error('boom'))).toBe(false); + expect(isStaleWriteRejection(undefined)).toBe(false); + }); }); describe('memoizeEncryptionKey', () => { diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 29096e7400..56788e4472 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1,6 +1,7 @@ import { PreconditionFailedError, RUN_ERROR_CODES, + SlotConflictError, WorkflowWorldError, } from '@workflow/errors'; import type { @@ -17,10 +18,14 @@ import { getQueueTopicPrefix, HealthCheckPayloadSchema, HOOK_RESUME_INPUT_VERSION, + isSlotId, + maxSlotOf, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + slotEventId, ulidToDate, + usesSlotIdentity, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; @@ -482,7 +487,21 @@ function recordRequestedEventCursor( } /** - * Appends events whose IDs are not already present in `target`. + * Appends events whose IDs are not already present in `target`, keeping the log + * in slot order. + * + * Arrival order is not log order under slot identity. A slot is reserved when + * its event is issued and written when that issue resolves, so a lower slot can + * be committed after a higher one, and a merge that only appends leaves the + * array in the order the events were *learned*, not the order they occupy. + * That difference decides races: the replay consumes this array positionally + * — the delivery barriers in `pendingDeliveryBarriers` are keyed on the index — + * so a `step_completed` sitting ahead of a `wait_completed` it actually + * follows makes the replay take the branch the log does not record, and the + * next event it reads belongs to a step it never started. + * + * Sorting by event id *is* sorting by slot: ids are zero-padded to a fixed + * width, and one log never mixes them with ULID ids. * * Pass the IDs currently present in `target` when appending repeatedly to the * same array. The set is updated alongside `target`. @@ -506,13 +525,19 @@ export function appendUniqueEvents( } const ids = targetIds ?? new Set(target.map((event) => event.eventId)); + let outOfOrder = false; for (const event of events) { if (ids.has(event.eventId)) { continue; } ids.add(event.eventId); + outOfOrder ||= + target.length > 0 && event.eventId < target[target.length - 1].eventId; target.push(event); } + if (outOfOrder && isSlotId(target[0].eventId)) { + target.sort((a, b) => (a.eventId < b.eventId ? -1 : 1)); + } } /** @@ -708,6 +733,100 @@ export interface LoadedEventLog { cursor: string | null; } +/** + * A loaded snapshot that also tracks the run's event slots, for the numbering + * where the event id is itself the concurrency fence. Slot state is per-replay: + * it is rebuilt from the events every time the log is loaded, never carried + * across a restart. + */ +export interface MutableEventLog extends LoadedEventLog { + /** + * Highest slot present in `events`, or 0 for a log that is empty or + * ULID-numbered. Maintained by `mergeLoadedEvents` from the events merged in, + * never from the array's last element: events are appended without sorting, + * so after a merge the last element need not be the newest. + */ + maxSlot: number; + /** + * Next slot `reserveSlot` will hand out. Only a writer holding the log's + * write chain may draw from it. + */ + nextSlot: number; + /** + * The stale-write rejection this log's tail stopped at, if any. + * + * A rejected claim means the log is missing an event, so nothing else decided + * from it may land either. Recording the rejection here fails the rest of the + * batch locally, instead of spending a round-trip each to be told the same + * thing by the backend. + */ + claimRejection?: unknown; + /** + * Tail of the chain of creates numbered off this log, or `undefined` when + * none is in flight. + * + * Slot claims are taken one at a time. A claim only fences out a concurrent + * writer if it names the slot right after the log's committed tail: numbering + * a whole concurrent batch up front hands its later writes slots far enough + * above the tail that a foreign event landing in between clears every fence + * they carry, and the batch commits decisions taken without it. + */ + writeChain?: Promise; +} + +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ +export function toMutableEventLog( + events: Event[], + cursor: string | null, + slotFloor = 0 +): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); + return { + events, + cursor, + maxSlot, + nextSlot: maxSlot + 1, + }; +} + +/** + * Merges loaded events into `log` in place, keeping `maxSlot` current and + * advancing the reservation pointer past the events merged in. + * + * The merged events are the ones a rejected claim was missing, so the log is + * usable again and its recorded rejection is cleared. + */ +export function mergeLoadedEvents( + log: MutableEventLog, + events: readonly Event[] +): void { + appendUniqueEvents(log.events, events); + log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); + log.claimRejection = undefined; +} + +/** + * Claims the next free slot in `log`. + * + * Only call this while holding the log's write chain: the claim is the fence, + * and it only fences anything while it names the slot immediately after the + * tail this writer has seen. + */ +export function reserveSlot(log: MutableEventLog): number { + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; +} + /** * Whether the optimistic-concurrency guard for event creation is enabled. * **On by default** where the runtime executes: replay-context creates send a @@ -824,8 +943,24 @@ export function preconditionSnapshotParams( } /** - * The events a rejecting World attached to a `PreconditionFailedError`, when it - * returned the ones the client's snapshot was missing inline. + * Whether a World rejected an event creation because the replay that produced it + * had not seen the whole event log. + * + * A lost slot claim rejects with 409 and a failed watermark comparison with 412, + * so the two stay separately countable. They prove the same thing and are + * recovered the same way, by restarting the replay over the corrected log. + */ +export function isStaleWriteRejection(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + +/** + * The events a rejecting World attached to its rejection, when it returned the + * ones the client's snapshot was missing inline. + * + * A slot conflict carries them as typed fields; the watermark guard carries them + * in `details`. Both are read here so callers recover from either without + * branching. * * Returns `null` for anything else — no details, a World that did not implement * this, or a payload that does not narrow cleanly. Callers fall back to @@ -841,14 +976,25 @@ export function preconditionEventDelta( error: unknown, runId: string ): { events: Event[]; cursor: string | null } | null { - if (!PreconditionFailedError.is(error)) { - return null; - } - const details = error.details; - if (typeof details !== 'object' || details === null) { + let events: unknown; + let cursor: unknown; + if (SlotConflictError.is(error)) { + // A truncated delta is not a delta: the restart has to see every event it + // was missing, and only a full reload can guarantee that. + if (error.hasMore) { + return null; + } + events = error.events; + cursor = error.cursor; + } else if (PreconditionFailedError.is(error)) { + const details = error.details; + if (typeof details !== 'object' || details === null) { + return null; + } + ({ events, cursor } = details as { events?: unknown; cursor?: unknown }); + } else { return null; } - const { events, cursor } = details as { events?: unknown; cursor?: unknown }; if (!Array.isArray(events) || events.length === 0) { return null; } @@ -874,6 +1020,167 @@ export type EventCreator = ( params?: CreateEventParams ) => Promise; +/** + * The concurrency fence a replay-context event creation carries. Exactly one of + * the two schemes is ever populated: the precondition snapshot for a run guarded + * by the event-log watermark, `eventId`/`maxSlot` for a run that numbers its + * events by slot. + */ +export interface EventCreateFence extends PreconditionSnapshotParams { + eventId?: string; + maxSlot?: number; +} + +/** + * The fence to attach to a replay-context event creation, under whichever + * scheme the run uses. + * + * Neither scheme is retried in place. A rejection under either one proves the + * replay derived this event from a log that was missing another, and correlation + * ids are drawn from one seeded sequence in mint order — so a replay over the + * corrected log mints different ids and re-posting this write would persist an + * event no correct replay produces. Recovery is a restarted replay + * ({@link isStaleWriteRejection}), never a re-send. + * + * Claiming a slot counts as reserving it, so a caller that fences several + * creates from one log gets a distinct slot per create. Empty when the run is + * fenced neither way, which leaves the create unfenced. + * + * `extraEvents` is how many events *besides* the one being created this write + * publishes: a lazy inline `step_started` also materializes the `step_created` + * it deferred. Those events take the slots immediately below the claim, so this + * reserves them too and names the top one — a World that writes a pair + * derives the lower id from the one it was given. + * + * The reservation has to happen here rather than at the World or its backend. + * Slots are handed out for a whole concurrent batch synchronously, before any of + * it lands, so a second event numbered off the log as the backend sees it would + * take the slot already promised to the next write in the batch — and every + * write after the first in a fan-out would lose its claim. + */ +export function eventCreateFenceFor( + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): EventCreateFence { + if (usesSlotIdentity(specVersion)) { + return reserveSlotFence(log, options?.extraEvents ?? 0).fence; + } + return preconditionSnapshotParams(log.events, log.cursor); +} + +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): { fence: EventCreateFence; slot: number } { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + const slot = reserveSlot(log); + return { fence: { eventId: slotEventId(slot), maxSlot }, slot }; +} + +/** + * Runs one create with the log's claim to itself, taking its slot only once + * every create ahead of it on the log has settled. + * + * A slot claim is an assertion about the tail: "nothing has been published + * since the view I decided from". Claims handed out up front to a concurrent + * batch can only assert that about the first of them — the rest sit above slots + * their own siblings have yet to fill, so a foreign event landing in that space + * satisfies their fences too and they commit on a view that is already missing + * it. Taking claims one at a time keeps every write's fence tight against the + * tail the writer actually saw. + * + * A stale-write rejection therefore stops the whole batch rather than only its + * own write: the batch was decided from a log missing an event, so none of it + * should land. The rejection is recorded on the log and rethrown for the claims + * behind it without a round-trip, since their fences all name the tail it just + * proved wrong. + */ +async function withSerializedClaim( + log: MutableEventLog, + extraEvents: number, + op: (fence: EventCreateFence) => Promise +): Promise { + const ahead = log.writeChain; + let done!: () => void; + log.writeChain = new Promise((resolve) => { + done = resolve; + }); + if (ahead) { + await ahead; + } + try { + if (log.claimRejection !== undefined) { + throw log.claimRejection; + } + const { fence, slot } = reserveSlotFence(log, extraEvents); + const result = await op(fence); + log.maxSlot = Math.max(log.maxSlot, slot); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); + return result; + } catch (error) { + // The slots this attempt drew are not the writer's, and the tail is no + // lower than the claim that lost. Rewinding leaves the next claim naming + // the same slot, which is correct for a write that failed without taking it + // (an entity conflict, say) and is retried on its own. + log.nextSlot = log.maxSlot + 1; + if (isStaleWriteRejection(error)) { + log.claimRejection = error; + } + throw error; + } finally { + done(); + } +} + +/** + * Runs one event create under whichever fence its run uses. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fenced create for a claim issued from a concurrent batch — an inline + * step's `step_started`, or a suspension flush's writes. + * + * Under slot numbering this serializes the batch's claims so each one is taken + * against the tail its writer actually saw ({@link withSerializedClaim}); under + * the watermark every claim in the batch legitimately carries the same + * snapshot, so they all run concurrently off one fence. + * + * Neither scheme re-issues a rejected claim at a free number. A rejection says + * this replay decided from a log it had not fully seen, and the missing event + * can be the one that would have sent the workflow down another branch — with + * correlation ids counted in branch order, every id after that branch moves + * with it, so the re-issued write would land under an identity that now names a + * different step. The rejection propagates and the run replays over the + * corrected log ({@link isStaleWriteRejection}). + */ +export function claimFenceFor( + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + return (op) => withSerializedClaim(log, options?.extraEvents ?? 0, op); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index 49769f2518..a6baf45da0 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -10,7 +10,10 @@ * 2. The restart reloads the whole event log with no cursor, because a hole is * defined by ULID time while a cursor filters lexicographically — unless * the World attached the missing events to the 412, which the runtime - * consumes with no events.list round trip at all (first restart only). + * consumes with no events.list round trip at all. A run numbering its events + * by slot heals from its cursor when nothing was attached, since slot ids + * sort in write order and density proves afterwards that the merge closed + * the gap; a short count sends it back to the full reload. * 3. Restarts are bounded; once the bound is spent the runtime schedules a * delayed re-invocation instead of failing the run — and that escalation is * itself counted on the queue message, so a run that can never observe its @@ -19,16 +22,25 @@ * Modeled on wait-completion-replay.test.ts, but with real ULID event IDs so * latestEventStateUpdatedAt() actually derives snapshot times. */ -import { PreconditionFailedError, RUN_ERROR_CODES } from '@workflow/errors'; +import { + PreconditionFailedError, + RUN_ERROR_CODES, + SlotConflictError, +} from '@workflow/errors'; import { type CreateEventRequest, type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, + slotIdBody, type WorkflowRun, type World, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { runtimeLogger } from '../logger.js'; import { registerStepFunction } from '../private.js'; import { workflowEntrypoint } from '../runtime.js'; @@ -37,12 +49,11 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; -import { createContext } from '../vm/index.js'; import { getPreconditionMaxInProcessRestarts, getPreconditionMaxReinvocations, getPreconditionReinvokeDelaySeconds, + PRECONDITION_MAX_INPROCESS_RESTARTS, } from './constants.js'; import { setWorld } from './world.js'; @@ -89,6 +100,8 @@ interface SnapshotParams { stateUpdatedAt: number | undefined; stateEventCount: number | undefined; stateCursor: string | undefined; + /** The claimed position, on a run that numbers its events by slot. */ + eventId?: string | undefined; } async function runPreconditionScenario(options: { @@ -99,6 +112,17 @@ async function runPreconditionScenario(options: { * or a payload the runtime must refuse to narrow (`malformed`). */ attachDelta?: 'complete' | 'malformed'; + /** + * Number the run's events by slot rather than by ULID, and reject with the + * 409 a taken slot produces instead of the 412. + */ + slotIdentity?: boolean; + /** + * Slot mode only: land a second out-of-band event ahead of the hook and hide + * it from the page above the cursor, so a restart that tops up incrementally + * ends holding a log that is short of its own highest slot. + */ + hideFirstOutsideEventFromCursor?: boolean; }) { vi.spyOn(Date, 'now').mockReturnValue(+fixedNow); @@ -113,14 +137,20 @@ async function runPreconditionScenario(options: { undefined ); - const { globalThis: vmGlobalThis } = createContext({ + const SPEC = options.slotIdentity + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; + + // Correlation ids are ULIDs in both modes: slot identity numbers event ids + // only. Each kind draws from its own sequence, so these are the first id of + // three separate sequences rather than three draws of one. + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); - const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; - const syncStep0CorrelationId = `step_${vmUlid(+startedAt)}`; - const waitCorrelationId = `wait_${vmUlid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; + const syncStep0CorrelationId = `step_${generate('step')}`; + const waitCorrelationId = `wait_${generate('wait')}`; const workflowRun: WorkflowRun = { runId, @@ -128,23 +158,35 @@ async function runPreconditionScenario(options: { status: 'running', input: workflowArgs, deploymentId, - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, startedAt, createdAt: startedAt, updatedAt: startedAt, }; // Real ULID event IDs at controlled times so latestEventStateUpdatedAt() - // resolves an actual epoch-ms snapshot from the loaded log. + // resolves an actual epoch-ms snapshot from the loaded log. Under slot + // identity the id is instead the next free slot, or the one the writer + // claimed, and the log stays dense from slot 1. const hostUlid = monotonicFactory(); let eventIndex = 0; - const event = (data: CreateEventRequest, atMs?: number): Event => { + let nextSlot = FIRST_SLOT; + const event = ( + data: CreateEventRequest, + atMs?: number, + claimedEventId?: string + ): Event => { const t = atMs ?? +startedAt + ++eventIndex * 100; + let eventId = `evnt_${hostUlid(t)}`; + if (options.slotIdentity) { + eventId = claimedEventId ?? `evnt_${slotIdBody(nextSlot)}`; + nextSlot = Math.max(nextSlot, slotFromId(eventId) ?? nextSlot) + 1; + } return { ...data, - specVersion: data.specVersion ?? SPEC_VERSION_CURRENT, + specVersion: data.specVersion ?? SPEC, runId, - eventId: `evnt_${hostUlid(t)}`, + eventId, createdAt: new Date(t), } as Event; }; @@ -152,19 +194,19 @@ async function runPreconditionScenario(options: { const staleEvents: Event[] = [ event({ eventType: 'run_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, eventData: { deploymentId, workflowName, input: workflowArgs }, }), - event({ eventType: 'run_started', specVersion: SPEC_VERSION_CURRENT }), + event({ eventType: 'run_started', specVersion: SPEC }), event({ eventType: 'hook_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { token: hookToken }, }), event({ eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { stepName: 'syncStep', @@ -177,12 +219,12 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, }), event({ eventType: 'step_completed', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { result: await dehydrateStepReturnValue(undefined, runId, undefined), @@ -190,7 +232,7 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'wait_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: waitCorrelationId, eventData: { resumeAt: new Date(+startedAt - 1_000) }, }), @@ -200,10 +242,30 @@ async function runPreconditionScenario(options: { const staleEventsCursor = 'cursor-after-stale-events'; const OUTSIDE_EVENT_MS = +startedAt + 5_000; + // Lands ahead of the winning delivery and is withheld from the page above + // the cursor, so an incremental top-up comes back holding fewer events than + // its own highest slot names. + const hiddenOutsideEvent = options.hideFirstOutsideEventFromCursor + ? event( + { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: hookCorrelationId, + eventData: { + payload: await dehydrateStepReturnValue( + { value: 'hook-poke' }, + runId, + undefined + ), + }, + }, + OUTSIDE_EVENT_MS - 1 + ) + : undefined; const hookReceivedEvent = event( { eventType: 'hook_received', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { payload: await dehydrateStepReturnValue( @@ -234,7 +296,9 @@ async function runPreconditionScenario(options: { }) => { const data = params.pagination?.cursor === staleEventsCursor - ? durableEvents.slice(staleEvents.length) + ? durableEvents + .slice(staleEvents.length) + .filter((e) => e !== hiddenOutsideEvent) : [...durableEvents]; return { data, @@ -263,6 +327,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt?: number; stateEventCount?: number; stateCursor?: string; + eventId?: string; } ) => { createParams.push({ @@ -270,6 +335,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt: params?.stateUpdatedAt, stateEventCount: params?.stateEventCount, stateCursor: params?.stateCursor, + eventId: params?.eventId, }); if (request.eventType === 'run_started') { @@ -279,11 +345,32 @@ async function runPreconditionScenario(options: { if (request.eventType === 'wait_completed') { // The out-of-band hook payload becomes durable just before the // wait_completed commit — this is the exact race the guard closes. + if (hiddenOutsideEvent && !durableEvents.includes(hiddenOutsideEvent)) { + durableEvents.push(hiddenOutsideEvent); + } if (!durableEvents.includes(hookReceivedEvent)) { durableEvents.push(hookReceivedEvent); } if (waitCompletedRejections < (options.rejectWaitCompletedTimes ?? 0)) { waitCompletedRejections++; + // Both rejections say the same thing — "you decided from a log you + // had not fully seen" — and differ only in how the writer found out: + // a watermark comparison, or the slot it claimed already being + // occupied by the event it was missing. + if (options.slotIdentity) { + throw new SlotConflictError( + `Event ${params?.eventId} is already taken.`, + { + eventId: params?.eventId ?? '', + events: + options.attachDelta === 'complete' + ? [hookReceivedEvent] + : undefined, + cursor: + options.attachDelta === 'complete' ? 'cursor-409' : undefined, + } + ); + } throw new PreconditionFailedError( 'Run state is stale: the client event log is missing at least one event at or before its snapshot.', options.attachDelta === undefined @@ -304,17 +391,30 @@ async function runPreconditionScenario(options: { !!request.eventData && (request.eventData as { input?: unknown }).input !== undefined; let effectiveRequest = request; + // A claim covers the whole write, so a lazy start's synthesized + // step_created takes the claimed position and the step_started the one + // after it — the two extra slots the caller reserved for exactly this. + let claimedEventId = params?.eventId; + const takeClaim = () => { + const claim = claimedEventId; + claimedEventId = undefined; + return claim; + }; if (lazyStepStart) { const lazyData = request.eventData as { stepName?: string; input?: unknown; }; - const syntheticStepCreated = event({ - eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: request.correlationId, - eventData: { stepName: lazyData.stepName, input: lazyData.input }, - } as CreateEventRequest); + const syntheticStepCreated = event( + { + eventType: 'step_created', + specVersion: SPEC, + correlationId: request.correlationId, + eventData: { stepName: lazyData.stepName, input: lazyData.input }, + } as CreateEventRequest, + undefined, + takeClaim() + ); durableEvents.push(syntheticStepCreated); createdEvents.push(syntheticStepCreated); const { input: _strippedInput, ...startEventData } = lazyData; @@ -324,7 +424,7 @@ async function runPreconditionScenario(options: { } as CreateEventRequest; } - const created = event(effectiveRequest); + const created = event(effectiveRequest, undefined, takeClaim()); durableEvents.push(created); createdEvents.push(created); if (effectiveRequest.eventType === 'step_started') { @@ -344,7 +444,7 @@ async function runPreconditionScenario(options: { const queue = vi.fn().mockResolvedValue({ messageId: 'msg_step' }); const fakeWorld = { - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, createQueueHandler: vi.fn((_prefix, handler) => { capturedHandler = handler; return vi.fn(); @@ -931,8 +1031,6 @@ async function inlineClaimRejectionScenario() { }; } -pinSharedCorrelationIds(); - describe('precondition guard through the real replay loop', () => { let originalGuard: string | undefined; let originalRestartBound: string | undefined; @@ -1054,6 +1152,89 @@ describe('precondition guard through the real replay loop', () => { ); }); + it('heals a slot-numbered log from its cursor instead of reloading it whole', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + }); + await result.handlerInvocation; + + expect(result.waitCompletedRejectionCount()).toBe(1); + // Slot ids sort in write order, so the event the snapshot was missing is + // strictly above the cursor and one incremental page brings it in. The + // full reload the ULID path needs exists only because a hole defined by + // ULID time can sort below the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + + const waitCreates = result.createParams.filter( + (c) => c.eventType === 'wait_completed' + ); + expect(waitCreates).toHaveLength(2); + // No watermark is sent on a run that fences by position. + expect(waitCreates[0]?.stateUpdatedAt).toBeUndefined(); + // The rejected claim sat at the position the hook had just taken; the + // restarted replay claims the one after it. + expect(slotFromId(waitCreates[1]?.eventId ?? '')).toBe( + (slotFromId(waitCreates[0]?.eventId ?? '') ?? 0) + 1 + ); + + // Replay after the restart observed the hook and took the hook branch. + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + + it('falls back to the full reload when a slot top-up leaves the log short of its own highest slot', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + hideFirstOutsideEventFromCursor: true, + }); + await result.handlerInvocation; + + // A slot log is dense from slot 1, so a complete one holds exactly + // `maxSlot` events. The page above the cursor withheld one, the count came + // up short, and the restart fell through to the authoritative load — the + // check is what lets the cheap path be taken in the first place. + expect(cursorlessLoads(result.listEvents)).toBe(1); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(2); + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + + it('spends a larger in-process restart budget when restarts heal from the cursor', async () => { + // One more rejection than a reloading run is allowed to absorb. That run + // would be out of budget and re-invoked; this one keeps recovering in + // process, because each of its restarts costs one incremental page rather + // than a full reload and so is not worth a queue hop to avoid. + const rejections = PRECONDITION_MAX_INPROCESS_RESTARTS + 1; + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: rejections, + }); + + await expect(result.handlerInvocation).resolves.toBeUndefined(); + expect(result.waitCompletedRejectionCount()).toBe(rejections); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(rejections + 1); + // Every one of them healed from the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + }); + it('falls back to the full reload when the 412 payload does not narrow to events', async () => { const result = await runPreconditionScenario({ rejectWaitCompletedTimes: 1, @@ -1082,6 +1263,31 @@ describe('precondition guard through the real replay loop', () => { expect(cursorlessLoads(result.listEvents)).toBe(1); }); + it('trusts an attached delta on every restart of a slot-numbered run', async () => { + const warn = vi.spyOn(runtimeLogger, 'warn'); + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 2, + attachDelta: 'complete', + }); + await result.handlerInvocation; + + // Density checks the merged log here, so a delta that left a hole would be + // caught rather than replayed over. Nothing rests on the World's own + // bookkeeping, and no restart has to spend a load of any kind to be sure. + expect(result.waitCompletedRejectionCount()).toBe(2); + expect(cursorlessLoads(result.listEvents)).toBe(0); + expect( + warn.mock.calls + .filter( + ([message]) => + message === + 'Event creation rejected as stale; restarting replay in-process' + ) + .map(([, fields]) => (fields as { source?: string }).source) + ).toEqual(['inline-delta', 'inline-delta']); + }); + it('reports whether a restarted replay reloaded the events it was missing', async () => { const warn = vi.spyOn(runtimeLogger, 'warn'); const result = await runPreconditionScenario({ diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index a28dda7b78..898dc9951e 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -339,8 +339,9 @@ async function dispatchPendingOps(params: { // validated by the world — parallel dispatch would otherwise record a // spurious hook_conflict against the run's own disposed hook (e.g. a // dispose→recreate loop reusing one token). Different tokens have no - // claim interaction, so token groups run in parallel with each other - // and with the non-hook ops below. + // claim interaction, so token groups run in parallel with each other, + // but the whole hook phase is awaited before any step/wait op is issued + // (see the barrier below). const hookOpsByToken = new Map< string, (PendingHook | PendingHookDispose)[] @@ -365,8 +366,9 @@ async function dispatchPendingOps(params: { hookOpsByToken.set(key, [op as PendingHook | PendingHookDispose]); } } + const hookPhasePromises: Promise[] = []; for (const group of hookOpsByToken.values()) { - opsPromises.push( + hookPhasePromises.push( (async () => { for (const op of group) { if (op.type === 'hook') { @@ -378,6 +380,25 @@ async function dispatchPendingOps(params: { })() ); } + // Barrier: every hook_created is durable before any step is dispatched. + // This mirrors the node:vm suspension handler, which settles its hook + // phase before the step phase. A step that receives an AbortSignal can + // call controller.abort() as soon as it starts running, and that resume + // targets the abort controller's hook by token: if the hook row does not + // exist yet the resume throws HookNotFoundError and the abort is lost + // (serialization.ts treats the resume as best-effort). Dispatching steps + // concurrently with hook creation makes that a race decided by write + // latency. + if (hookPhasePromises.length > 0) { + // Settle rather than Promise.all so a rejecting group cannot leave its + // siblings' in-flight writes as unhandled rejections. + const rejections = (await Promise.allSettled(hookPhasePromises)) + .filter((r): r is PromiseRejectedResult => r.status === 'rejected') + .map((r) => r.reason); + if (rejections.length > 0) { + throw rejections[0]; + } + } for (const op of pendingOperations) { if (op.type === 'step' && !op.hasCreatedEvent) { @@ -748,6 +769,10 @@ export async function runWorkflowWithQuickJS(params: { encryptionKey, port, runInput, + // Fail closed on Hook retention when the World doesn't declare support, + // matching the node:vm engine's world-capability gate (see hook.ts). + worldSupportsHookRetention: + world.capabilities?.hookRetention?.active === true, }); runtimeLogger.debug('QuickJS runtime: VM returned', { diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 71e674df65..8c2f420f41 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -103,6 +103,7 @@ describe('runQuickJSWorkflow', () => { workflowId: 'workflow//test//workflow', workflowRun: makeRun(), events: [], + worldSupportsHookRetention: true, }); expect(result.suspended?.pendingOperations).toContainEqual( @@ -134,6 +135,7 @@ describe('runQuickJSWorkflow', () => { workflowId: 'workflow//test//workflow', workflowRun: makeRun(), events: [], + worldSupportsHookRetention: true, }); expect(result.suspended?.pendingOperations).toContainEqual( diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 8043564a05..7543da736a 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -207,6 +207,15 @@ export interface QuickJSRuntimeOptions { * (eventually-consistent read after the parent's start() wrote it). */ runInput?: RunInput; + /** + * Whether the configured World supports `experimental_minRetention` for + * Hooks (`world.capabilities.hookRetention.active === true`). Gates the + * VM's `createHook()` retention path so the QuickJS engine fails closed on + * Worlds without the capability, matching the node:vm engine's + * world-capability gate (see `hook.ts`). Defaults to unsupported when + * omitted. + */ + worldSupportsHookRetention?: boolean; } // ---- VM Bootstrap Code ---- @@ -569,6 +578,13 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { if (options.isWebhook === true && options.experimental_minRetention !== undefined) { throw new Error('Webhook hooks do not support \`experimental_minRetention\`. Use a non-webhook \`createHook()\` with \`resumeHook()\`.'); } + // Fail closed when the configured World does not declare hook-retention + // support (parity with the node:vm engine's world-capability gate in + // hook.ts). The host injects \`__worldSupportsHookRetention\` per run from + // \`world.capabilities.hookRetention.active\`; absence means unsupported. + if (options.experimental_minRetention !== undefined && globalThis.__worldSupportsHookRetention !== true) { + throw new Error('The configured World does not support \`experimental_minRetention\` for Hooks.'); + } var token = options.token || globalThis.__generateNanoid(); var correlationId = "hook_" + globalThis.__generateUlid(); var isDisposed = false; @@ -971,6 +987,8 @@ export async function runQuickJSWorkflow( options: QuickJSRuntimeOptions ): Promise { const { workflowCode, workflowId, workflowRun, events } = options; + const worldSupportsHookRetention = + options.worldSupportsHookRetention === true; const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); @@ -1065,6 +1083,14 @@ export async function runQuickJSWorkflow( `globalThis.__ulidTimestamp = ${runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt)};` ).dispose(); + // Whether the configured World supports Hook retention. Gates the VM's + // `createHook({ experimental_minRetention })` path so QuickJS fails + // closed on Worlds without the capability, matching the node:vm engine + // (see hook.ts's world-capability gate). + vm.evalCode( + `globalThis.__worldSupportsHookRetention = ${worldSupportsHookRetention};` + ).dispose(); + // `process.env` — parity with the node:vm engine, which exposes a frozen // copy of the host env (vm/index.ts). Injected per run so the snapshot of // the env is taken at invocation time, same as node. diff --git a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts index 3e37a92f2e..5853a0ce36 100644 --- a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts +++ b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts @@ -22,13 +22,12 @@ import { } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { workflowEntrypoint } from '../runtime.js'; import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; -import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; vi.mock('@vercel/functions', () => ({ waitUntil: vi.fn() })); @@ -82,15 +81,14 @@ async function runResumeConsumerScenario(options: { undefined ); - // Derive the hook correlation id the seeded VM will compute during replay, - // so the preloaded / re-ensured hook_received matches the workflow's own - // createHook call (id assignment order: the hook is the first id derived). - const { globalThis: vmGlobalThis } = createContext({ + // Derive the hook correlation id the replay will compute, so the preloaded / + // re-ensured hook_received matches the workflow's own createHook call. The + // workflow creates one hook, so it draws its kind's first id. + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); - const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; const workflowRun: WorkflowRun = { runId, @@ -253,8 +251,6 @@ async function runResumeConsumerScenario(options: { }; } -pinSharedCorrelationIds(); - describe('lazy hook resume consumer preload (Perf Option A)', () => { afterEach(() => { setWorld(undefined); diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 8904600556..9d96422c7c 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 4bbe37fa56..a0e9e615a3 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -1,6 +1,7 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; import type { Event, World } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { createWorld } from '@workflow/world-local'; @@ -171,7 +172,7 @@ describe('executeStep — compute instance stamping', () => { counter += 1; }); - it('stamps computeInstanceId on step_started without displacing the precondition snapshot', async () => { + it('stamps computeInstanceId on step_started without displacing the claim fence', async () => { const world = makeWorld(); const stepName = uniqueStepName(); const { runId, stepId } = await setupRunningStep({ @@ -197,7 +198,7 @@ describe('executeStep — compute instance stamping', () => { workflowStartedAt: Date.now(), stepId, stepName, - preconditionSnapshot, + claimFence: (op) => op(preconditionSnapshot), }); const started = createSpy.mock.calls.filter( @@ -210,3 +211,153 @@ describe('executeStep — compute instance stamping', () => { expect(started[0]?.[2]).toMatchObject(preconditionSnapshot); }); }); + +// A run that numbers its events by position rejects a claim whose slot was +// taken, and the rejection carries the events that took it. When those events +// show the same step already started, the loser is in the ordinary "another +// handler owns this step" position and skips — the outcome an unfenced write +// reaches via EntityConflictError. Anything less than proof of the same call +// must propagate instead, because correlation ids are positional: a replay +// that diverged can reach the same step number naming a different call. +describe('executeStep — slot rejection carrying a duplicate start', () => { + afterEach(() => { + counter += 1; + }); + + function startedEvent(opts: { + stepId: string; + stepName: string; + input?: unknown; + }) { + return { + eventType: 'step_started', + correlationId: opts.stepId, + eventData: { + stepName: opts.stepName, + ...(opts.input !== undefined ? { input: opts.input } : {}), + }, + }; + } + + async function runAgainstRejection(opts: { + events: unknown[]; + lazyStepInput?: Uint8Array; + onBody?: () => void; + }) { + const world = makeWorld(); + const stepName = uniqueStepName(); + let bodyRuns = 0; + const { runId, stepId } = await setupRunningStep({ + world, + stepName, + onBody: () => { + bodyRuns += 1; + opts.onBody?.(); + }, + }); + + const rejection = new SlotConflictError('slot taken', { + eventId: 'evnt_00000000000000000000000007', + events: opts.events.map((build) => + typeof build === 'function' + ? (build as (ids: { stepId: string; stepName: string }) => unknown)({ + stepId, + stepName, + }) + : build + ), + }); + + const run = () => + executeStep({ + world, + workflowRunId: runId, + workflowName: 'wf', + workflowStartedAt: Date.now(), + stepId, + stepName, + lazyStepInput: opts.lazyStepInput, + // Take the awaited claim path so the rejection is translated before a + // body ever runs; the optimistic path reconciles the same way but + // would run the body first and muddy the assertion. + suppressOptimisticStart: true, + claimFence: () => Promise.reject(rejection), + }); + + return { run, bodyRuns: () => bodyRuns, stepName, stepId }; + } + + it('skips when the delta already started this step (no lazy input to compare)', async () => { + const { run, bodyRuns } = await runAgainstRejection({ + events: [ + (ids: { stepId: string; stepName: string }) => startedEvent(ids), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('skips when the delta started this step with byte-identical input', async () => { + const input = new Uint8Array([1, 2, 3]); + const { run, bodyRuns } = await runAgainstRejection({ + lazyStepInput: input, + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 3]) }), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('propagates when the delta started the same slot under a different step name', async () => { + const { run } = await runAgainstRejection({ + events: [ + (ids: { stepId: string }) => + startedEvent({ ...ids, stepName: 'step//./other//someOtherStep' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta started this step with different input bytes', async () => { + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 4]) }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta start carries a remote ref instead of inline bytes', async () => { + // A ref says nothing about the value behind it, so identity is unprovable + // and the replay restart is the only correct answer. + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: { ref: 'payload_abc' } }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta holds no start for this step', async () => { + const { run } = await runAgainstRejection({ + events: [ + { eventType: 'step_completed', correlationId: 'step_other' }, + (ids: { stepName: string }) => + startedEvent({ ...ids, stepId: 'step_other' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); +}); diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index c412271a2a..8fbb78c13b 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -4,6 +4,7 @@ import { FatalError, RetryableError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowRuntimeError, @@ -51,9 +52,10 @@ import { } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; import { + type EventCreateFence, type EventCreator, + type FencedCreate, memoizeEncryptionKey, - type PreconditionSnapshotParams, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; import { @@ -86,6 +88,73 @@ function extractInlineDelta( }; } +/** + * Byte-equality for two dehydrated step inputs, used to decide whether a + * conflicting `step_started` describes the same call as ours. Only inline + * bytes can answer that: a `SerializedData` that came back as a remote ref + * carries no payload, and two refs being unequal says nothing about the + * values behind them, so anything that is not a pair of `Uint8Array`s is + * reported as "cannot tell". + */ +function sameSerializedInput( + a: SerializedData | undefined, + b: SerializedData | undefined +): boolean { + if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false; + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} + +/** + * Decide whether a slot rejection carrying a delta is a *benign duplicate*: + * the events we lost the slot to already contain a `step_started` for this + * exact step, so a concurrent handler is running (or has run) the very call + * we were about to claim. The step is then someone else's to finish and we + * can skip, exactly as we do on the `EntityConflictError` the unfenced path + * would have raised instead. + * + * The identity test is deliberately strict. Correlation ids are positional + * ordinals of one seeded sequence, so a replay that diverged — a + * `Promise.race` between a hook and a step resolving the other way, say — can + * arrive at the same correlation id naming a different call. Matching on it + * alone would let such a replay adopt a foreign step's completion as its own. So + * the step name must match, and on the lazy path (where we hold the input) + * the inputs must be byte-identical. Anything we cannot prove identical + * returns false and takes the replay restart, which is always correct and + * merely slower. + */ +function isBenignDuplicateStart( + events: unknown[], + stepId: string, + stepName: string, + lazyStepInput: SerializedData | undefined +): boolean { + for (const candidate of events) { + if (!candidate || typeof candidate !== 'object') continue; + const event = candidate as { + eventType?: unknown; + correlationId?: unknown; + eventData?: { stepName?: unknown; input?: unknown } | null; + }; + if (event.eventType !== 'step_started') continue; + if (event.correlationId !== stepId) continue; + if (event.eventData?.stepName !== stepName) continue; + if (lazyStepInput === undefined) return true; + if ( + sameSerializedInput( + lazyStepInput, + event.eventData?.input as SerializedData + ) + ) { + return true; + } + } + return false; +} + export interface StepExecutorParams { world: World; workflowRunId: string; @@ -141,20 +210,25 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Precondition-guard snapshot of the event log the caller's replay loaded, to - * attach to this step's `step_started` claim. On the lazy inline path the - * claim is the step's FIRST durable write (its `step_created` is deferred), - * so without this the claim would bypass the optimistic-concurrency guard - * entirely: a replay working from a stale view could claim — and then commit - * — a step scheduled without observing an event it never loaded. A - * guard-enforcing World rejects a stale claim with `PreconditionFailedError` - * (412); executeStep does NOT translate that rejection (re-claiming in place - * would still commit the stale schedule), so it propagates for the caller to - * abandon the batch and restart its replay. Undefined when the guard is - * disabled or the caller has no snapshot; Worlds that don't enforce the guard - * ignore it. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies. + * + * On the lazy inline path the claim is the step's FIRST durable write (its + * `step_created` is deferred), so without a fence it would be unguarded + * entirely: a replay working from a stale view could claim — and then commit — + * a step scheduled without observing an out-of-band event. A fencing World + * rejects such a claim with `SlotConflictError` (409) or + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, see + * `claimFenceFor`. Either way executeStep does NOT translate a rejection that + * reaches it, so an unretried one propagates for the caller to abandon the + * batch and force a fresh replay. + * + * Undefined when the caller has no snapshot; a World that does not fence + * ignores it. */ - preconditionSnapshot?: PreconditionSnapshotParams; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -268,6 +342,10 @@ export async function executeStep( // put us on the Vercel branch with nothing to build a host from, making // `https://` the base URL of every step. const isVercel = Boolean(process.env.VERCEL_URL); + // Unfenced when the caller passes no fence, which runs the create plainly. A + // World that does not fence also ignores a fence it is handed, so passing one + // is never wrong. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -488,6 +566,37 @@ export async function executeStep( }); return { type: 'skipped' }; } + if ( + SlotConflictError.is(err) && + isBenignDuplicateStart( + err.events, + stepId, + stepName, + params.lazyStepInput + ) + ) { + // We lost the slot to a writer that had already started this same + // step, so this is the ordinary "another handler owns it" outcome + // wearing a slot rejection — the same thing an unfenced write reports + // as EntityConflictError. Skip rather than restart the replay: the + // restart would re-derive this identical claim and lose again. + // + // The delta on the error is deliberately NOT merged into the log. + // Adopting events this VM never replayed would let the next claim + // fence itself against a tail it cannot account for, which is the + // one thing the fence exists to prevent. Siblings in the batch fail + // with their own rejection and the handler defers as usual. + runtimeLogger.debug('Step already started by a concurrent writer', { + stepName, + stepId, + workflowRunId, + }); + span?.setAttributes({ + ...Attribute.StepSkipped(true), + ...Attribute.StepSkipReason('completed'), + }); + return { type: 'skipped' }; + } if (TooEarlyError.is(err)) { const timeoutSeconds = Math.max(1, err.retryAfter ?? 1); runtimeLogger.debug('Step retryAfter timestamp not yet reached', { @@ -529,13 +638,15 @@ export async function executeStep( let step: Step; // Params for the `step_started` create on either path below: the ambient - // compute-instance stamp plus the optimistic-concurrency claim guard. - const startEventParams: CreateEventParams = { + // compute-instance stamp plus whichever fence the claim is running under. + const startEventParams = ( + fence: EventCreateFence | undefined + ): CreateEventParams => ({ computeInstanceId: COMPUTE_INSTANCE_ID, - // Spread as a unit: the three snapshot fields describe one snapshot and - // must travel together — see StepExecutorParams.preconditionSnapshot. - ...params.preconditionSnapshot, - }; + // Spread as a unit: the fence's fields describe one fence and must + // travel together — see StepExecutorParams.claimFence. + ...fence, + }); // `Date.now()` taken immediately before the `step_started` create is // issued (either path below) — anchors RSFS's end point. See // StepLatencyEventData.rsfs and the call sites below. @@ -575,26 +686,29 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart: + // the body result is discarded, and unless the rejection proves + // another writer already started this same step (a benign + // duplicate, skipped) it propagates to the caller. + return runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Guard the claim — see StepExecutorParams.preconditionSnapshot. A - // stale (412) rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - startEventParams + startEventParams(fence) + ) ); } ); @@ -635,26 +749,28 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Guard the claim — see StepExecutorParams.preconditionSnapshot. A - // stale (412) rejection is intentionally NOT translated by - // startErrorToResult below, so it propagates to the caller for a - // fresh replay. - startEventParams + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry propagates to the caller for a fresh replay, + // except where startErrorToResult below can read the rejection's delta + // as "another writer already started this exact step" and skip. + const startResult = await runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + startEventParams(fence) + ) ); if (!startResult.step) { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 4e189c77d5..02b1eb815f 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,9 +31,9 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { + claimFenceFor, type EventCreator, - type LoadedEventLog, - preconditionSnapshotParams, + type MutableEventLog, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; @@ -45,14 +45,14 @@ export interface SuspensionHandlerParams { requestId?: string; /** * The runtime's loaded event log. Every event creation this suspension makes - * is sent with the precondition snapshot derived from it, so a backend that - * has recorded an event the replay did not see rejects the write with a 412 - * instead of accepting a divergent event. The rejection is not retried here: - * the event's correlation id was minted by *this* replay's seeded sequence, - * so re-committing it against a corrected log would persist an event no - * correct replay produces. The caller restarts the replay instead. + * carries a fence derived from it, the event slot it claims, so a backend that + * has recorded an event the replay did not see rejects the write instead of + * accepting a divergent event. The rejection is not retried here: the event's + * correlation id was minted by *this* replay's seeded sequence, so + * re-committing it against a corrected log would persist an event no correct + * replay produces. The caller restarts the replay instead. */ - eventLog?: LoadedEventLog; + eventLog?: MutableEventLog; /** * Turbo mode only: a promise that resolves once the backgrounded * `run_started` has landed (the run exists). When present, every world write @@ -276,19 +276,19 @@ export async function handleSuspension({ reporter.withEventCreate(params, (p) => world.events.create(runId, data, p) ); - // Adds the optimistic-concurrency guard when the caller supplied a loaded - // event log; without one it creates directly (callers with no replay - // snapshot, e.g. tests). A stale (412) rejection propagates to the caller, - // which restarts the replay from a corrected log — it is not retried here, - // because the event's correlation id was minted by *this* replay's seeded - // sequence, so re-committing it against a corrected log would persist an - // event no correct replay produces. + // Fences the create against the run's event log when the caller supplied a + // loaded one; without it the create goes out unfenced (callers with no replay + // snapshot, e.g. tests). A rejection propagates to the caller, which restarts + // the replay from a corrected log — it is not retried here, because the + // event's correlation id was minted by *this* replay's seeded sequence, so + // re-committing it against a corrected log would persist an event no correct + // replay produces. const createGuarded: EventCreator = (data, params) => eventLog - ? createEvent(data, { - ...params, - ...preconditionSnapshotParams(eventLog.events, eventLog.cursor), - }) + ? claimFenceFor( + eventLog, + run.specVersion + )((fence) => createEvent(data, { ...params, ...fence })) : createEvent(data, params); // Separate queue items by type const stepItems = suspension.steps.filter( diff --git a/packages/core/src/runtime/wait-completion-replay.test.ts b/packages/core/src/runtime/wait-completion-replay.test.ts index 27ca033d37..9a35ac110d 100644 --- a/packages/core/src/runtime/wait-completion-replay.test.ts +++ b/packages/core/src/runtime/wait-completion-replay.test.ts @@ -5,8 +5,8 @@ import { type WorkflowRun, type World, } from '@workflow/world'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdGenerator } from '../correlation-id.js'; import { registerStepFunction } from '../private.js'; import { workflowEntrypoint } from '../runtime.js'; import { @@ -14,8 +14,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; -import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; vi.mock('@vercel/functions', () => ({ @@ -86,14 +84,13 @@ async function runStaleWaitReplayScenario(options: { undefined ); - const { globalThis: vmGlobalThis } = createContext({ + const generate = createCorrelationIdGenerator({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); - const hookCorrelationId = `hook_${ulid(+startedAt)}`; - const syncStep0CorrelationId = `step_${ulid(+startedAt)}`; - const waitCorrelationId = `wait_${ulid(+startedAt)}`; + const hookCorrelationId = `hook_${generate('hook')}`; + const syncStep0CorrelationId = `step_${generate('step')}`; + const waitCorrelationId = `wait_${generate('wait')}`; const workflowRun: WorkflowRun = { runId, @@ -435,8 +432,6 @@ function expectHookBranchQueued( ); } -pinSharedCorrelationIds(); - describe('workflow handler wait completion replay', () => { afterEach(() => { setWorld(undefined); diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index ff511bbcf9..225ced8337 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -27,9 +27,11 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -43,15 +45,16 @@ import { createContext } from './vm/index.js'; import { createCreateHook } from './workflow/hook.js'; import { createSleep } from './workflow/sleep.js'; +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext( events: Event[], replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -73,10 +76,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -95,12 +94,22 @@ function setupWorkflowContext( return ctx; } -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 4); +const WAIT_IDS = correlationIds('wait', 4); +const HOOK_IDS = correlationIds('hook', 4); async function runWithDiscontinuation( ctx: WorkflowOrchestratorContext, @@ -142,7 +151,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -150,7 +159,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -158,7 +167,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -166,7 +175,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: hookPayload }, createdAt: new Date(), }, @@ -174,7 +183,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -182,7 +191,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterHook' }, createdAt: new Date(), }, @@ -190,7 +199,7 @@ async function buildEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -269,7 +278,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -277,7 +286,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -285,7 +294,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -293,7 +302,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -301,7 +310,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -309,7 +318,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -317,7 +326,7 @@ async function buildWaitEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -392,7 +401,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -400,7 +409,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -408,7 +417,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -416,7 +425,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt: RESUME_AT }, createdAt: new Date(), }, @@ -424,7 +433,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_failed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', error: stepAError }, createdAt: new Date(), }, @@ -432,7 +441,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -440,7 +449,7 @@ async function buildFailedEventLog(): Promise { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterFailure' }, createdAt: new Date(), }, diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 64ffeaa276..a624eba172 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -2,9 +2,11 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -90,15 +92,16 @@ import { createSleep } from './workflow/sleep.js'; * one cache exactly like the replay loop inside a single production queue * delivery does (see the `ReplayPayloadCache` class docstring). */ +const FIXED_TIMESTAMP = 1753481739458; + function setupWorkflowContext( events: Event[], replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', - fixedTimestamp: 1753481739458, + fixedTimestamp: FIXED_TIMESTAMP, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; @@ -122,10 +125,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -144,13 +143,22 @@ function setupWorkflowContext( return ctx; } -// Deterministic correlation IDs from the ULID generator with seed 'test' -const CORR_IDS = [ - '01K11TFZ62YS0YYFDQ3E8B9YCV', - '01K11TFZ62YS0YYFDQ3E8B9YCW', - '01K11TFZ62YS0YYFDQ3E8B9YCX', - '01K11TFZ62YS0YYFDQ3E8B9YCY', -]; +/** + * The correlation IDs the seeded generator hands out for one kind, in draw + * order. Each kind draws from its own sequence, so a fixture indexes into the + * kind it is naming. + */ +function correlationIds(kind: CorrelationIdKind, count: number): string[] { + const generate = createCorrelationIdGenerator({ + seed: 'test', + fixedTimestamp: FIXED_TIMESTAMP, + }); + return Array.from({ length: count }, () => generate(kind)); +} + +const STEP_IDS = correlationIds('step', 4); +const WAIT_IDS = correlationIds('wait', 4); +const HOOK_IDS = correlationIds('hook', 4); async function runWithDiscontinuation( ctx: WorkflowOrchestratorContext, @@ -228,7 +236,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -236,7 +244,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'wait_created', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -244,7 +252,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -255,7 +263,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'wait_completed', - correlationId: `wait_${CORR_IDS[1]}`, + correlationId: `wait_${WAIT_IDS[0]}`, eventData: { resumeAt }, createdAt: new Date(), }, @@ -263,7 +271,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[0]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -271,7 +279,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterSleep' }, createdAt: new Date(), }, @@ -279,16 +287,16 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, ]; } - // ULID draw order in this body: `stepA()` takes CORR_IDS[0], `sleep()` - // takes CORR_IDS[1], and then whichever branch is resumed FIRST takes - // CORR_IDS[2] while the other takes CORR_IDS[3]. + // Correlation-id draw order in this body: `stepA()` takes STEP_IDS[0], `sleep()` + // takes WAIT_IDS[0], and then whichever branch is resumed FIRST takes + // STEP_IDS[1] while the other takes STEP_IDS[2]. function workflowBody(ctx: WorkflowOrchestratorContext) { const useStep = createUseStep(ctx); const sleep = createSleep(ctx); @@ -331,7 +339,7 @@ describe('step result delivery ordering across replays', () => { } // The log's ordering was reproduced: the sleep branch resumed first and - // drew CORR_IDS[2] for `afterSleep`, so both `step_created` events at + // drew STEP_IDS[1] for `afterSleep`, so both `step_created` events at // the tail matched their consumers and the run suspends with both // follow-up steps pending. expect(pendingStepNames(ctx).sort()).toEqual(['afterSleep', 'afterStep']); @@ -369,7 +377,7 @@ describe('step result delivery ordering across replays', () => { expect(error).toBeDefined(); // FAILS on `main`: the step result now wins, `afterStep` draws - // CORR_IDS[2], and replay diverges at evnt_5 with the production error + // STEP_IDS[1], and replay diverges at evnt_5 with the production error // shape ("... belongs to \"afterSleep\", but the current step consumer // is \"afterStep\""). if (!WorkflowSuspension.is(error)) { @@ -396,7 +404,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'hook_created', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', isWebhook: false }, createdAt: new Date(), }, @@ -404,7 +412,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -412,7 +420,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_2', runId: 'wrun_test', eventType: 'step_started', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA' }, createdAt: new Date(), }, @@ -422,7 +430,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_3', runId: 'wrun_test', eventType: 'hook_received', - correlationId: `hook_${CORR_IDS[0]}`, + correlationId: `hook_${HOOK_IDS[0]}`, eventData: { token: 'test-token', payload: hookPayload }, createdAt: new Date(), }, @@ -430,7 +438,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_4', runId: 'wrun_test', eventType: 'step_completed', - correlationId: `step_${CORR_IDS[1]}`, + correlationId: `step_${STEP_IDS[0]}`, eventData: { stepName: 'stepA', result: stepAResult }, createdAt: new Date(), }, @@ -438,7 +446,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_5', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[2]}`, + correlationId: `step_${STEP_IDS[1]}`, eventData: { stepName: 'afterHook' }, createdAt: new Date(), }, @@ -446,7 +454,7 @@ describe('step result delivery ordering across replays', () => { eventId: 'evnt_6', runId: 'wrun_test', eventType: 'step_created', - correlationId: `step_${CORR_IDS[3]}`, + correlationId: `step_${STEP_IDS[2]}`, eventData: { stepName: 'afterStep' }, createdAt: new Date(), }, @@ -454,8 +462,8 @@ describe('step result delivery ordering across replays', () => { } /** - * ULID draw order: `createHook()` takes CORR_IDS[0], `stepA()` takes - * CORR_IDS[1], then the branch resumed FIRST takes CORR_IDS[2]. + * Correlation-id draw order: `createHook()` takes HOOK_IDS[0], `stepA()` takes + * STEP_IDS[0], then the branch resumed FIRST takes STEP_IDS[1]. * * Two ways of consuming the hook. Both subscribe before `hook_received` is * consumed, so both take the same delivery path in `workflow/hook.ts` (the @@ -471,7 +479,7 @@ describe('step result delivery ordering across replays', () => { * settles the promise returned by `next()`, and only then does the loop * body run. Those extra hops are enough for a memo-warm `step_completed` * — which on an unfixed runtime resolves inside its own queue slot with - * no detached chain at all — to draw CORR_IDS[2] first. + * no detached chain at all — to draw STEP_IDS[1] first. * - `await hook` (control): resolution resumes the branch's continuation * directly, so it reaches `afterHook()` in the first microtask and stays * ahead of the step result even on a warm cache. @@ -566,7 +574,7 @@ describe('step result delivery ordering across replays', () => { expect(error).toBeDefined(); // FAILS on `main`: the step result overtakes the hook payload, - // `afterStep` draws CORR_IDS[2], and replay diverges at evnt_5 with the + // `afterStep` draws STEP_IDS[1], and replay diverges at evnt_5 with the // production error shape. if (!WorkflowSuspension.is(error)) { throw error; diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index 8708c02e5d..df554d5455 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -1,6 +1,5 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; @@ -32,7 +31,6 @@ function setupWorkflowContext( seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -46,10 +44,6 @@ function setupWorkflowContext( generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -68,7 +62,7 @@ async function makeStepEvents(): Promise { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step1', result: await dehydrateStepReturnValue('one', 'wrun_test', undefined), @@ -79,7 +73,7 @@ async function makeStepEvents(): Promise { eventId: 'evnt_1', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCW', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9W', eventData: { stepName: 'step2', result: await dehydrateStepReturnValue('two', 'wrun_test', undefined), @@ -162,7 +156,7 @@ describe('step hydration memoization through the step consumer', () => { eventId: 'evnt_0', runId: 'wrun_test', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'obj', result: await dehydrateStepReturnValue( diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index faf852a54b..9f61e11e68 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -6,7 +6,6 @@ import { import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; import { createCorrelationIdGenerator } from './correlation-id.js'; @@ -50,7 +49,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { if (hostRegistry) { (context.globalThis as any)[WORKFLOW_CLASS_REGISTRY] = hostRegistry; } - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -65,10 +63,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -87,7 +81,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), @@ -113,7 +107,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -167,7 +161,7 @@ describe('createUseStep', () => { 1, 2, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCV", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9V", "stepName": "add", "type": "step", }, @@ -213,7 +207,7 @@ describe('createUseStep', () => { 1, 2, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCV", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9V", "stepName": "add", "type": "step", }, @@ -222,7 +216,7 @@ describe('createUseStep', () => { 3, 4, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCW", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9W", "stepName": "add", "type": "step", }, @@ -231,7 +225,7 @@ describe('createUseStep', () => { 5, 6, ], - "correlationId": "step_01K11TFZ62YS0YYFDQ3E8B9YCX", + "correlationId": "step_01K11TFZ62CHHYKN8SS4KKNC9X", "stepName": "add", "type": "step", }, @@ -245,7 +239,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'step//input.js//my_step_function', result: await dehydrateStepReturnValue( @@ -411,7 +405,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_created', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', input: new Uint8Array(), @@ -457,7 +451,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_created', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', input: new Uint8Array(), @@ -488,7 +482,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_started', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', }, @@ -528,7 +522,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_retrying', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: new Uint8Array(), @@ -566,7 +560,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', result: await dehydrateStepReturnValue(42, 'wrun_test', undefined), @@ -597,7 +591,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', result: await dehydrateStepReturnValue(42, 'wrun_test', undefined), @@ -627,7 +621,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'subtract', error: serializedError, @@ -663,7 +657,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -704,7 +698,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -745,7 +739,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_failed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { stepName: 'add', error: serializedError, @@ -776,7 +770,7 @@ describe('createUseStep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_completed', // Wrong event type for a step! - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'step_01K11TFZ62CHHYKN8SS4KKNC9V', eventData: { resumeAt: new Date(), }, @@ -796,7 +790,7 @@ describe('createUseStep', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for step'); - expect(workflowError?.message).toContain('step_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('step_01K11TFZ62CHHYKN8SS4KKNC9V'); expect(workflowError?.message).toContain('add'); expect(workflowError?.message).toContain('wait_completed'); }); diff --git a/packages/core/src/test-support/correlation-id-scheme.ts b/packages/core/src/test-support/correlation-id-scheme.ts deleted file mode 100644 index 32a000dfdd..0000000000 --- a/packages/core/src/test-support/correlation-id-scheme.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { afterAll, beforeAll } from 'vitest'; - -/** - * Pins a test file to the run-wide shared correlation-id sequence. - * - * Replay tests that drive the real `workflowEntrypoint` against an event log - * with hardcoded correlation ids can only match under the scheme those ids were - * minted by, and the fixtures in this repo predate per-kind sequences. Files - * whose fixture ids are derived rather than written out run under whichever - * scheme `WORKFLOW_PER_KIND_CORRELATION_IDS` selects; per-kind minting itself is - * covered by `correlation-id.test.ts`. - */ -export function pinSharedCorrelationIds(): void { - let original: string | undefined; - beforeAll(() => { - original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - }); - afterAll(() => { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } - }); -} diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fdeb2bd97a..17d450b3ed 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -2,25 +2,51 @@ import { types } from 'node:util'; import { HookConflictError, WorkflowRuntimeError } from '@workflow/errors'; import type { Event, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; -import { decodeTime, monotonicFactory } from 'ulid'; +import { decodeTime } from 'ulid'; import { afterEach, assert, describe, expect, it, vi } from 'vitest'; +import { + type CorrelationIdKind, + createCorrelationIdGenerator, +} from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { runIdCreatedAt } from './runtime/run-id-time.js'; import { setWorld } from './runtime/world.js'; import { dehydrateStepReturnValue, dehydrateWorkflowArguments, hydrateWorkflowReturnValue, } from './serialization.js'; -import { pinSharedCorrelationIds } from './test-support/correlation-id-scheme.js'; -import { createContext } from './vm/index.js'; import { replayWorkflow, resumeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; -pinSharedCorrelationIds(); +/** + * The correlation IDs the seeded generator hands out for a run, per kind, in + * draw order. Each kind draws from its own sequence, so a fixture indexes into + * the kind it is naming. The seed and clock mirror what `runWorkflow` derives + * from the run. + */ +function correlationIds(runId: string) { + const generate = createCorrelationIdGenerator({ + seed: `${runId}:workflow:test-deployment`, + fixedTimestamp: + runIdCreatedAt(runId) ?? +new Date('2024-01-01T00:00:00.000Z'), + }); + const take = (kind: CorrelationIdKind, count: number) => + Array.from({ length: count }, () => generate(kind)); + return { + step: take('step', 6), + wait: take('wait', 6), + hook: take('hook', 6), + }; +} + +const IDS_WRUN_123 = correlationIds('wrun_123'); +const IDS_TEST_RUN_123 = correlationIds('test-run-123'); +const IDS_WRUN_ULID = correlationIds('wrun_01K75533W56DAE35VY3082DN3P'); describe('runWorkflow', () => { const getWorkflowTransformCode = (workflowName?: string) => @@ -180,7 +206,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -190,7 +216,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -436,7 +462,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -446,7 +472,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -509,16 +535,15 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - // Derive the IDs the run should mint, in draw order, with the same seeded - // factory runWorkflow uses: the stream draw first, then the step that - // follows it. The step's recorded event only matches if the stream draw left - // the sequence on `fixedTimestamp`. + // Derive the IDs the run should mint with the same generator runWorkflow + // uses. The stream draw is what this test is about: it carries no seed time + // of its own, so it only lands on `fixedTimestamp` if the binding routes it + // through the run's generator. const seed = `${workflowRunId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`; const fixedTimestamp = +startedAt; - const vm = createContext({ seed, fixedTimestamp }); - const ulid = monotonicFactory(() => vm.globalThis.Math.random()); - const expectedStreamId = ulid(fixedTimestamp); - const stepCorr = `step_${ulid(fixedTimestamp)}`; + const generate = createCorrelationIdGenerator({ seed, fixedTimestamp }); + const expectedStreamId = generate('stream'); + const stepCorr = `step_${generate('step')}`; const events: Event[] = [ { @@ -612,7 +637,7 @@ describe('runWorkflow', () => { eventId: 'event-step1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -622,7 +647,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -632,7 +657,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -648,7 +673,7 @@ describe('runWorkflow', () => { eventId: 'event-step2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -658,7 +683,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -668,7 +693,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -684,7 +709,7 @@ describe('runWorkflow', () => { eventId: 'event-step3-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -694,7 +719,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -704,7 +729,7 @@ describe('runWorkflow', () => { eventId: 'event-5', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -780,8 +805,8 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - const startStepId = 'step_01HK153X00SFW49DWMQP3J810S'; - const branchStepId = 'step_01HK153X00SFW49DWMQP3J810T'; + const startStepId = `step_${IDS_WRUN_123.step[0]}`; + const branchStepId = `step_${IDS_WRUN_123.step[1]}`; const events: Event[] = [ { eventId: 'event-run-created', @@ -950,7 +975,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -960,7 +985,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -970,7 +995,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -1003,7 +1028,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -1054,7 +1079,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1064,7 +1089,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1074,7 +1099,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1090,7 +1115,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1148,7 +1173,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1158,7 +1183,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1168,7 +1193,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1184,7 +1209,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1242,7 +1267,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1252,7 +1277,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -1262,7 +1287,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1278,7 +1303,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -1333,7 +1358,7 @@ describe('runWorkflow', () => { const events: Event[] = [ { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7M', + correlationId: `step_${IDS_WRUN_ULID.step[0]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K755385N02MMWXYHFCQSP9P0', eventData: { @@ -1343,7 +1368,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7N', + correlationId: `step_${IDS_WRUN_ULID.step[1]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K755386GHGAFYYDC58V17E3T', eventData: { @@ -1353,7 +1378,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7P', + correlationId: `step_${IDS_WRUN_ULID.step[2]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75538D4Q4X8PJ1ZNDZD5R0W', eventData: { @@ -1363,7 +1388,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7Q', + correlationId: `step_${IDS_WRUN_ULID.step[3]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75538Y9GEHXJQXT3JB89M4C', eventData: { @@ -1373,7 +1398,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_started', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7R', + correlationId: `step_${IDS_WRUN_ULID.step[4]}`, runId: 'wrun_01K75533W56DAE35VY3082DN3P', eventId: 'evnt_01K75539CD2PAH419SKJ2X5V5T', eventData: { @@ -1383,7 +1408,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7R', + correlationId: `step_${IDS_WRUN_ULID.step[4]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1399,7 +1424,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7Q', + correlationId: `step_${IDS_WRUN_ULID.step[3]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1415,7 +1440,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7P', + correlationId: `step_${IDS_WRUN_ULID.step[2]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1431,7 +1456,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7N', + correlationId: `step_${IDS_WRUN_ULID.step[1]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1447,7 +1472,7 @@ describe('runWorkflow', () => { }, { eventType: 'step_completed', - correlationId: 'step_01K75533W5WAVWBK9YGJQC6R7M', + correlationId: `step_${IDS_WRUN_ULID.step[0]}`, eventData: { stepName: 'promiseRaceStressTestDelayStep', result: await dehydrateStepReturnValue( @@ -1704,7 +1729,7 @@ describe('runWorkflow', () => { { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, args: [1, 2], }, ]); @@ -1735,7 +1760,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -1806,13 +1831,13 @@ describe('runWorkflow', () => { { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, args: [1, 2], }, { type: 'step', stepName: 'add', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, args: [3, 4], }, ]); @@ -2191,7 +2216,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2249,7 +2274,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'wrong-token', payload: await dehydrateStepReturnValue( @@ -2301,7 +2326,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2317,7 +2342,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2396,7 +2421,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMA', eventData: { token: 'test-token', @@ -2410,7 +2435,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMA', eventData: { token: 'test-token', @@ -2424,7 +2449,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, resumeId: '01JXAMPLE0000000000000RSMB', eventData: { token: 'test-token', @@ -2484,7 +2509,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2500,7 +2525,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2567,7 +2592,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2583,7 +2608,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2641,7 +2666,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2657,7 +2682,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2673,7 +2698,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -2683,7 +2708,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRun.runId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -2752,7 +2777,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -2768,7 +2793,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -2778,7 +2803,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -2840,7 +2865,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'my-custom-token', payload: await dehydrateStepReturnValue( @@ -2943,7 +2968,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_created', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: {}, createdAt: new Date(), }, @@ -2994,7 +3019,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'claim-only-token', conflictingRunId: 'wrun_conflicting_owner', @@ -3061,7 +3086,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'my-duplicate-token', conflictingRunId: 'wrun_conflicting', @@ -3118,7 +3143,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_conflict', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'conflicting-token', }, @@ -3933,7 +3958,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -3943,7 +3968,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -3996,7 +4021,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -4006,7 +4031,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:06.000Z'), }, @@ -4095,7 +4120,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4105,7 +4130,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4115,7 +4140,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4125,7 +4150,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4179,7 +4204,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4189,7 +4214,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[1]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4199,7 +4224,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -4250,7 +4275,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -4260,7 +4285,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -4276,7 +4301,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:03.000Z'), }, @@ -4286,7 +4311,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:03.000Z'), }, @@ -4344,7 +4369,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt, }, @@ -4354,7 +4379,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: resumeAt, }, @@ -4407,7 +4432,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4417,7 +4442,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4432,7 +4457,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'wait_completed', - correlationId: 'wait_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `wait_${IDS_TEST_RUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -4442,7 +4467,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4452,7 +4477,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('step done', ops), @@ -4500,7 +4525,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', }, @@ -4510,7 +4535,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', result: await dehydrateStepReturnValue('first done', ops), @@ -4522,7 +4547,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork1', result: await dehydrateStepReturnValue('duplicate', ops), @@ -4533,7 +4558,7 @@ describe('runWorkflow', () => { eventId: 'event-3', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'doWork2', }, @@ -4543,7 +4568,7 @@ describe('runWorkflow', () => { eventId: 'event-4', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRT', + correlationId: `step_${IDS_TEST_RUN_123.step[1]}`, eventData: { stepName: 'doWork2', result: await dehydrateStepReturnValue('second done', ops), @@ -4602,7 +4627,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4612,7 +4637,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('done', ops), @@ -4668,7 +4693,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', }, @@ -4678,7 +4703,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `step_${IDS_TEST_RUN_123.step[0]}`, eventData: { stepName: 'doWork', result: await dehydrateStepReturnValue('done', ops), @@ -4932,7 +4957,7 @@ describe('runWorkflow', () => { eventId: 'event-0', runId: workflowRun.runId, eventType: 'hook_created' as const, - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: {}, createdAt: new Date(), }, @@ -4940,7 +4965,7 @@ describe('runWorkflow', () => { eventId: 'event-1', runId: workflowRun.runId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -4956,7 +4981,7 @@ describe('runWorkflow', () => { eventId: 'event-2', runId: workflowRun.runId, eventType: 'hook_disposed', - correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + correlationId: `hook_${IDS_TEST_RUN_123.hook[0]}`, eventData: { token: 'test-token', }, @@ -5062,9 +5087,9 @@ describe('runWorkflow', () => { // Correlation IDs match the deterministic ULID generator for the seed // `${runId}:${workflowName}:${+startedAt}` - const stepA = 'step_01HK153X00SFW49DWMQP3J810S'; - const stepB = 'step_01HK153X00SFW49DWMQP3J810T'; - const stepC = 'step_01HK153X00SFW49DWMQP3J810V'; + const stepA = `step_${IDS_WRUN_123.step[0]}`; + const stepB = `step_${IDS_WRUN_123.step[1]}`; + const stepC = `step_${IDS_WRUN_123.step[2]}`; const events: Event[] = [ { @@ -5292,7 +5317,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-created', runId: workflowRunId, eventType: 'hook_created', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token' }, createdAt: new Date('2024-01-01T00:00:00.200Z'), }, @@ -5300,7 +5325,7 @@ describe('runWorkflow', () => { eventId: 'evnt-wait-created', runId: workflowRunId, eventType: 'wait_created', - correlationId: 'wait_01HK153X00SFW49DWMQP3J810T', + correlationId: `wait_${IDS_WRUN_123.wait[0]}`, eventData: { resumeAt: new Date('2024-01-02T00:00:00.000Z') }, createdAt: new Date('2024-01-01T00:00:00.300Z'), }, @@ -5308,7 +5333,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-1', runId: workflowRunId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token', payload: payload1, @@ -5319,7 +5344,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', input: payload1 }, createdAt: new Date('2024-01-01T00:00:01.100Z'), }, @@ -5327,7 +5352,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', }, @@ -5337,7 +5362,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-1-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'processPayload', result: stepResult1, @@ -5348,7 +5373,7 @@ describe('runWorkflow', () => { eventId: 'evnt-hook-2', runId: workflowRunId, eventType: 'hook_received', - correlationId: 'hook_01HK153X00SFW49DWMQP3J810S', + correlationId: `hook_${IDS_WRUN_123.hook[0]}`, eventData: { token: 'test-token', payload: payload2, @@ -5359,7 +5384,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', input: payload2 }, createdAt: new Date('2024-01-01T00:00:02.100Z'), }, @@ -5367,7 +5392,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', }, @@ -5377,7 +5402,7 @@ describe('runWorkflow', () => { eventId: 'evnt-step-2-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810W', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'processPayload', result: stepResult2, @@ -5480,7 +5505,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -5490,7 +5515,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', }, @@ -5500,7 +5525,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s1-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810S', + correlationId: `step_${IDS_WRUN_123.step[0]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -5517,7 +5542,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -5527,7 +5552,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', }, @@ -5537,7 +5562,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s2-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810T', + correlationId: `step_${IDS_WRUN_123.step[1]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -5554,7 +5579,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-created', runId: workflowRunId, eventType: 'step_created', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -5564,7 +5589,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-started', runId: workflowRunId, eventType: 'step_started', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', }, @@ -5574,7 +5599,7 @@ describe('runWorkflow', () => { eventId: 'evnt-s3-completed', runId: workflowRunId, eventType: 'step_completed', - correlationId: 'step_01HK153X00SFW49DWMQP3J810V', + correlationId: `step_${IDS_WRUN_123.step[2]}`, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( @@ -5646,15 +5671,17 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; - // Derive deterministic correlation IDs using the same seeded ULID - // factory runWorkflow uses internally, so events match what the runtime - // expects. The seed mirrors runWorkflow's `runId:workflowName:deploymentId`. + // Derive deterministic correlation IDs using the same generator runWorkflow + // uses internally, so events match what the runtime expects. The seed + // mirrors runWorkflow's `runId:workflowName:deploymentId`. const seed = `${workflowRunId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`; - const vm = createContext({ seed, fixedTimestamp: +startedAt }); - const ulid = monotonicFactory(() => vm.globalThis.Math.random()); - const hookCorr = `hook_${ulid(+startedAt)}`; - const waitCorr = `wait_${ulid(+startedAt)}`; - const stepCorr = `step_${ulid(+startedAt)}`; + const generate = createCorrelationIdGenerator({ + seed, + fixedTimestamp: +startedAt, + }); + const hookCorr = `hook_${generate('hook')}`; + const waitCorr = `wait_${generate('wait')}`; + const stepCorr = `step_${generate('step')}`; const payload1 = await dehydrateStepReturnValue( { done: false }, diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index e3499816fb..ec7f49bc1c 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -14,18 +14,18 @@ import { parseWorkflowName } from '@workflow/utils/parse-name'; import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; +import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -82,7 +82,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -109,6 +117,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -130,6 +139,12 @@ interface WorkflowSessionOptions { readonly replayPayloadCache: ReplayPayloadCache; readonly runReadyBarrier?: Promise; readonly worldCapabilities?: WorldCapabilities; + /** + * The caller's event log for this execution. Its only use here is the + * end-of-run drain, whose writes have to be ordered with the caller's + * terminal write — see {@link drainPendingQueueItems}. + */ + readonly eventLog?: MutableEventLog; } /** @@ -139,7 +154,10 @@ interface WorkflowSessionOptions { export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; - resume(events: Event[]): Promise; + resume( + events: Event[], + eventLog?: MutableEventLog + ): Promise; } /** A finished execution attempt: the workflow's output or a live boundary. */ @@ -193,7 +211,13 @@ export function replayWorkflow( /** Warm-start: advance a retained session by appending events. */ export function resumeWorkflow( session: WorkflowSession, - events: Event[] + events: Event[], + /** + * The caller's event log for this attempt. A retained session outlives the + * log it was built with, so the drain has to number from the caller's + * current one rather than the one captured at session creation. + */ + eventLog?: MutableEventLog ): Promise { return traceExecution( 'retained', @@ -203,7 +227,7 @@ export function resumeWorkflow( span?.setAttributes({ ...Attribute.WorkflowArgumentsCount(session.argumentCount), }); - const result = await session.resume(events); + const result = await session.resume(events, eventLog); return result.type === 'replay' ? result : recordResult(result, span); } ); @@ -269,7 +293,12 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain — see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { const result = await replayWorkflow({ workflowCode, @@ -279,6 +308,7 @@ export async function runWorkflow( replayPayloadCache, runReadyBarrier, worldCapabilities, + eventLog, }); if (result.type === 'suspended') throw result.suspension; return result.output; @@ -292,6 +322,7 @@ async function createWorkflowSession({ replayPayloadCache, runReadyBarrier, worldCapabilities, + eventLog, }: WorkflowSessionOptions): Promise<{ session: WorkflowSession; execution: Promise; @@ -335,6 +366,8 @@ async function createWorkflowSession({ fixedTimestamp, }); + let currentEventLog = eventLog; + const initialInterruption = withResolvers(); let state: WorkflowSessionState = { type: 'running', @@ -369,14 +402,11 @@ async function createWorkflowSession({ state satisfies never; }; - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); const generateCorrelationId = createCorrelationIdGenerator({ seed, - fixedTimestamp, // Correlation IDs must be replay-stable. `startedAt` differs between a // turbo delivery and a later server-backed replay, so use fixedTimestamp. - positional: () => ulid(fixedTimestamp), - perKind: isPerKindCorrelationIdsEnabled(), + fixedTimestamp, }); const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) @@ -387,22 +417,34 @@ async function createWorkflowSession({ // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; onWorkflowError( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, @@ -486,11 +528,10 @@ async function createWorkflowSession({ // @ts-expect-error - `@types/node` says symbol is not valid, but it does work vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; // Serialization mints stream ids through this symbol, and calls it with no - // seed time. `monotonicFactory` returns `encodeTime(lastTime)` on its - // increment branch, so one such call latches the *host* wall clock into - // `lastTime` and every id the run mints afterwards carries that timestamp - // instead of `fixedTimestamp` — a value that differs on every replay. - // Binding the seed time here keeps the whole run on one replay-stable clock. + // seed time of its own. Routing it through the run's generator keeps those + // ids on `fixedTimestamp` like every other id the run mints, and puts them + // in their own sequence so serializing a stream does not renumber the + // entities created after it. // @ts-expect-error - `@types/node` says symbol is not valid, but it does work vmGlobalThis[STABLE_ULID] = () => generateCorrelationId('stream'); @@ -1043,7 +1084,8 @@ async function createWorkflowSession({ vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + currentEventLog ); throw error; @@ -1081,7 +1123,8 @@ async function createWorkflowSession({ vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + currentEventLog ); return { type: 'completed', output, resultType: typeof result }; @@ -1093,7 +1136,12 @@ async function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - async resume(nextEvents) { + async resume(nextEvents, nextEventLog) { + // A retained session outlives the log its replay was numbered from, so + // every attempt's drain numbers from the caller's current one. + if (nextEventLog) { + currentEventLog = nextEventLog; + } switch (state.type) { case 'suspended': { // The full O(known events) prefix compare is required: the runtime diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index fbc2c8063e..25bae24caf 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -6,7 +6,6 @@ import { import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { aliasSerializationClass, @@ -24,7 +23,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -34,7 +36,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { // mirror that here so `hook.getConflict()` can construct the // conflicting run through the registry. aliasSerializationClass(RUN_CLASS_ID, Run, context.globalThis); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); return { runId: 'wrun_test', @@ -43,17 +44,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -72,7 +69,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -98,7 +95,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', }, @@ -126,7 +123,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', payload: await dehydrateStepReturnValue( @@ -160,7 +157,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', }, @@ -187,7 +184,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'wrong-token', conflictingRunId: 'wrun_conflicting', @@ -233,7 +230,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type for a hook! - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -254,7 +251,7 @@ describe('createCreateHook', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for hook'); - expect(workflowError?.message).toContain('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(workflowError?.message).toContain('step_completed'); }); @@ -265,7 +262,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -275,7 +272,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -312,7 +309,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: {}, createdAt: new Date(), }, @@ -360,7 +357,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', conflictingRunId: 'wrun_conflicting_owner', @@ -392,7 +389,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -415,7 +412,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: {}, createdAt: new Date(), }, @@ -423,7 +420,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { payload: await dehydrateStepReturnValue( { data: 'after-ready' }, @@ -449,7 +446,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -474,6 +471,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ @@ -481,7 +537,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -491,7 +547,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -507,7 +563,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -523,7 +579,7 @@ describe('createCreateHook', () => { eventId: 'evnt_3', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -552,7 +608,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -582,7 +638,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', conflictingRunId: 'wrun_conflicting', @@ -614,7 +670,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -640,7 +696,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -650,7 +706,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -666,7 +722,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -733,7 +789,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -751,7 +807,7 @@ describe('createCreateHook', () => { // Wait for events to process (hook_created sets hasCreatedEvent on queue item) await vi.waitFor(() => { - const item = ctx.invocationsQueue.get('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + const item = ctx.invocationsQueue.get('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(item?.type === 'hook' && item.hasCreatedEvent).toBe(true); }); @@ -759,7 +815,7 @@ describe('createCreateHook', () => { hook.dispose(); const queueItem = ctx.invocationsQueue.get( - 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'hook_01K11TFZ62C752Z96G9MRSN85J' ); expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { @@ -775,7 +831,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -785,7 +841,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -801,7 +857,7 @@ describe('createCreateHook', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -817,7 +873,7 @@ describe('createCreateHook', () => { eventId: 'evnt_3', runId: 'wrun_123', eventType: 'hook_disposed', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -853,7 +909,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -884,7 +940,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -987,7 +1043,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_conflict', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'my-conflicting-token', }, @@ -1050,7 +1106,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -1060,7 +1116,7 @@ describe('createCreateHook', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'hook_received', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', payload: await dehydrateStepReturnValue( @@ -1099,7 +1155,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData: { token: 'test-token', }, @@ -1115,7 +1171,7 @@ describe('createCreateHook', () => { // Wait for events to process (hook_created sets hasCreatedEvent on queue item) await vi.waitFor(() => { - const item = ctx.invocationsQueue.get('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + const item = ctx.invocationsQueue.get('hook_01K11TFZ62C752Z96G9MRSN85J'); expect(item?.type === 'hook' && item.hasCreatedEvent).toBe(true); }); @@ -1235,7 +1291,7 @@ describe('createCreateHook', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_created', - correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'hook_01K11TFZ62C752Z96G9MRSN85J', eventData, createdAt: new Date(), }, diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e95998b858..80e59aeb61 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -175,8 +175,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -192,6 +193,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -457,8 +483,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -578,8 +606,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index 5b2d2e6edf..816b2b9ec8 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -2,7 +2,6 @@ import { ReplayDivergenceError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; import { createCorrelationIdGenerator } from '../correlation-id.js'; import { EventsConsumer } from '../events-consumer.js'; @@ -18,7 +17,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { seed: 'test', fixedTimestamp: 1753481739458, }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const ctx: WorkflowOrchestratorContext = { suspensionGeneration: 0, @@ -42,10 +40,6 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { generateCorrelationId: createCorrelationIdGenerator({ seed: 'test', fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) @@ -65,7 +59,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -75,7 +69,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -96,7 +90,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -106,7 +100,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', createdAt: new Date(), }, ]); @@ -124,7 +118,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -134,7 +128,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:02.000Z'), }, @@ -152,7 +146,7 @@ describe('createSleep', () => { expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('wait_completed'); expect(workflowError?.message).toContain('resumeAt'); - expect(workflowError?.message).toContain('wait_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('wait_01K11TFZ62FEAJPFZ0JMCV2A5V'); }); it('should invoke workflow error handler when wait_completed resumeAt is invalid', async () => { @@ -161,7 +155,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -171,7 +165,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date(Number.NaN), }, @@ -214,7 +208,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'step_completed', // Wrong event type for a wait! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { stepName: 'unexpectedStep', result: ['test'], @@ -234,7 +228,7 @@ describe('createSleep', () => { const workflowError = await errorReceived.promise; expect(workflowError).toBeInstanceOf(ReplayDivergenceError); expect(workflowError?.message).toContain('Unexpected event type for wait'); - expect(workflowError?.message).toContain('wait_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('wait_01K11TFZ62FEAJPFZ0JMCV2A5V'); expect(workflowError?.message).toContain('step_completed'); }); @@ -244,7 +238,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -264,7 +258,7 @@ describe('createSleep', () => { // Check that the wait item has been updated with hasCreatedEvent const waitItem = ctx.invocationsQueue.get( - 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V' ); expect(waitItem).toBeDefined(); expect(waitItem?.type).toBe('wait'); @@ -283,7 +277,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'hook_received', // Wrong event type for a wait! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { token: 'test-token', payload: { data: 'test' }, @@ -310,7 +304,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:05.000Z'), }, @@ -329,7 +323,7 @@ describe('createSleep', () => { // Queue item should still exist (wait_created is not terminal) expect(ctx.invocationsQueue.size).toBe(1); const waitItem = ctx.invocationsQueue.get( - 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V' ); expect(waitItem).toBeDefined(); expect(waitItem?.type).toBe('wait'); @@ -344,7 +338,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -354,7 +348,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -384,7 +378,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -394,7 +388,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -404,7 +398,7 @@ describe('createSleep', () => { eventId: 'evnt_2', runId: 'wrun_123', eventType: 'wait_completed', // Duplicate! - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt: new Date('2024-01-01T00:00:01.000Z'), }, @@ -431,7 +425,7 @@ describe('createSleep', () => { eventId: 'evnt_0', runId: 'wrun_123', eventType: 'wait_created', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt, }, @@ -441,7 +435,7 @@ describe('createSleep', () => { eventId: 'evnt_1', runId: 'wrun_123', eventType: 'wait_completed', - correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + correlationId: 'wait_01K11TFZ62FEAJPFZ0JMCV2A5V', eventData: { resumeAt, }, diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index e552e8e28a..35467bbcce 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -821,6 +821,58 @@ export class PreconditionFailedError extends WorkflowWorldError { } } +/** + * Thrown when the backend rejects an event creation because the event slot the + * client named was already taken by another writer (HTTP 409). + * + * Runs name their events by position in the log, and whoever writes a position + * first owns it. The loser has by definition been replaying against an event log + * missing at least one event, so retrying the same write can never succeed: the + * client has to merge the events it was missing, replay, and propose whatever + * slot that replay lands on. The rejection carries those events inline so the + * common case costs no extra round-trip. + * + * `PreconditionFailedError` (412) is the equivalent rejection for a run whose + * events are guarded by the `stateUpdatedAt` watermark. + * + * The workflow runtime handles this automatically. Users interacting with world + * storage backends directly may encounter it. + * + * @property eventId - The event id, naming a slot, that was already taken. + * @property events - The events recorded after the client's snapshot, in + * ascending slot order. Empty when the backend could not read them, in which + * case the client reloads the log itself. + * @property cursor - Cursor to continue the delta from, or `null`. + * @property hasMore - Whether events beyond `events` remain to be fetched. + */ +export class SlotConflictError extends WorkflowWorldError { + readonly eventId: string; + readonly events: unknown[]; + readonly cursor: string | null; + readonly hasMore: boolean; + + constructor( + message: string, + options: { + eventId: string; + events?: unknown[]; + cursor?: string | null; + hasMore?: boolean; + } + ) { + super(message, { status: 409 }); + this.name = 'SlotConflictError'; + this.eventId = options.eventId; + this.events = options.events ?? []; + this.cursor = options.cursor ?? null; + this.hasMore = options.hasMore ?? false; + } + + static is(value: unknown): value is SlotConflictError { + return isError(value) && value.name === 'SlotConflictError'; + } +} + /** * Thrown when awaiting `run.returnValue` on a workflow run that was cancelled. * diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 4490e7ed4f..9e0cdb3774 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -5,6 +5,7 @@ export { PreconditionFailedError, RunExpiredError, RunNotSupportedError, + SlotConflictError, StepNotRegisteredError, ThrottleError, TooEarlyError, diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 16f740c4f1..17b2ec2089 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -466,6 +466,11 @@ export async function deleteJSON(filePath: string): Promise { await withWindowsRetry(() => fs.unlink(filePath)); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } @@ -580,6 +585,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * An event log orders by slot — the position is the order — and a writer that + * loses a slot re-proposes above the winner while keeping the stamp it started + * with, so `createdAt` there disagrees with the log. Such an item reports one + * shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -616,6 +629,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -720,7 +734,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -748,8 +762,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -770,7 +784,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index dfa56584df..28aa5fa908 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,10 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + reenqueueActiveRuns, + SPEC_VERSION_SLOT_IDENTITY, +} from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +75,9 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity. Every world reads + // both schemes whatever this says. + specVersion: SPEC_VERSION_SLOT_IDENTITY, capabilities: { hookRetention: { active: true }, // world-local deduplicates concurrent `hook_received` writes sharing a diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 9524ff31ad..9a809bd2ed 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -6,11 +6,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -35,9 +37,15 @@ import { isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, + SLOT_RETRY_BUDGET_MS, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, + slotRetryDelay, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -92,6 +100,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -138,6 +147,19 @@ function getHookRetentionLimitMs(): number { // being module-level. Claim files remain the durable ownership record; // cross-process Hook token handoffs use `withHookTokenClaimLock`. +/** + * The time an event orders and paginates by. Slot order is the log's order — + * the position *is* the order, the way the sort key is for the other backends — + * so every slot-named event reports the same time and lets the event-id + * tie-break do the ordering. Ordering those by `createdAt` reads the log in an + * order no replay produced: a writer that loses a slot re-proposes above the + * winner while keeping the stamp it started with, and a caller that reserves + * slots for a whole flush commits them in whatever order the network returns. A + * ULID-numbered run keeps its wall-clock order, which its ids agree with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -175,6 +197,20 @@ const HookResumeClaimSchema = z.object({ eventId: z.string(), payloadDigest: z.string().optional(), }); + +/** + * How long a writer that adopted a claim's pinned event id waits for that event + * to become reader-visible. + * + * It is waiting on the other writer of the same resume, which holds the position + * staged under `.locks` while it re-checks the run's terminal marker and links + * the file into `events/`: two filesystem operations, so the window is short and + * the wait almost always ends on the first look. The bound exists for the one + * case that never ends — a writer that died between staging and promoting, whose + * staged file no one will ever promote or clean up. + */ +const PINNED_EVENT_WAIT_MS = 2_000; + /** * Whether a token claim held by another `(runId, hookId)` can never become * live again and may therefore be released by a new claimant: @@ -262,11 +298,40 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; } +/** + * The run's committed `hook_received` for one resume, if it has one. + * + * The resume claim records where the event is going to be published, and that + * is only a hint: another instance can allocate the same + * position for an unrelated event from its own book, and then the claim points + * at a stranger. The key the event itself carries is the durable identity, so + * this scan is what decides whether a resume has already been recorded. + */ +async function findCommittedResumeEvent( + basedir: string, + runId: string, + resumeId: string +): Promise { + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + filePrefix: `${runId}-`, + filter: (event) => + event.eventType === 'hook_received' && event.resumeId === resumeId, + limit: 1, + getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, + getId: (event) => event.eventId, + }); + return result.data[0] ?? null; +} + /** * Repair an "event-first orphan": the hook entity write is deferred * until after the `hook_created` event publish commits (so a failed @@ -489,6 +554,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -508,6 +575,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -515,6 +583,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -575,6 +644,65 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -638,6 +766,72 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A create reserves its position before running the validation and + // materialization that may still reject it. Handing the + // reservation back on the way out — whether the create throws or returns + // another writer's event — is what keeps the log dense: an abandoned slot + // below a sibling's published one is a hole that can never be filled, and + // a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ + let eventCommitted = false; + /** + * Hands the slots of a create that never published back to the allocator. + * + * The allocator recycles a returned position only while it is still the + * top of the book, which is the case that keeps a run dense: a create that + * takes the next position and then publishes nothing would otherwise leave + * the log one short forever. A position that already sits below a + * sibling's published event stays empty by design — reusing it there would + * put a later event below an earlier one. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } + throw error; + } finally { + // Every reservation this create still holds goes back, including on + // the paths that succeed without publishing anything: a create that + // converged on another writer's event returns a result and leaves its + // own position untouched, and keeping it reserved would stop the next + // event of the run from taking it. Handing back a position that *was* + // published is a no-op — the publish recorded it as written, so it + // cannot be handed out again. + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -648,7 +842,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -676,9 +872,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, createImpl); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -687,6 +885,22 @@ export function createEventsStorage( // so concurrent / cross-process workers converge on a single // event in the log. let eventId = `evnt_${monotonicUlid()}`; + // Set when a resume claim names `eventId` — either this write reserved + // the claim, or it adopted the position another writer of the same resume + // pinned. Losing the publish at a pinned position is resolved against + // that other writer instead of by moving to a free position, which would + // give one resume two events. + let pinnedEventId: string | undefined; + // Whether this write is the one that reserved the resume claim. The + // claim's owner is the only writer allowed to move off the pinned + // position while the resume is still unrecorded. + let ownsResumeClaim = false; + // Whether the position this write ends up publishing to was chosen by + // this world for idempotency rather than claimed by the caller. A hook + // retry adopts the canonical `hook_created` position recorded in the + // token claim, so losing that position says the hook is already + // created, not that the caller's log is short an event. + let convergedOnCanonicalPosition = false; const now = new Date(); // For run_created events, use client-provided runId or generate one server-side @@ -713,6 +927,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -794,8 +1020,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -814,6 +1046,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -830,6 +1063,18 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and a ULID names no position: the replay + // would read it wherever its timestamp happens to sort rather than where + // the writer meant it to go. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -851,7 +1096,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -867,20 +1112,109 @@ export function createEventsStorage( } } - // ============================================================ - // VALIDATION: Terminal state and event ordering checks - // ============================================================ - // Lazy step start: a step_started carrying step-creation data // (stepName + input) is allowed to arrive with no prior step_created // — it creates the step on the fly (see the materialization block // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. const createsChildEntity = isChildEntityCreationEvent(data); const lazyStepStart = createsChildEntity && data.eventType === 'step_started'; + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so it takes that position outright instead of + // allocating one, and every other event allocates above it even when it + // is the first to arrive here. Allocation is append-only (see SlotBook), + // so `run_created` cannot get the first position by asking for the lowest + // free one: a `run_started` racing it (start() issues the creation and + // the queue send in parallel) may already have moved the book past it. + const ownsFirstSlot = data.eventType === 'run_created'; + // An event id names a position in the run's log, so it is either claimed + // by a caller that holds the log (and is therefore asserting the log is + // complete up to that position) or allocated here for a caller that has + // no log — a step completion reporting in, a cancellation from an API + // call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // A claim has to clear the log's tail, not merely be free: a + // position left unwritten by an abandoned reservation stays empty + // for good, and a caller numbering from a stale snapshot aims + // straight at it, landing an event below events another replay + // already consumed. Rejected before the materialization below + // creates the step, hook or wait this event will now never + // accompany. See SlotBook.highestWritten. + const tail = await slots.highestWritten(effectiveRunId); + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (slot <= tail) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + let slot: number; + if (ownsFirstSlot) { + slot = RUN_CREATED_SLOT; + slots.claim(effectiveRunId, slot); + } else { + slot = await slots.reserve(effectiveRunId); + } + reserved.add(slot); + eventId = slotEventId(slot); + } + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1077,10 +1411,11 @@ export function createEventsStorage( // converges too. Gated on `resumeId` so the historical single-write // path is untouched. if (data.eventType === 'hook_received' && params?.resumeId) { + const resumeId = params.resumeId; const claimPath = hookResumeClaimPath( basedir, effectiveRunId, - params.resumeId + resumeId ); const converge = async ( claim: z.infer @@ -1109,20 +1444,35 @@ export function createEventsStorage( `hook_received resumeId "${params.resumeId}" already recorded with a different payload` ); } - const existing = await readJSONWithFallback( + const pinned = await readJSONWithFallback( basedir, 'events', `${effectiveRunId}-${claim.eventId}`, EventSchema, tag ); - if (existing) { - return { event: existing }; + // The event the claim points at answers for this resume only if it + // is this resume's. The pinned position can hold an unrelated + // event another instance allocated it for, and returning that + // would report a step's event as the resume's. + if (pinned?.resumeId === resumeId) { + return { event: pinned }; + } + const committed = await findCommittedResumeEvent( + basedir, + effectiveRunId, + resumeId + ); + if (committed) { + return { event: committed }; } - // Claim exists but its event is not yet visible (a crash between - // the claim write and the append). Adopt the pinned eventId and - // fall through to (re)write the event idempotently at that path. + // The resume has no event yet: the pinning writer is mid-publish + // (its event is staged, not yet linked into `events/`), or it + // crashed between the claim write and the append. Adopt the pinned + // position and fall through to (re)write the event there, so the + // two writers of this resume still contend for one position. eventId = claim.eventId; + pinnedEventId = claim.eventId; return null; }; @@ -1151,7 +1501,15 @@ export function createEventsStorage( : {}), } satisfies z.infer) ); - if (!won) { + if (won) { + // The claim now names this position for this resume, and the + // other writer is on its way to adopting it. Losing the publish + // here therefore has to be resolved against that writer rather + // than by moving to a free position, which would give one resume + // two events. + pinnedEventId = eventId; + ownsResumeClaim = true; + } else { const winner = await readJSON(claimPath, HookResumeClaimSchema); if (winner) { const converged = await converge(winner); @@ -1265,13 +1623,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1554,6 +1939,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1575,10 +1961,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -1617,6 +2007,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1632,24 +2023,35 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1662,15 +2064,39 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -1964,10 +2390,26 @@ export function createEventsStorage( canonicalEventId = pinned; } - // The canonical ULID also makes converging writes byte-identical. + if (canonicalEventId !== eventId) { + // Moving to the canonical position leaves the position this + // write was going to take unused. Hand it back so the allocator + // can fill it rather than leaving a hole the log can never close. + const abandoned = slotFromId(eventId); + if (abandoned !== undefined && reserved.delete(abandoned)) { + slots.release(effectiveRunId, abandoned); + } + } + convergedOnCanonicalPosition = true; + + // A canonical ULID also makes converging writes byte-identical. + // A slot id carries no time, and reading one as a ULID yields the + // epoch, so a slot-numbered run keeps the wall clock and the two + // writers converge on one event through the position instead of + // through the bytes. eventId = canonicalEventId; - const canonicalCreatedAt = - ulidToDate(eventId.replace(/^evnt_/, '')) ?? now; + const canonicalCreatedAt = slotMode + ? now + : (ulidToDate(eventId.replace(/^evnt_/, '')) ?? now); event = { ...data, // The first claim fixes the retention deadline even when its @@ -2136,6 +2578,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2149,10 +2592,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2214,11 +2661,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2234,115 +2687,244 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + // `run_created` is excluded even though it named its own position: the + // first slot is the only position it can ever occupy, so losing it means + // the run already has a creation event, and appending a second one above + // it would be worse than the duplicate the publish is reporting. + const reallocatesSlot = + slotMode && + params?.eventId === undefined && + !ownsFirstSlot && + // A write that adopted the canonical position of an existing hook is + // pinned to it for the same reason `run_created` is pinned to the + // first slot: moving somewhere free would publish a second + // `hook_created` for one hook. + !convergedOnCanonicalPosition; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + const pinnedDeadline = Date.now() + PINNED_EVENT_WAIT_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (pinnedEventId === eventId && params?.resumeId) { + // This resume's claim names this position and something else holds + // it. Taking a free position on the strength of that alone would + // publish a second event for one resume — the duplicate the claim + // exists to prevent, and one the log can never be rid of. So find out + // who holds it first. + const occupant = await readJSONWithFallback( + basedir, + 'events', + compositeKey, + EventSchema, + tag + ); + if (occupant?.resumeId === params.resumeId) { + // The other writer of this same resume got there first; its event + // is this write's answer. + return { event: occupant }; + } + const committed = await findCommittedResumeEvent( + basedir, + effectiveRunId, + params.resumeId + ); + if (committed) { + // The other writer published elsewhere: it reallocated around a + // stranger sitting on the pinned position. + return { event: committed }; + } + // Nobody has recorded this resume yet, so both of its writers are + // still live candidates and only one of them may leave the pinned + // position: if both moved, they could publish two events for one + // resume. That one is the writer that owns the claim, and only once + // it can see a stranger on the position — an unrelated event another + // instance allocated it for, which will never become this resume's. + // Anyone else waits for the owner's event to show up anywhere in the + // log, which the scan above will find. + const strangerHoldsPosition = occupant !== null; + if ( + !(ownsResumeClaim && strangerHoldsPosition) && + Date.now() < pinnedDeadline + ) { + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + continue; + } + // Out of patience. Placing the resume's event somewhere beats leaving + // it unrecorded: an owner that died between staging and promoting, or + // between the claim and the append, is never coming back, and a resume + // with no event hangs the run for good. The claim keeps pointing where + // it always did, which costs nothing — a recorded resume is found by + // the scan above, keyed on the resume itself. + if (!reallocatesSlot) { + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` + ); + } + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again above whatever the log has reached rather than surfacing a + // conflict the caller cannot act on. Re-reading rather than + // incrementing bounds the search: every round at least one writer + // wins, so the top of the log is never further than the number of + // writers still contending for it. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2363,14 +2945,54 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + if (slotMode && !convergedOnCanonicalPosition) { + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` @@ -2408,6 +3030,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2456,6 +3079,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2516,6 +3140,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2548,6 +3173,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 605e8b3370..4d18306042 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; import { WorkflowWorldError } from '@workflow/errors'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { lock } from 'proper-lockfile'; import { decodeTime, monotonicFactory } from 'ulid'; import { z } from 'zod'; @@ -213,6 +214,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -234,38 +275,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * In slot mode the event takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -284,6 +322,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..2c13904ed0 --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,639 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { EntityConflictError, SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('rejects a claim on a free position below the log’s tail', async () => { + // The undercut that a "is the position free?" check cannot catch. A + // position claimed by a write that then failed is never filled, so a log + // carries holes below its tail — and a caller numbering from a snapshot + // that predates the events above one of those holes aims straight at it. + // Let it land and the event sits below events another replay has already + // consumed: the log stays internally consistent while its order silently + // changes, which is enough to flip a race between a step and a sleep from + // one replay to the next. A claim asserts a complete log, so a claim that + // does not clear the tail is a conflict, exactly as a taken one is. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await expect( + createStep(runId, 'step_early', slotEventId(2)) + ).rejects.toThrow(SlotConflictError); + // The hole stays a hole, and the log stays in slot order. + await expect(slotsOf(runId)).resolves.toEqual([1, 3]); + }); + + it('accepts the claim immediately above a tail with a hole below it', async () => { + // The fence rejects at-or-below, so the first position above the tail has + // to stay writable — otherwise every write following a hole would conflict + // forever and the run could never make progress again. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + expect(await createStep(runId, 'step_next', slotEventId(4))).toBe( + slotEventId(4) + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 3, 4]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('numbers a run densely when every write it makes lands', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('reuses a rejected write’s position when nothing published above it', async () => { + // The rejected op took a position and gave it back with nothing above it, so + // the next write lands there and the log stays dense. Below a published + // event the position stays a hole instead: see the SlotBook's own tests, + // where the ordering can be forced. + const runId = await newSlotRun(); + await expect( + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }) + ).rejects.toThrow(); + await createStep(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT, FIRST_SLOT + 1]); + }); +}); + +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts when another instance takes a claimed slot', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. + const runId = await newSlotRun(); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); +}); + +async function createHook(runId: string, hookId: string, token: string) { + await storage.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: hookId, + eventData: { token, hookId }, + }); + return { hookId, token }; +} + +/** + * One writer of a resume. A resume has two of these — `resumeHook`'s direct + * write and the queue consumer's re-ensure — and they are byte-for-byte + * identical, which is what makes them converge on one event. + */ +function resume( + from: Storage, + runId: string, + hook: { hookId: string; token: string }, + resumeId: string, + payload = new Uint8Array([1, 2, 3]) +) { + return from.events.create( + runId, + { + eventType: 'hook_received', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: hook.hookId, + eventData: { token: hook.token, payload }, + }, + { resumeId, resumePayloadDigest: `digest_${resumeId}` } + ); +} + +async function hookReceivedCount(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.filter((event) => event.eventType === 'hook_received').length; +} + +describe('a resume with two writers', () => { + it('leaves no hole behind a converged redelivery', async () => { + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + + const first = await resume(storage, runId, hook, 'resume_1'); + const second = await resume(storage, runId, hook, 'resume_1'); + expect(second.event.eventId).toBe(first.event.eventId); + + // The redelivery took a position to write at and then converged on the + // event that already existed, so it published nothing. Unless it hands that + // position back, the next event lands above a hole and the run can no longer + // prove its log is complete. + await resume(storage, runId, hook, 'resume_2'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + + it('commits one event when both writers race across instances', async () => { + // The two writers of a resume are usually two processes: the request that + // resumed the hook and the queue consumer that replays the run. They share + // the data directory and nothing else, so the claim on disk is the only + // thing that can converge them. + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + const other = createStorage(testDir); + + const [a, b] = await Promise.all([ + resume(storage, runId, hook, 'resume_1'), + resume(other, runId, hook, 'resume_1'), + ]); + + expect(b.event.eventId).toBe(a.event.eventId); + await expect(hookReceivedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('numbers distinct resumes of one hook densely under load', async () => { + const runId = await newSlotRun(); + const hook = await createHook(runId, 'hook_a', 'tok:1'); + const other = createStorage(testDir); + const resumeIds = ['r_0', 'r_1', 'r_2', 'r_3', 'r_4']; + + const results = await Promise.all( + resumeIds.flatMap((resumeId, index) => { + const payload = new Uint8Array([index]); + return [ + resume(storage, runId, hook, resumeId, payload), + resume(other, runId, hook, resumeId, payload), + ]; + }) + ); + + // Each resume's pair shares one event; the five resumes are distinct and sit + // in the five positions above the hook. + expect(new Set(results.map((r) => r.event.eventId)).size).toBe( + resumeIds.length + ); + await expect(hookReceivedCount(runId)).resolves.toBe(resumeIds.length); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5, 6, 7]); + }); +}); + +async function hookCreatedCount(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.filter((event) => event.eventType === 'hook_created').length; +} + +describe('a hook created twice', () => { + it('commits one event when the retry allocated its own position', async () => { + // Two replays of the same run both reach `hook.create`. Neither holds an + // event log, so each takes a position of its own choosing; the token claim + // then points the second at the first's event. Losing that position says + // the hook exists, so the retry must stop there. Reallocating around it — + // which is what a writer that picked its own position normally does — would + // publish a second `hook_created` for one hook. + const runId = await newSlotRun(); + const hook = { hookId: 'hook_a', token: 'tok:1' }; + + await createHook(runId, hook.hookId, hook.token); + await expect(createHook(runId, hook.hookId, hook.token)).rejects.toThrow( + EntityConflictError + ); + + await expect(hookCreatedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('leaves no hole behind the retry it turned away', async () => { + // The retry reserved a position before the claim redirected it. Handing it + // back is what keeps the run's log dense, and a dense log is the only thing + // that lets a client prove it holds every event. + const runId = await newSlotRun(); + await createHook(runId, 'hook_a', 'tok:1'); + await createHook(runId, 'hook_a', 'tok:1').catch(() => {}); + + await createHook(runId, 'hook_b', 'tok:2'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('commits one event when both replays race across instances', async () => { + const runId = await newSlotRun(); + const other = createStorage(testDir); + + const outcomes = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'hook_a', + eventData: { token: 'tok:1', hookId: 'hook_a' }, + }), + other.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'hook_a', + eventData: { token: 'tok:1', hookId: 'hook_a' }, + }), + ]); + + expect( + outcomes.filter((outcome) => outcome.status === 'fulfilled') + ).toHaveLength(1); + await expect(hookCreatedCount(runId)).resolves.toBe(1); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..c7ade143a9 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,322 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + FIRST_SLOT, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts above the slot the run’s own creation event owns', async () => { + // `run_created` takes the first slot outright — nothing can precede it — so + // an allocation never hands that position out. + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('leaves a hole in the persisted log unfilled', async () => { + // Allocation is append-only: the free position sits below a published event, + // and an event placed there would order before one that already happened. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('stays above published events when a lower position comes free', async () => { + // The corruption this rules out: a step completion allocating late, dropping + // into a hole, and landing below the step_started it reports on. Replay reads + // the log in slot order and cannot consume that. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + const published = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(published)); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBeGreaterThan(published); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; a book that only moved + // on publish would give them all the same position and fail all but one. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => RUN_CREATED_SLOT + 1 + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('clears events another instance published after this book was read', async () => { + // The inversion this rules out, and the reason every allocation re-reads the + // log: a second storage instance over the same data directory publishes + // without touching this book, so a book that trusted itself would hand out a + // position below events that already exist. The publish cannot catch it — + // the position is genuinely free — and the replay that reads a completion + // before its own start diverges for good. + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3, 4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('honours a floor above where the book has reached', async () => { + // start() publishes the run entity before its `run_created` event and issues + // the queue send in parallel, so the delivery's `run_started` can allocate + // while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, 5)).resolves.toBe(5); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); +}); + +describe('release', () => { + it('does not recycle an abandoned interior slot', async () => { + // The position may already sit below a sibling that published, and no + // caller can tell from here. A hole is a position nothing ever wrote; an + // inversion is an event a replay reads before the one it followed. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('recycles the top slot, so an abandoned write leaves no hole', async () => { + // Nothing was handed out above it and nothing on disk sits above it, so the + // position provably precedes no event and can be handed out again. The common + // abandonment is a create that converged on another writer's event before + // publishing anything, and leaving those behind would make a log that lost + // nothing unable to prove it. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBe(abandoned); + }); + + it('does not recycle the top slot once the log has moved above it', async () => { + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + await writeEvents(1, abandoned, abandoned + 1); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBe(abandoned + 2); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('highestWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(2); + }); + + it('reports the tail of a log with a hole below it', async () => { + // The case the tail exists to catch: slot 3 was claimed by a write that + // never published, so it is free — but a claim on it would land below the + // event at 4 that a replay may already have consumed. + await writeEvents(1, 2, 4); + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(4); + }); + + it('is below the first slot for a run with no events', async () => { + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); + }); + + it('ignores a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); + }); +}); + +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('stops holding back a claim that resolved', async () => { + // Released before anything allocated for the run, so no position in this + // instance was ever handed out above it and the log — which is the authority + // on what published — reaches only slot 1. Nothing can be inverted by + // seeding the book from disk alone. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + +describe('observe', () => { + it('moves allocation above a position the client claimed', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(RUN_CREATED_SLOT + 1); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..8e4a777307 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,381 @@ +/** + * Slot allocation for the Local World. + * + * Runs name their events by position: `evnt_…001` is the first event of the run, + * `evnt_…002` the second. A replay reads the log in slot + * order, so the order slots are handed out in has to be an order some execution + * could have produced — which makes allocation strictly *append-only*: a slot is + * only ever handed out above every position this book has seen. + * + * Filling a hole is what that rules out, and it is worth naming why, because the + * alternative looks appealing (it keeps the log dense). A position left unwritten + * by an abandoned reservation can sit below events that are already published. + * Hand it to the next caller and a `step_completed` lands below its own + * `step_started`; the replay reaches a completion for a step it has not started + * and diverges, and every later replay diverges the same way. A hole costs a + * reader the ability to prove its copy of the log is complete. An inversion + * costs the run. + * + * An abandoned reservation is recycled in the one case where it provably sits + * below nothing: it was the top of the book when it was released, so no sibling + * was ever handed a position above it and none was read from disk above it. That + * keeps the common abandonment (a create that converged on another writer's event + * before publishing anything) from leaving the log permanently un-provable. + * + * Three properties do the work: + * + * - An allocation reads the log's tail from disk and picks the position above + * both it and the highest one the book knows of, written or outstanding. + * Reading disk every time is what makes the order safe across storage + * instances: a book that has not seen another instance's writes would + * otherwise hand out a position *below* them, and that inversion is silent + * because the position is genuinely free (see below for what it costs). + * - Once the tail is in hand, handing out a slot is a *synchronous* set + * operation, so concurrent callers in one process get distinct slots with no + * lock: each bumps the ceiling before the next one reads it. + * - The event publish is `writeExclusive`, which is the authority. The tail + * read is a hint: when it turns out to be stale (another process wrote the + * slot between the read and the publish), the publish fails and the caller is + * told so, rather than a duplicate being written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** + * The highest position this book has ever seen written, claimed or handed out. + * Allocation goes above it and it never descends, which is what keeps a + * released position from being recycled below events already published. + */ + ceiling: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet answers `false`, + * and that answer is not cached: the resilient-start path creates the run + * moments later, and caching "no" would strand it on ULID ids for the rest of + * this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the position above the run's tail on disk and above every one this + * book knows of, and at or above `minSlot`. Distinct for every concurrent + * caller; the publish still has to prove the position was actually free. + * + * `minSlot` defaults to the position above the run's first slot, which is + * reserved for its own `run_created`: that event needs no allocation, and it + * may not be on disk yet when a concurrent `run_started` allocates (start() + * issues the creation and the queue send in parallel, and the run entity is + * published before its event). + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * The highest position this run has published, read from the log, or + * `FIRST_SLOT - 1` for a log with no events. + * + * This is the tail a claim has to clear. "Free" is not the property a claim + * needs: allocation is append-only, so a position left unwritten by an + * abandoned reservation stays empty for good, and a caller numbering from a + * snapshot that predates the events above such a hole aims straight at it. Let + * that write land and the event sits *below* events another replay already + * consumed — the log stays internally consistent while its order silently + * changes, which is enough to flip a race between a step and a sleep from one + * replay to the next. + * + * Reading the tail before the write also lets a doomed claim be rejected + * *before* the create materializes its step, hook or wait: the entity mutation + * runs ahead of the event publish, so a claim that only fails at the publish + * leaves an entity behind with no event, and the caller's re-proposal at the + * next slot then collides with its own orphan. A tail read here is not a + * promise — the publish is still the authority, and another instance sharing + * the data directory may have written above it — but it turns the case that + * actually happens (a caller numbering from a stale log) into a clean + * conflict. + */ + highestWritten(runId: string): Promise; + /** + * Forgets a reserved or claimed slot whose publish is never going to happen, + * so nothing waits on it. + * + * The position is handed out again only if it is still the highest one this + * book knows of, where it provably sits below nothing. Anywhere else it stays + * empty: it may already sit below a sibling that published, and recycling it + * there would put a later event below an earlier one. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run numbers by slot, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + /** runId → in-flight tail read, so a burst of writers shares one scan. */ + const syncs = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const outstanding = new Set(claims.get(runId)); + const book: RunSlots = { + written, + outstanding, + ceiling: Math.max(FIRST_SLOT - 1, ...written, ...outstanding), + }; + books.set(runId, book); + return book; + } + + /** Folds the run's published positions on disk into the book kept for it. */ + async function merge(runId: string, book: RunSlots): Promise { + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + } + return book; + } + + /** + * The run's book with the log's current tail folded in. + * + * Every allocation and every tail read goes through this, because the book on + * its own only knows what *this* instance did: another instance sharing the + * data directory publishes without touching it. One scan is shared by all + * callers waiting on it, so a burst of concurrent writers pays for one readdir + * and still leaves with distinct positions (the scan resolves first, the + * synchronous takes follow). + */ + function synced(runId: string): Promise { + const inFlight = syncs.get(runId); + if (inFlight) { + return inFlight; + } + // Seeding reads the log itself, so a book being opened for the first time is + // already current and a second scan would buy nothing. + const seeding = !books.has(runId); + const opened = open(runId); + const scan = + seeding && opened instanceof Promise + ? opened + : Promise.resolve(opened).then((book) => merge(runId, book)); + const pending = scan.finally(() => syncs.delete(runId)); + syncs.set(runId, pending); + return pending; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + + function take(book: RunSlots, minSlot: number): number { + const slot = Math.max(book.ceiling + 1, minSlot); + book.outstanding.add(slot); + book.ceiling = slot; + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { + // The take is synchronous once the scan resolves, so two concurrent + // callers sharing one scan still leave with different positions. + return take(await synced(runId), minSlot); + }, + + claim(runId, slot) { + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } + const book = books.get(runId); + if (book) { + book.outstanding.add(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + }, + + async highestWritten(runId) { + const book = await synced(runId); + return Math.max(FIRST_SLOT - 1, ...book.written); + }, + + release(runId, slot) { + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + return; + } + book.outstanding.delete(slot); + // The position is handed out again only while it is still the top of the + // book: nothing was handed out above it and nothing was read from disk + // above it, so it cannot land below an event that already exists. Below + // the top the ceiling stays where it is, and the position stays a hole. + if (book.ceiling === slot) { + book.ceiling = Math.max( + FIRST_SLOT - 1, + ...book.written, + ...book.outstanding + ); + } + }, + + observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + }, + + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + await merge(runId, book); + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. + }, + + clear() { + modes.clear(); + books.clear(); + claims.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0019_run_scoped_event_keys.sql b/packages/world-postgres/src/drizzle/migrations/0019_run_scoped_event_keys.sql new file mode 100644 index 0000000000..980739cbbc --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0019_run_scoped_event_keys.sql @@ -0,0 +1,10 @@ +-- Event ids are unique per run, not globally. Under slot identity (spec 6) +-- every run numbers its own log from 1, so "evnt_0...001" exists once per run +-- and the old global primary key would make the second run to reach slot 1 +-- collide with the first. +-- +-- The run leads the key so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id index redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 4ce969faa2..f4562493cc 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -134,6 +134,13 @@ "when": 1785619990000, "tag": "0018_add_hook_token_retention", "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1785801600000, + "tag": "0019_run_scoped_event_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 578c78cf2d..a17748c2df 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -153,7 +153,11 @@ export const events = schema.table( > >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: every run numbers its own log + // from 1, so `evnt_0…001` exists once per run. The run leads the key so the + // range scans in `list` stay a single index seek, which also makes a + // separate `run_id` index redundant. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 463ba42fba..98263ce464 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,8 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + reenqueueActiveRuns, + SPEC_VERSION_SLOT_IDENTITY, +} from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +66,9 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity. Every world reads + // both schemes whatever this says. + specVersion: SPEC_VERSION_SLOT_IDENTITY, capabilities: { hookRetention: { active: true } }, ...storage, ...streamer, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..ede2a0efa5 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,230 @@ +/** + * Slot identity for the postgres world. + * + * Runs name their events by position: `evnt_…001` is the first event of the + * run, `evnt_…002` the second. Contention on a position is the + * point — it is what makes a concurrent write detectable rather than silent — + * so the two things this module has to get right are that a position is written + * at most once and that a position this allocator loses is retried rather than + * abandoned as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so the highest id names the highest written position. A log + * holds ids of exactly one scheme, so that id also reports which scheme the run + * was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event at the position the caller claimed, or at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; + try { + return await options.write({ + primary: () => primary, + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 938ef77c3b..4e5bea1ceb 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -67,6 +74,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + highestEventId, + highestSlotOf, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; const DAY_MS = 24 * 60 * 60 * 1000; @@ -484,7 +498,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -495,7 +509,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -506,6 +520,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { if ( @@ -544,6 +616,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -639,7 +725,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -685,7 +777,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -705,6 +797,107 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and a ULID names no position: the replay would read + // it wherever its timestamp happens to sort rather than where the writer + // meant it to go. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway, so the query + // costs a slot-mode write nothing extra. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // An event id names a position in the run's log, so it is either claimed + // by a caller that holds the log (and is therefore asserting the log is + // complete up to that position) or allocated at write time for a caller + // that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + // A claim has to clear the log's tail, not merely be free. A position + // claimed by a write that then failed is never filled, so a log can + // carry holes below its tail; a caller numbering from a snapshot that + // predates the events above such a hole aims straight at it, and a + // free-position check would accept the write. The event then lands + // *below* events another replay has already consumed — the log stays + // internally consistent while its order silently changes, which is + // enough to flip a race between a step and a sleep from one replay to + // the next. + // + // Checked here, before the materialization below creates the step, hook + // or wait this event would now never accompany: a caller that + // re-proposes at the next position would otherwise collide with its own + // orphan and read that as "my write already landed". + seedHighestEventId = + seedHighestEventId ?? + (await highestEventId(drizzle, effectiveRunId)) ?? + undefined; + if (claimedSlot <= highestSlotOf(seedHighestEventId)) { + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -730,17 +923,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1228,41 +1423,51 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1272,99 +1477,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1527,7 +1737,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1562,20 +1772,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1664,47 +1875,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1736,9 +1949,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } @@ -1790,17 +2037,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..3a96d033a3 --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,462 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('rejects a claim on a free position below the log’s tail', async () => { + // The undercut that a "is the position free?" check cannot catch. A + // position claimed by a write that then failed is never filled, so a log + // carries holes below its tail — and a caller numbering from a snapshot + // that predates the events above one of those holes aims straight at it. + // Let it land and the event sits below events another replay has already + // consumed: the log stays internally consistent while its order silently + // changes. A claim asserts a complete log, so a claim that does not clear + // the tail is a conflict, exactly as a taken one is. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await expect( + createStep(runId, 'step_early', slotEventId(2)) + ).rejects.toThrow(SlotConflictError); + // The hole stays a hole, and the log stays in slot order. + expect(ascending(await slotsOf(runId))).toEqual([1, 3]); + }); + + test('accepts the claim immediately above a tail with a hole below it', async () => { + // The fence rejects at-or-below, so the first position above the tail has + // to stay writable — otherwise every write following a hole would + // conflict forever and the run could never make progress again. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + expect(await createStep(runId, 'step_next', slotEventId(4))).toBe( + slotEventId(4) + ); + expect(ascending(await slotsOf(runId))).toEqual([1, 3, 4]); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(3) } + ); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a batch of lazy starts', async () => { + // The suspension flush's lazy starts each reserve two positions and claim + // the top one, so the batch's claims are every second slot. A second + // event numbered off the log rather than off the claim would take the + // position below the *next* claim instead of below its own, leaving the + // batch's first reserved slot a hole the log can never close. + // + // The batch is issued one claim at a time because that is how the client + // issues it: a claim asserts the log is complete up to that position, so + // each one is taken against the tail its writer has actually seen. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = []; + for (const [index, eventId] of claims.entries()) { + started.push( + await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ); + } + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free rather than abandoning the one it lost, + // so a burst of concurrent writers still numbers itself densely. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('numbers a burst so the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index d514576c7b..c0b7f65491 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -1,6 +1,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -94,6 +95,16 @@ describe('isRetryableEventPostError', () => { expect(isRetryableEventPostError(new ThrottleError('429'))).toBe(false); }); + it('does not retry a lost event slot', () => { + // Re-issuing the same write is guaranteed to lose the slot again; only a + // merge and a replay can produce a write that lands. + expect( + isRetryableEventPostError( + new SlotConflictError('taken', { eventId: 'evnt_x' }) + ) + ).toBe(false); + }); + it('retries a body-parse failure (write may have landed)', () => { expect( isRetryableEventPostError( diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index fc0d3bd87f..25985a6c5c 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -40,6 +40,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -203,9 +204,13 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { export function isRetryableEventPostError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried in-process. // (425/429 are intentionally left to the runtime's retry-after handling.) + // A slot conflict is doubly definitive: the write lost a race for its event + // id, so re-issuing it unchanged is guaranteed to lose again. Only a merge and + // a replay can produce a write that can land. if ( EntityConflictError.is(err) || RunExpiredError.is(err) || + SlotConflictError.is(err) || TooEarlyError.is(err) || ThrottleError.is(err) ) { diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 181c678fec..f788468974 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -2,10 +2,16 @@ import { EntityConflictError, PreconditionFailedError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, } from '@workflow/errors'; +import { + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotIdBody, +} from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -28,7 +34,7 @@ import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; describe('throwForErrorResponse', () => { const call = ( status: number, - body = '{"message":"boom"}', + body: string | Uint8Array = '{"message":"boom"}', headers: Record = {} ) => throwForErrorResponse(status, headers, body, 'createEvent', 'http://x'); @@ -108,6 +114,109 @@ describe('throwForErrorResponse', () => { /createEvent failed: HTTP 500 plain text oops/ ); }); + + /** + * The slot-conflict 409 is the only v4 error body that arrives as CBOR: it + * carries the event-log delta the client replays from, whose payloads are + * byte strings that JSON cannot represent. Decoding it as JSON would lose the + * delta silently and mis-type the error as an entity conflict, which the + * runtime reads as "my write already landed". + */ + describe('slot conflict', () => { + const PAYLOAD = new Uint8Array([1, 2, 3]); + const conflictBody = ( + overrides: Record = {} + ): Uint8Array => + new Uint8Array( + encode({ + success: false, + error: 'slot-conflict', + message: "Event slot 'evnt_…003' is already taken", + details: { eventId: 'evnt_from_details' }, + events: [{ eventId: 'evnt_x', eventData: { output: PAYLOAD } }], + cursor: 'eid:evnt_x', + hasMore: false, + ...overrides, + }) + ); + + it('decodes a CBOR body into SlotConflictError with the delta intact', () => { + try { + call(409, conflictBody(), { + 'content-type': 'application/cbor', + 'x-wf-event-id': 'evnt_from_header', + }); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + const conflict = err as SlotConflictError; + expect(conflict.eventId).toBe('evnt_from_header'); + expect(conflict.cursor).toBe('eid:evnt_x'); + expect(conflict.hasMore).toBe(false); + // Binary payloads survive: this is what a JSON error path destroys. + expect(conflict.events).toHaveLength(1); + expect( + (conflict.events[0] as { eventData: { output: Uint8Array } }) + .eventData.output + ).toEqual(PAYLOAD); + // Not the 409 → EntityConflictError mapping, which the runtime reads as + // "the write I am retrying already landed". + expect(EntityConflictError.is(err)).toBe(false); + } + }); + + it('falls back to the eventId in details when the header is absent', () => { + try { + call(409, conflictBody(), { 'content-type': 'application/cbor' }); + expect.unreachable(); + } catch (err) { + expect((err as SlotConflictError).eventId).toBe('evnt_from_details'); + } + }); + + it('reports an empty delta when the backend could not read one', () => { + try { + call(409, conflictBody({ events: undefined, cursor: null }), { + 'content-type': 'application/cbor', + }); + expect.unreachable(); + } catch (err) { + const conflict = err as SlotConflictError; + expect(conflict.events).toEqual([]); + expect(conflict.cursor).toBeNull(); + } + }); + + it('reads a JSON-encoded slot conflict too', () => { + // Nothing in the protocol forbids a JSON encoding of the same body; only + // the delta's binary payloads require CBOR. + try { + call( + 409, + JSON.stringify({ + error: 'slot-conflict', + message: 'taken', + events: [], + cursor: 'eid:evnt_y', + hasMore: true, + }), + { 'x-wf-event-id': 'evnt_j' } + ); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + expect((err as SlotConflictError).hasMore).toBe(true); + } + }); + + it('leaves an ordinary 409 as EntityConflictError', () => { + // Entity materialization conflicts share the status and are how the + // runtime recognizes a duplicate write. + expect(() => + call(409, JSON.stringify({ error: 'conflict', message: 'exists' })) + ).toThrowError(EntityConflictError); + }); + }); }); /** @@ -954,6 +1063,117 @@ describe('createWorkflowRunEventV4 over HTTP', () => { expect('stateUpdatedAt' in (capturedMeta ?? {})).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('sends the claimed eventId and maxSlot in the frame meta', async () => { + // A slot-numbered run names its own event ids, so the id has to reach the + // wire: the backend reads it from the frame meta and inserts it + // conditionally. Dropped, the backend mints a ULID instead and the run + // silently reverts to server-assigned identity mid-log. + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const eventId = slotEventId(4); + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `wait_${slotIdBody(1)}`, + eventId, + maxSlot: 3, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventId).toBe(eventId); + expect(capturedMeta?.maxSlot).toBe(3); + agent.assertNoPendingInterceptors(); + }); + + it('omits eventId and maxSlot from the frame meta for a ULID-numbered run', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + expect('eventId' in (capturedMeta ?? {})).toBe(false); + expect('maxSlot' in (capturedMeta ?? {})).toBe(false); + agent.assertNoPendingInterceptors(); + }); }); /** diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index f26187f856..0015d9de03 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,6 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ +import { SlotConflictError } from '@workflow/errors'; import { type Event, getEventDataPayloadField } from '@workflow/world'; import { decode } from 'cbor-x'; import { coerceEventDates } from './event-coerce.js'; @@ -74,13 +75,33 @@ async function fetchV4( errorFromV4Response( response.status, headersToRecord(response.headers), - await response.text(), + await readErrorBody(response), opName, url ), }); } +/** + * The error body as bytes when it is CBOR, as text otherwise. + * + * Most v4 error responses are JSON, because the client sends no `Accept` header + * and the backend's error encoder defaults to it. A slot conflict is the + * exception: its body carries the event-log delta the client needs, whose + * payloads are byte strings that JSON cannot represent, so the backend encodes + * that one as CBOR regardless of the `Accept` header. + */ +async function readErrorBody(response: Response): Promise { + if (isCborContentType(response.headers.get('content-type'))) { + return new Uint8Array(await response.arrayBuffer()); + } + return await response.text(); +} + +function isCborContentType(contentType: string | null | undefined): boolean { + return contentType?.toLowerCase().includes('application/cbor') ?? false; +} + /** Flatten a fetch `Headers` into the record shape throwForErrorResponse * expects (it mirrors the v3 `makeRequest` error contract). */ function headersToRecord(headers: Headers): Record { @@ -217,6 +238,21 @@ export interface CreateEventV4Input { * without a loaded event log; older servers ignore it entirely. */ stateUpdatedAt?: number; + /** + * The event's id, claimed by the client rather than minted by the server. It + * encodes the event's position in the run's log. The server inserts it + * conditionally and answers 409 `slot-conflict` when the slot is already + * taken. Sent only for a run stamped with slot identity: a run guarded by the + * watermark instead is rejected with 400, and a server that predates the field + * ignores it and mints an id of its own. + */ + eventId?: string; + /** + * The highest slot the client has seen in the run's event log (0 when it has + * seen none). Observability only: slots are dense, so a persisted slot more + * than one past this is a permanent hole in the log. Ignored by older servers. + */ + maxSlot?: number; /** * Number of loaded events at or below `stateUpdatedAt` (i.e. the loaded * log's length). Sent with `stateUpdatedAt` so the backend can also reject @@ -353,6 +389,8 @@ function buildPostFrameMeta( if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } + if (input.eventId !== undefined) meta.eventId = input.eventId; + if (input.maxSlot !== undefined) meta.maxSlot = input.maxSlot; if (input.stateEventCount !== undefined) { meta.stateEventCount = input.stateEventCount; } @@ -366,6 +404,73 @@ function buildPostFrameMeta( return meta; } +/** + * The backend's machine-readable code for a lost event slot. Paired with 409 + * rather than 412 so a slot conflict stays distinguishable from the + * `stateUpdatedAt` watermark's staleness rejection while both are live. + */ +const V4_SLOT_CONFLICT_CODE = 'slot-conflict'; + +/** The fields a v4 error body may carry, whatever encoding it arrived in. */ +interface V4ErrorBody { + message?: unknown; + /** Machine-readable code. The backend names this field `error`. */ + error?: unknown; + code?: unknown; + events?: unknown; + cursor?: unknown; + hasMore?: unknown; + details?: unknown; +} + +/** Decode an error body as CBOR or JSON, or `undefined` if it is neither. */ +function decodeErrorBody( + errorBody: string | Uint8Array +): V4ErrorBody | undefined { + try { + const value = + typeof errorBody === 'string' + ? (JSON.parse(errorBody) as unknown) + : (decode(errorBody) as unknown); + return value && typeof value === 'object' + ? (value as V4ErrorBody) + : undefined; + } catch { + return undefined; + } +} + +/** + * Build the `SlotConflictError` for a 409 whose body names a taken slot. + * + * The conflicting event id comes from the response header rather than the body + * so the error is still actionable when the body failed to decode; the delta is + * best-effort in the other direction — an absent or malformed `events` leaves + * the runtime to reload the log itself, which is always correct. + */ +function slotConflictFromBody( + message: string, + responseHeaders: Record, + body: V4ErrorBody | undefined +): SlotConflictError { + const details = body?.details; + const detailEventId = + details && typeof details === 'object' && 'eventId' in details + ? (details as { eventId?: unknown }).eventId + : undefined; + const headerEventId = readHeader( + responseHeaders, + V4_RESPONSE_HEADERS.eventId + ); + return new SlotConflictError(message, { + eventId: + headerEventId ?? (typeof detailEventId === 'string' ? detailEventId : ''), + events: Array.isArray(body?.events) ? body.events : [], + cursor: typeof body?.cursor === 'string' ? body.cursor : null, + hasMore: body?.hasMore === true, + }); +} + /** * Build the typed error for a non-2xx v4 response. Reuses the shared * `errorForResponse` status → error-type contract (409→EntityConflictError, @@ -378,26 +483,29 @@ function buildPostFrameMeta( function errorFromV4Response( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; let details: unknown; - try { - const json = JSON.parse(errorBody) as { - message?: string; - code?: string; - events?: unknown; - cursor?: unknown; - }; - if (typeof json.message === 'string') message = json.message; - if (typeof json.code === 'string') code = json.code; - if (statusCode === 412) details = decodePreconditionDetails(json); - } catch { - // body wasn't JSON — keep the default message, append raw text below - if (errorBody) message += ` ${errorBody}`; + const decoded = decodeErrorBody(errorBody); + if (decoded) { + if (typeof decoded.message === 'string') message = decoded.message; + if (typeof decoded.code === 'string') code = decoded.code; + if (statusCode === 412) details = decodePreconditionDetails(decoded); + } else if (typeof errorBody === 'string' && errorBody) { + // Body was neither JSON nor CBOR — keep the default message and append the + // raw text so the response is still diagnosable. + message += ` ${errorBody}`; + } + + // A lost event slot is the one 409 that is not an entity conflict. The + // backend names its machine-readable code `error`; that field is read only + // here, so every other error keeps the status → type mapping below unchanged. + if (statusCode === 409 && decoded?.error === V4_SLOT_CONFLICT_CODE) { + return slotConflictFromBody(message, responseHeaders, decoded); } const retryAfter = parseRetryAfter( @@ -490,7 +598,7 @@ function hasUnusablePayload(candidate: Record): boolean { export function throwForErrorResponse( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): never { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 7e88df8737..34673fcf61 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -32,7 +32,11 @@ * the v3 path. */ -import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; +import { + HookNotFoundError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import { type AnyEventRequest, type CreateEventParams, @@ -434,6 +438,35 @@ function coerceNormalizedEvent(raw: Record): Event { return coerceEventDates(normalizeEventData(raw)); } +/** + * Runs an event create, normalizing the event-log delta a slot conflict carries + * into the same `Event` shape every other read path produces. + * + * The delta arrives as raw CBOR off the error response, so its nested dates are + * still ISO strings; the runtime merges these events into its loaded log and + * calls `.getTime()` on them, exactly as it does for the inline delta on the + * success path. Doing the coercion here rather than in the runtime keeps the + * wire's shape a concern of this adapter, and keeps `SlotConflictError.events` + * meaning the same thing for every World that raises it. + */ +async function withCoercedSlotConflictDelta( + op: () => Promise +): Promise { + try { + return await op(); + } catch (error) { + if (!SlotConflictError.is(error) || error.events.length === 0) { + throw error; + } + throw new SlotConflictError(error.message, { + eventId: error.eventId, + events: (error.events as Record[]).map(coerceEventDates), + cursor: error.cursor, + hasMore: error.hasMore, + }); + } +} + function decodeLegacyStructuredError(payload: Uint8Array): unknown { if (hasSerializedDataFormatPrefix(payload)) { return payload; @@ -587,9 +620,11 @@ export async function createWorkflowRunEvent( // the next queue delivery. Non-retryable // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. - return await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, params, config), - data.eventType + return await withCoercedSlotConflictDelta(() => + withEventPostRetry( + () => createWorkflowRunEventInner(id, data, params, config), + data.eventType + ) ); } catch (err) { // 404 on hook_disposed / hook_received → already-disposed hook. @@ -713,6 +748,13 @@ async function createWorkflowRunEventInner( ...(params?.resumePayloadDigest ? { resumePayloadDigest: params.resumePayloadDigest } : {}), + // Slot identity: the runtime names the event's own id, claiming that + // position in the run's event log. The server inserts it conditionally + // and answers 409 slot-conflict when another writer got there first. + // `maxSlot` rides along so the server can spot a gap, which slots being + // dense makes an unrecoverable corruption. + ...(params?.eventId ? { eventId: params.eventId } : {}), + ...(params?.maxSlot !== undefined ? { maxSlot: params.maxSlot } : {}), remoteRefBehavior, payload, ...meta, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 7d42a8b6ec..00b5aeee99 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { SPEC_VERSION_SLOT_IDENTITY } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,10 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6). It reads + // every earlier version too, so this only decides how the runs it creates + // from here on are numbered. + specVersion: SPEC_VERSION_SLOT_IDENTITY, capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 94687e8f20..3f5b32e392 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -32,7 +32,10 @@ import { version } from './version.js'; * `main` — rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. */ -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +// TEMPORARY — revert to '' before merge. Points e2e at the slot-identity +// backend branch deployment. +export const WORKFLOW_SERVER_URL_OVERRIDE = + 'https://workflow-server-git-peter-slot-event-identity.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter. diff --git a/packages/world/package.json b/packages/world/package.json index 98271e5930..d42386cca3 100644 --- a/packages/world/package.json +++ b/packages/world/package.json @@ -20,6 +20,7 @@ "scripts": { "build": "tsc", "dev": "tsc --watch", + "test": "vitest run src", "clean": "tsc --build --clean && rm -rf dist" }, "dependencies": { diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index b3dc6c9866..9df02e6b8d 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -867,6 +867,36 @@ export interface CreateEventParams { * across the SDK and the backend. */ skipPreload?: boolean; + /** + * The event's id, chosen by the client rather than the World. + * + * Only sent for a run whose spec version numbers events by slot + * (`SPEC_VERSION_SLOT_IDENTITY`), where the id encodes the event's position in + * the log and is therefore the client's claim on that position. + * + * Backend contract (for World implementers who want to support slot + * identity): treat the id as a claim to be won, not a hint. Insert it under a + * uniqueness constraint on `(runId, eventId)` and, when the id is already + * taken, reject the write with `SlotConflictError` (HTTP 409) instead of + * minting a different id — a lost slot means the client replayed against an + * event log missing at least one event, so its whole proposed event, not just + * its id, is suspect. Reject a mismatch in either direction with a 400: a + * ULID names a time and a slot names a position, so a log holding both sorts + * partly by one and partly by the other and no replay can read it in the order + * it was written. + * + * A World that ignores this field mints ids itself, which is correct only for + * a run that was never stamped with slot identity. + */ + eventId?: string; + /** + * The highest slot the client has observed in the run's event log, or 0 for a + * log holding no slot ids. Sent alongside {@link eventId} purely as + * an observability signal: because slots are dense, a persisted slot more + * than one past this is a hole, which is unrecoverable and worth alerting on. + * Worlds MAY ignore it. + */ + maxSlot?: number; } /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index d30909aa1e..49205702d9 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -106,16 +106,32 @@ export { PaginatedResponseSchema, StructuredErrorSchema, } from './shared.js'; +export { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, + slotEventId, + slotFromId, + slotIdBody, + slotRetryDelay, +} from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + usesSlotIdentity, } from './spec-version.js'; export type * from './steps.js'; export { diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts new file mode 100644 index 0000000000..acdfada627 --- /dev/null +++ b/packages/world/src/slot-identity.test.ts @@ -0,0 +1,107 @@ +import { ulid } from 'ulid'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + slotEventId, + slotFromId, + slotIdBody, +} from './slot-identity.js'; +import { ulidToDate } from './ulid.js'; + +describe('slotIdBody', () => { + it('pads to ULID width so a slot is accepted wherever a ULID is', () => { + const body = slotIdBody(FIRST_SLOT); + expect(body).toHaveLength(SLOT_ID_WIDTH); + expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); + // Crockford base32 starts with the decimal digits, so the padded body + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); + }); + + it('orders lexicographically by slot at a fixed width', () => { + const ascending = [1, 2, 9, 10, 100].map(slotIdBody); + expect([...ascending].sort()).toEqual(ascending); + }); + + it('rejects slots outside the dense numbering', () => { + expect(() => slotIdBody(0)).toThrow(); + expect(() => slotIdBody(-1)).toThrow(); + expect(() => slotIdBody(1.5)).toThrow(); + }); +}); + +describe('slotFromId', () => { + it('round-trips a prefixed id', () => { + expect(slotFromId(`step_${slotIdBody(42)}`)).toBe(42); + }); + + it('round-trips a bare body', () => { + expect(slotFromId(slotIdBody(42))).toBe(42); + }); + + it('reads no slot out of a ULID id', () => { + expect(slotFromId(`evnt_${ulid()}`)).toBeUndefined(); + expect(isSlotId(`evnt_${ulid()}`)).toBe(false); + }); + + it('reads no slot out of the all-zero range fence', () => { + // Slot 0 is the inclusive lower fence for range queries over a run's + // events, never an event. + expect(slotFromId('0'.repeat(SLOT_ID_WIDTH))).toBeUndefined(); + }); + + it('reads no slot out of a body of the wrong width', () => { + expect(slotFromId('evnt_1')).toBeUndefined(); + expect(slotFromId(`evnt_${'1'.repeat(SLOT_ID_WIDTH + 1)}`)).toBeUndefined(); + }); +}); + +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + +describe('maxSlotOf', () => { + it('finds the highest slot regardless of position', () => { + // A log is merged from several loads and is not sorted, so the last element + // is not necessarily the highest slot. + expect( + maxSlotOf([slotEventId(3), slotEventId(7), slotEventId(1)].map(toEvent)) + ).toBe(7); + }); + + it('reports 0 for an empty or ULID-numbered log', () => { + expect(maxSlotOf([])).toBe(0); + expect(maxSlotOf([toEvent(`evnt_${ulid()}`)])).toBe(0); + }); + + it('ignores ULID ids mixed in with slots', () => { + // A mixed log violates slot identity's purity invariant, but the scan must + // still report the highest slot rather than throwing or returning 0. + expect( + maxSlotOf([toEvent(`evnt_${ulid()}`), toEvent(slotEventId(2))]) + ).toBe(2); + }); +}); + +function toEvent(eventId: string): { eventId: string } { + return { eventId }; +} diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts new file mode 100644 index 0000000000..256bcf8837 --- /dev/null +++ b/packages/world/src/slot-identity.ts @@ -0,0 +1,121 @@ +/** + * Slot identity: dense, per-run numbering for event ids. + * + * A slot id's body is a decimal counter zero-padded to ULID width. Crockford's + * base32 alphabet begins with the ten decimal digits, so that body is a + * syntactically valid ULID body — every schema, sort key, range fence and + * cursor that accepted a ULID keeps accepting a slot — and because the width is + * fixed, lexicographic order is numeric order. + * + * Slots start at 1 and are handed out above every position the allocator has + * seen, never into a lower one that happens to be free. That makes contention + * explicit — two writers proposing one position cannot both win, and the loser + * is told which events it was missing — and it keeps slot order, which is the + * order a replay reads the log in, a linear extension of what actually + * happened. Filling a hole would place an event below ones that preceded it, + * and a replay reaching a `step_completed` below its own `step_started` diverges + * for good. Zero is left unused because the inclusive lower fence for + * range queries over a run's events is the all-zero id. + * + * Allocation being append-only does not make a published log gap-free, so + * `events.length === maxSlot` is not a completeness proof. A slot claimed by an + * operation that then fails for a reason of its own is never filled, and if a + * later slot has already been published the gap is permanent. Nothing may treat + * a missing slot as an event still on its way. + * + * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so + * nothing may read a time out of one. Use the event's own `createdAt` / + * `occurredAt`. + */ + +/** Width of a slot id's body: ULID width, so a slot is accepted wherever a ULID is. */ +export const SLOT_ID_WIDTH = 26; + +/** First slot in a run. Slot 0 is unused — it is the inclusive range fence. */ +export const FIRST_SLOT = 1; + +const SLOT_BODY_PATTERN = new RegExp(`^[0-9]{${SLOT_ID_WIDTH}}$`); + +/** + * The id body naming `slot`, e.g. `1` → `00000000000000000000000001`. Callers + * prepend their own prefix (`evnt_`, `step_`, `wait_`). + */ +export function slotIdBody(slot: number): string { + if (!Number.isInteger(slot) || slot < FIRST_SLOT) { + throw new Error( + `Slot must be an integer >= ${FIRST_SLOT}, received ${slot}` + ); + } + const body = String(slot).padStart(SLOT_ID_WIDTH, '0'); + if (body.length > SLOT_ID_WIDTH) { + throw new Error(`Slot ${slot} does not fit in ${SLOT_ID_WIDTH} digits`); + } + return body; +} + +/** + * The slot named by an id, or undefined if the id is not a slot id. Accepts + * both a prefixed id (`step_0…001`) and a bare body. + */ +export function slotFromId(id: string): number | undefined { + const underscore = id.indexOf('_'); + const body = underscore === -1 ? id : id.slice(underscore + 1); + if (!SLOT_BODY_PATTERN.test(body)) { + return undefined; + } + const slot = Number(body); + return slot >= FIRST_SLOT ? slot : undefined; +} + +/** Whether an id numbers itself by slot rather than by ULID. */ +export function isSlotId(id: string): boolean { + return slotFromId(id) !== undefined; +} + +/** The event id occupying `slot`. */ +export function slotEventId(slot: number): string { + return `evnt_${slotIdBody(slot)}`; +} + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** + * The highest slot named by any of `events`, or 0 when none names a slot. + * + * Scans rather than reading the last element: a log is merged from several + * loads and is not necessarily sorted, and callers use this value to pick the + * next free slot. + */ +export function maxSlotOf(events: readonly { eventId: string }[]): number { + let max = 0; + for (const event of events) { + const slot = slotFromId(event.eventId); + if (slot !== undefined && slot > max) { + max = slot; + } + } + return max; +} diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..3ec2152838 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -4,6 +4,8 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +15,20 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than the floor it requires', () => { + // Worlds stamp slot identity on new runs while still reading everything + // back to the compression version, so the ceiling sits above the floor. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +36,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +70,13 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('the version worlds stamp on new runs', () => { + it('is slot identity, and is readable by the worlds that stamp it', () => { + // Every world's `specVersion` is SPEC_VERSION_SLOT_IDENTITY. A world that + // rejected spec-6 would reject the runs it had just stamped spec-6 itself, + // at their first event after run_created. + expect(SPEC_VERSION_SLOT_IDENTITY).toBeGreaterThan(SPEC_VERSION_CURRENT); + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index cf516b772b..f74977d93f 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -31,13 +31,44 @@ export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; +/** + * Runs at this spec version or later number their events by dense per-run slots + * (`evnt_…001`) instead of ULIDs. Ids are minted by the client, and a write + * that proposes an event id already taken is rejected rather than renumbered — + * which is what lets a client prove its loaded log is complete. + * + * A run is in exactly one mode for life: the mode is read from the run's + * persisted `specVersion`, never from the build. A run whose log holds ULID + * event ids and is replayed by a slot-capable build would otherwise propose + * `evnt_…001`, a position its very first event already occupies. + * + * Correlation ids are unrelated: steps, waits, hooks and attributes use seeded + * ULIDs at every spec version. + */ +export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; + /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * The floor a world has to support, and a *lower* bar than the version worlds + * stamp on the runs they create ({@link SPEC_VERSION_SLOT_IDENTITY}) or the + * newest one this build can read ({@link SPEC_VERSION_MAX_SUPPORTED}). */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world reads further back + * than the floor it requires, and because a world that stamps slot identity + * still has to read the spec-5 runs it created before the switch. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -55,7 +86,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -63,5 +94,17 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; +} + +/** + * Whether a run numbers its events by slot. Always pass the run's persisted + * `specVersion`; see `SPEC_VERSION_SLOT_IDENTITY`. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run uses slot identity + */ +export function usesSlotIdentity(v: number | undefined | null): boolean { + if (v === undefined || v === null) return false; + return v >= SPEC_VERSION_SLOT_IDENTITY; } diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..92dbbd4d66 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every event named by a slot. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; diff --git a/workbench/fastify/public/index.html b/workbench/fastify/public/index.html deleted file mode 100644 index 59870afcbd..0000000000 --- a/workbench/fastify/public/index.html +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - Workflow SDK + Nitro Example - - - - -

Workflow SDK + Nitro Example

-
- - - - diff --git a/workbench/nextjs-turbopack/vercel.json b/workbench/nextjs-turbopack/vercel.json index 22da2a3327..ff2e7944b0 100644 --- a/workbench/nextjs-turbopack/vercel.json +++ b/workbench/nextjs-turbopack/vercel.json @@ -5,8 +5,7 @@ } }, "env": { - "WORKFLOW_PUBLIC_MANIFEST": "1", - "WORKFLOW_PER_KIND_CORRELATION_IDS": "1" + "WORKFLOW_PUBLIC_MANIFEST": "1" }, "regions": [ "iad1",