Slot event identity: number a run's events by position

.changeset/late-hook-delivery-divergence.md

@@ -0,0 +1,6 @@
+---
+'@workflow/core': patch
+'workflow': patch
+---
+
+Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched

.changeset/quickjs-hook-ordering.md

@@ -0,0 +1,5 @@
+---
+'@workflow/core': patch
+---
+
+Fix aborts raised inside a step being dropped on the QuickJS engine when the abort controller's hook was created in the same suspension.

.changeset/slot-duplicate-start-skip.md

@@ -0,0 +1,6 @@
+---
+'@workflow/core': patch
+'workflow': patch
+---
+
+Skip a step whose concurrent start another handler already wrote, instead of restarting the replay to rediscover it

.changeset/slot-event-identity.md

@@ -0,0 +1,11 @@
+---
+'@workflow/world-vercel': minor
+'@workflow/world-local': minor
+'@workflow/world-postgres': minor
+'@workflow/world': minor
+'@workflow/errors': minor
+'@workflow/core': minor
+'workflow': minor
+---
+
+Number new runs' events by position instead of by ULID, in every World, so a reader can prove its copy of an event log is complete. A creation whose position another writer already took is rejected with a `SlotConflictError`, and the runtime merges the events it was missing, replays, and claims a free position. Correlation IDs now come from a sequence per entity type rather than one shared across all of them, so creating a hook or a sleep no longer renames the steps after it.

.changeset/turbo-run-started-occurred-at.md

@@ -0,0 +1,5 @@
+---
+'@workflow/core': patch
+---
+
+Report the same workflow start time on an optimistically started run's first pass and its replays

.github/scripts/render-event-log-race-repro-results.js

@@ -407,6 +407,11 @@ function renderLatestFailures(entry) {
       `\nShowing 20 of ${entry.failing.length + entry.truncatedFailingCount} non-completed runs.`
     );
   }
+  // Deliberately not inlined here: the slices are large and this comment has a
+  // size limit, while the artifact has none.
+  console.log(
+    '\nThe `event-log-race-repro-results` artifact carries a window of the committed log around the divergent event, for a sample of the corruptions.'
+  );
   console.log('');
 }
 

.gitignore

@@ -43,6 +43,9 @@ workbench/nextjs-*/public/.well-known/workflow
 workbench/sveltekit/static/.well-known/workflow
 
 
+# Local e2e diagnostics dumps
+e2e-diagnostics-*.json
+
 # Event log race repro output (written to the repo root by the harness and by
 # scripts/event-log-race-repro-local.sh)
 event-log-race-repro-results.json

docs/content/docs/v5/api-reference/workflow-errors/index.mdx

@@ -82,6 +82,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow
     <Card href="/docs/api-reference/workflow-errors/precondition-failed-error" title="PreconditionFailedError">
         Thrown when an event creation is rejected because the client's event-log snapshot is stale.
     </Card>
+    <Card href="/docs/api-reference/workflow-errors/slot-conflict-error" title="SlotConflictError">
+        Thrown when an event creation is rejected because another writer already took the event's slot.
+    </Card>
     <Card href="/docs/api-reference/workflow-errors/too-early-error" title="TooEarlyError">
         Thrown when a request is made before the system is ready to process it.
     </Card>

docs/content/docs/v5/api-reference/workflow-errors/meta.json

@@ -15,6 +15,7 @@
     "throttle-error",
     "entity-conflict-error",
     "precondition-failed-error",
+    "slot-conflict-error",
     "run-expired-error",
     "run-not-supported-error",
     "too-early-error"

docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx

@@ -0,0 +1,76 @@
+---
+title: SlotConflictError
+description: Thrown when an event creation is rejected because another writer already took the event's slot.
+type: reference
+summary: Catch SlotConflictError when a world rejects an event creation whose slot in the run's event log was already taken.
+related:
+  - /docs/api-reference/workflow-errors/workflow-world-error
+  - /docs/api-reference/workflow-errors/precondition-failed-error
+---
+
+`SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics.
+
+Each event's id encodes its position in the run's log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on.
+
+The rejection carries the missing events inline so that merge usually costs no extra round-trip:
+
+- `events` — the events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them, in which case the client reloads the log itself.
+- `cursor` — cursor to continue the delta from.
+- `hasMore` — whether events beyond `events` remain to be fetched.
+
+[`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error) reports the same staleness for a run guarded by an event-log snapshot watermark. A run uses one scheme or the other for its whole life, decided when it is created.
+
+<Callout>
+The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly.
+</Callout>
+
+```typescript lineNumbers
+import { SlotConflictError } from "workflow/errors"
+declare const world: { events: { create(...args: any[]): Promise<any> } }; // @setup
+declare const runId: string; // @setup
+declare const event: any; // @setup
+
+try {
+  await world.events.create(runId, event);
+} catch (error) {
+  if (SlotConflictError.is(error)) { // [!code highlight]
+    console.log(`Slot ${error.eventId} taken; ${error.events.length} event(s) missed`);
+  }
+}
+```
+
+## API Signature
+
+### Properties
+
+<TSDoc
+definition={`
+interface SlotConflictError {
+  /** The event id, naming a slot, that was already taken. */
+  eventId: string;
+  /** The events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them. */
+  events: unknown[];
+  /** Cursor to continue the delta from, or null. */
+  cursor: string | null;
+  /** Whether events beyond \`events\` remain to be fetched. */
+  hasMore: boolean;
+  /** The error message. */
+  message: string;
+}
+export default SlotConflictError;`}
+/>
+
+### Static Methods
+
+#### `SlotConflictError.is(value)`
+
+Type-safe check for `SlotConflictError` instances. Preferred over `instanceof` because it works across module boundaries and VM contexts.
+
+```typescript
+import { SlotConflictError } from "workflow/errors"
+declare const error: unknown; // @setup
+
+if (SlotConflictError.is(error)) {
+  // error is typed as SlotConflictError
+}
+```

docs/content/docs/v5/api-reference/workflow-runtime/world/storage.mdx

@@ -289,6 +289,8 @@ const result = await world.hooks.list({ // [!code highlight]
 
 **Returns:** `{ data: Hook[], cursor?: string }`
 
+Hooks are ordered by hook ID. The hooks a workflow creates are listed in the order it created them. A hook the runtime creates on your behalf, such as the one backing an abort controller, draws its ID from a separate sequence, so it sorts at an arbitrary position among yours rather than at its creation position.
+
 ### Hook Type
 
 | Field | Type | Description |

docs/content/docs/v5/configuration/runtime-tuning.mdx

@@ -69,9 +69,11 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL
 
 ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`
 
-- Default: `3`
+- Default: `3`, or `12` on a run numbering its events by position (see [`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error))
 - How many times a single invocation restarts its replay in-process after a rejected event creation before it falls back to a re-invocation.
 - A restart reloads the event log and rebuilds the workflow from scratch, so it costs a replay but no queue round trip. A World may attach the missing events to its rejection, in which case the first restart needs no event-log request at all.
+- A run numbering its events by position reads only the page after its cursor instead of reloading the log, since positions are allocated in order and every event it was missing sorts above what it already has. Restarts are cheap enough there that the higher default is worth taking before a re-invocation and its delay.
+- Setting this overrides both defaults.
 
 ### `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS`
 
@@ -85,16 +87,17 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL
 - Delay before a re-invocation caused by a rejected event creation.
 - Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up — so the delay gives the other writers a moment to quiesce.
 
-### `WORKFLOW_PER_KIND_CORRELATION_IDS`
+### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS`
 
-- Default: disabled
-- Experimental. Gives each kind of entity a workflow creates — steps, waits, hooks, attribute writes, abort controllers, stream IDs — its own sequence of correlation IDs.
-- With one sequence shared by every kind, an ID is an ordinal over the whole run, so a single extra draw of any kind shifts every ID after it. Two concurrent replays of the same run that disagree about one `sleep()` then assign different IDs to every step that follows, and each writes events the other can neither match nor consume, which fails the run with `CORRUPTED_EVENT_LOG`. Per-kind sequences confine that to the kind that actually differs.
-- IDs remain ordered within a kind, so hooks created by your workflow are still listed in creation order. A hook the runtime creates for you, such as the one backing an abort controller, draws from its own kind and so is listed at an arbitrary position relative to your hooks rather than at its creation position.
-- A run must replay under the scheme that minted its IDs. A replay that switches schemes mid-run assigns IDs its own earlier events do not carry, so it can consume none of them and the run fails.
-  - On Vercel, a run keeps replaying on the deployment it started on, so it only ever sees the value baked into that deployment. Changing the setting affects new runs only.
-  - Elsewhere — `@workflow/world-postgres`, `@workflow/world-local`, any self-hosted process — nothing pins a run to the code that started it. Turn the setting on during a quiet window with no runs in flight, and roll the new value out to your whole fleet at once: a rolling deploy that leaves both values live replays one run under two schemes concurrently, which is the failure the setting exists to reduce.
-- Set `1` to enable.
+- Default: `15`
+- Base for the randomized wait before an in-process replay restart re-derives, in milliseconds. The wait doubles with each restart the invocation has spent, up to [`WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`](#workflow_precondition_restart_backoff_max_ms), and is drawn uniformly from zero to that bound.
+- The wait exists for runs with several replays in flight at once — a fan-out of steps completing together, or a burst of hooks. A rejected event creation means another replay got there first; if every loser re-derives immediately they all contend again, and none of them pulls far enough ahead to finish. Drawing each wait from the full range spreads the retries apart.
+- Set to `0` to restart without waiting.
+
+### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`
+
+- Default: `400`
+- Ceiling on the wait described above.
 
 ## Inline execution
 
@@ -262,6 +265,12 @@ These variables are primarily for tests, debugging, or unusual deployments.
 - Delay before the unconsumed-event check fires.
 - Minimum: `10`.
 
+### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`
+
+- Default: `15000`
+- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run.
+- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected.
+
 ### `WORKFLOW_LOCK_POLL_INTERVAL_MS`
 
 - Default: `10`

packages/core/e2e/event-log-race-repro.test.ts

@@ -137,6 +137,31 @@ interface ReproRunResult {
     resumesFailed: number;
     stragglers?: number;
   };
+  /**
+   * The committed log around the divergent event, for corruptions only. A
+   * divergence is a disagreement between the order the log records and the
+   * order a replay reconstructs, so the log's own ordering is the only
+   * evidence that distinguishes the candidate causes — and the run is on an
+   * ephemeral preview deployment, so it has to be captured while the job is
+   * still running rather than read back afterwards.
+   */
+  logSlice?: LogSliceEntry[];
+}
+
+/**
+ * One committed event, projected to the fields that decide replay order:
+ * its position (`slot`), what it resolves (`eventType`/`correlationId`), and
+ * both clock domains. `occurredAt` is the client/VM moment and `createdAt`
+ * the persisted event time the sandbox clock is driven from; a race between a
+ * `sleep` and a step is decided by that clock, so the two have to be
+ * comparable side by side.
+ */
+interface LogSliceEntry {
+  slot: number | string;
+  eventType: string;
+  correlationId?: string;
+  occurredAt?: string;
+  createdAt?: string;
 }
 
 function envNumber(name: string, fallback: number) {
@@ -392,6 +417,98 @@ function validateStormReturn(value: unknown): {
   return { stragglers };
 }
 
+/**
+ * Reads a terminal-failed run's error through `returnValue()`, which hydrates
+ * the stored payload into an Error. Returns undefined when the read itself
+ * fails — the outcome is already known from `errorCode`, so a missing message
+ * degrades the report rather than the classification.
+ */
+async function readFailureMessage(
+  run: Run<unknown>
+): Promise<{ name?: string; message?: string } | undefined> {
+  try {
+    await run.returnValue();
+    return undefined;
+  } catch (err) {
+    if (WorkflowRunFailedError.is(err)) {
+      const cause = err.cause;
+      return {
+        name: cause instanceof Error ? cause.name : err.name,
+        message: cause instanceof Error ? cause.message : err.message,
+      };
+    }
+    return undefined;
+  }
+}
+
+/**
+ * How many events either side of the divergent one to keep. The window has to
+ * span a whole round of the storm — width branches, each with a step create,
+ * start and completion, plus the round's waits — or it can miss the very
+ * event whose position explains the divergence.
+ */
+const LOG_SLICE_RADIUS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RADIUS', 45);
+
+/**
+ * Cap on how many corruptions carry a slice. The results JSON is rendered into
+ * a PR comment, and a body over GitHub's limit is rejected outright, so the
+ * slices are a sample rather than a complete record.
+ */
+const LOG_SLICE_MAX_RUNS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RUNS', 6);
+
+let logSlicesCaptured = 0;
+
+/** Worlds hand timestamps back as a Date or as the stored ISO string. */
+function isoOrUndefined(value: unknown): string | undefined {
+  if (value instanceof Date) return value.toISOString();
+  return typeof value === 'string' ? value : undefined;
+}
+
+/** Ordinal of a slot-numbered id, or the raw id when it is a ULID. */
+function idOrdinal(id: string): number | string {
+  const body = id.slice(id.indexOf('_') + 1);
+  return /^\d+$/.test(body) ? Number(body) : id;
+}
+
+/**
+ * Reads the committed log and returns the window around the divergent event
+ * named in `message`. Best-effort: the report is a measurement, so a failed
+ * read costs a slice rather than the attempt's classification.
+ */
+async function readLogSlice(
+  runId: string,
+  message: string | undefined
+): Promise<LogSliceEntry[] | undefined> {
+  if (logSlicesCaptured >= LOG_SLICE_MAX_RUNS) return undefined;
+  try {
+    const world = await getWorld();
+    const { data: events } = await world.events.list({ runId });
+    const projected: LogSliceEntry[] = events.map((event) => ({
+      slot: idOrdinal(event.eventId),
+      eventType: event.eventType,
+      correlationId: event.correlationId,
+      occurredAt: isoOrUndefined(event.occurredAt),
+      createdAt: isoOrUndefined(event.createdAt),
+    }));
+
+    // The corruption message names the last divergent event; centre on it when
+    // it is there, and otherwise keep the tail, where a divergence that ran out
+    // of recovery replays ends up.
+    const divergent = message?.match(/evnt_[0-9A-Z]+/)?.[0];
+    const at = divergent
+      ? projected.findIndex((entry) => entry.slot === idOrdinal(divergent))
+      : -1;
+    const centre = at >= 0 ? at : projected.length - 1;
+    logSlicesCaptured += 1;
+    return projected.slice(
+      Math.max(0, centre - LOG_SLICE_RADIUS),
+      centre + LOG_SLICE_RADIUS + 1
+    );
+  } catch {
+    return undefined;
+  }
+}
+
 async function pollTerminalRun(
   run: Run<unknown>,
   startedAt: number,
@@ -431,14 +548,27 @@ async function pollTerminalRun(
         errorCode?: string;
         error?: { name?: string; message?: string };
       };
+      // `runs.get` hands back the raw serialized error payload, not an Error, so
+      // reading `.message` off it yields undefined and the report records the
+      // code with no diagnosis. Read the failure through the public
+      // return-value path, which hydrates it. For a corruption that message
+      // carries the divergent event and what the replay was waiting for, which
+      // is the whole reason to keep the report.
+      const hydrated = await readFailureMessage(run);
+      const outcome = classifyFailure(failure.errorCode);
+      const errorMessage = hydrated?.message ?? failure.error?.message;
       return {
         ...base,
-        outcome: classifyFailure(failure.errorCode),
+        outcome,
         status: runData.status,
         errorCode: failure.errorCode,
-        errorMessage: failure.error?.message,
-        errorName: failure.error?.name,
+        errorMessage,
+        errorName: hydrated?.name ?? failure.error?.name,
         durationMs: Date.now() - startedAt,
+        logSlice:
+          outcome === 'CORRUPTED_EVENT_LOG'
+            ? await readLogSlice(run.runId, errorMessage)
+            : undefined,
       };
     }
 

packages/core/e2e/utils.ts

@@ -659,7 +659,11 @@ async function getRunDiagnostics(tracked: TrackedRun): Promise<string> {
           const elapsed = baseTime
             ? ((event.createdAt?.getTime?.() ?? 0) - baseTime) / 1000
             : 0;
-          const prefix = `  +${elapsed.toFixed(1)}s`;
+          // The event's own id leads the line: on a run numbering its events
+          // by position it says where in the log the event sits, which is what
+          // a diagnosis of an out-of-order or gapped log needs and what the
+          // correlation id below cannot report.
+          const prefix = `  ${event.eventId} +${elapsed.toFixed(1)}s`;
           let detail = event.eventType;
           if ('eventData' in event) {
             const data = (event as any).eventData;