From 288b42bd9d7077cab921cdb13cf8527dd47d5045 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:36:20 -0700 Subject: [PATCH 1/3] perf(core): consume run_started replay page --- .changeset/stream-run-started-page.md | 7 + packages/core/src/runtime-trace-mode.test.ts | 3 - packages/core/src/runtime.test.ts | 79 ++--- packages/core/src/runtime.ts | 310 +++++------------- packages/core/src/runtime/constants.ts | 6 +- packages/core/src/runtime/step-executor.ts | 89 ++--- packages/core/src/runtime/step-latency.ts | 15 +- .../core/src/runtime/suspension-handler.ts | 34 -- packages/core/src/serialization.ts | 97 +----- packages/core/src/set-attributes.test.ts | 54 --- packages/core/src/set-attributes.ts | 14 - packages/core/src/step/context-storage.ts | 10 - packages/core/src/step/writable-stream.ts | 22 +- .../src/telemetry/semantic-conventions.ts | 9 +- packages/core/src/workflow.test.ts | 144 +------- packages/core/src/workflow.ts | 21 +- .../src/writable-stream-telemetry.test.ts | 28 -- packages/core/src/writable-stream.test.ts | 173 +--------- packages/world-vercel/src/events-v4.test.ts | 135 ++++---- packages/world-vercel/src/events-v4.ts | 133 +++++--- packages/world-vercel/src/events.test.ts | 103 ++++++ packages/world-vercel/src/events.ts | 98 ++++-- packages/world/src/events.ts | 17 - 23 files changed, 477 insertions(+), 1124 deletions(-) create mode 100644 .changeset/stream-run-started-page.md diff --git a/.changeset/stream-run-started-page.md b/.changeset/stream-run-started-page.md new file mode 100644 index 0000000000..270e239ebf --- /dev/null +++ b/.changeset/stream-run-started-page.md @@ -0,0 +1,7 @@ +--- +'@workflow/core': patch +'@workflow/world': patch +'@workflow/world-vercel': patch +--- + +Load the first replay page from the `run_started` response and await startup before Turbo replay. diff --git a/packages/core/src/runtime-trace-mode.test.ts b/packages/core/src/runtime-trace-mode.test.ts index fbf5928b61..66d106271b 100644 --- a/packages/core/src/runtime-trace-mode.test.ts +++ b/packages/core/src/runtime-trace-mode.test.ts @@ -309,9 +309,6 @@ describe('workflowEntrypoint trace modes', () => { (e) => e.name === 'workflow.run_started.create.start' ); expect(runStartedCreateEvent).toBeDefined(); - expect( - runStartedCreateEvent?.attributes['workflow.run_started.skip_preload'] - ).toBe(false); // Queue-delivered invocation spans use the CONSUMER kind, matching // queue-delivered step.execute spans. diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..af1307bd82 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -1485,14 +1485,15 @@ describe('workflowEntrypoint turbo mode', () => { /** * Drives the handler with a first-invocation message (runInput present) at the * given delivery `attempt`. `runStartedGate`, when provided, holds the - * `run_started` create until released — its resolution pushes - * 'run_started_resolved' so tests can assert the body ran before or after it. + * `run_started` create until released. `stepStartedGate` can separately hold + * the optimistic step claim. */ async function driveTurbo(opts: { runId: string; attempt: number; source: string; runStartedGate?: Promise; + stepStartedGate?: Promise; }) { const { runId, attempt, source } = opts; const order = turboOrder; @@ -1528,6 +1529,8 @@ describe('workflowEntrypoint turbo mode', () => { } if (data.eventType === 'step_started') { order.push('step_started_called'); + if (opts.stepStartedGate) await opts.stepStartedGate; + order.push('step_started_resolved'); const d = data.eventData as { stepName?: string; input?: unknown }; if (d?.input !== undefined) { rec({ @@ -1597,40 +1600,41 @@ describe('workflowEntrypoint turbo mode', () => { return { handlerPromise, order, eventsCreate }; } - it('backgrounds run_started and forces optimistic start on the first delivery', async () => { - let release!: () => void; - const gate = new Promise((r) => { - release = r; + it('awaits run_started before forcing optimistic start on the first delivery', async () => { + let releaseRunStarted!: () => void; + const runStartedGate = new Promise((resolve) => { + releaseRunStarted = resolve; + }); + let releaseStepStarted!: () => void; + const stepStartedGate = new Promise((resolve) => { + releaseStepStarted = resolve; }); const { handlerPromise, order, eventsCreate } = await driveTurbo({ runId: 'wrun_turbo_first', attempt: 1, source: oneStepWorkflow, - runStartedGate: gate, + runStartedGate, + stepStartedGate, }); - // The body runs while run_started is still in flight — proving run_started - // was backgrounded AND optimistic start was forced (the env flag is off). - // The full VM replay leading up to the body can exceed vi.waitFor's default - // 1s timeout on slow CI runners (notably Windows), so widen it. - await vi.waitFor(() => expect(order).toContain('body'), { - timeout: 15_000, - }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(order).not.toContain('body'); expect(order).not.toContain('run_started_resolved'); - // The lazy step_started is chained on the run-ready barrier, so it is not - // even issued until run_started lands. expect(order).not.toContain('step_started_called'); - release(); - const res = await handlerPromise; - expect(res.status).toBe(204); - // After release: step_started fires, ordered strictly after run_started. - expect(order).toContain('step_started_called'); + releaseRunStarted(); + await vi.waitFor(() => expect(order).toContain('body'), { + timeout: 15_000, + }); expect(order.indexOf('run_started_resolved')).toBeLessThan( order.indexOf('step_started_called') ); - // run_started was created exactly once (idempotent first write). + expect(order).not.toContain('step_started_resolved'); + + releaseStepStarted(); + const res = await handlerPromise; + expect(res.status).toBe(204); const runStartedCreates = eventsCreate.mock.calls.filter( (c) => (c[1] as any).eventType === 'run_started' ); @@ -1667,37 +1671,6 @@ describe('workflowEntrypoint turbo mode', () => { ); }); - it('asks the World to skip the run_started preload only under turbo', async () => { - // The backgrounded run_started is used purely as a write barrier and its - // preloaded events are never read (preloadedEvents is forced to []), so - // turbo passes skipPreload to drop the wasted server-side - // list+resolve that the chained first step_started waits behind. - const turbo = await driveTurbo({ - runId: 'wrun_turbo_skip_preload', - attempt: 1, - source: oneStepWorkflow, - }); - expect((await turbo.handlerPromise).status).toBe(204); - const turboRunStarted = turbo.eventsCreate.mock.calls.find( - (c) => (c[1] as any).eventType === 'run_started' - ); - expect((turboRunStarted?.[2] as any)?.skipPreload).toBe(true); - - // A redelivery (attempt > 1) is not turbo: it awaits run_started and - // consumes the preload to skip its initial events.list, so it must NOT ask - // the server to skip it. - const redeliver = await driveTurbo({ - runId: 'wrun_turbo_skip_preload_redeliver', - attempt: 2, - source: oneStepWorkflow, - }); - expect((await redeliver.handlerPromise).status).toBe(204); - const redeliverRunStarted = redeliver.eventsCreate.mock.calls.find( - (c) => (c[1] as any).eventType === 'run_started' - ); - expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); - }); - it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 82e0a9fc1a..9be82ae629 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -541,7 +541,7 @@ export function workflowEntrypoint( // or the run_started setup below. let workflowRun: WorkflowRun | undefined; // Server-supplied per-run event ceiling from the run_started - // response. Undefined ⇒ no enforcement (older servers, turbo). + // response. Undefined means an older World supplied no limit. let maxEventsLimit: number | undefined; let workflowStartedAt = -1; let preloadedEventLog: MutableEventLog | undefined; @@ -557,10 +557,6 @@ export function workflowEntrypoint( // Epoch ms the `run_started` response was received/parsed // by the SDK — anchors RSFS (run_started → first step's // start POST). Set once, in the run_started setup below. - // Under turbo, run_started is backgrounded rather than - // awaited, so this is stamped at the point the run is - // synthesized locally instead of the real response — see - // StepLatencyTracking.rsfsAnchorMs. let runStartedReceivedAtMs: number | undefined; // Wall-clock ms spent committing hook_created events before // the first step ran, accumulated across suspension passes @@ -575,9 +571,7 @@ export function workflowEntrypoint( let preStepBlockingBeforeAttrMs: number | undefined; // Turbo mode fast-paths the very first delivery of the very - // first invocation, where it is provably safe to: background - // `run_started`, skip the initial event-log load (nothing has - // been written yet), and force optimistic inline start (no + // first invocation by forcing optimistic inline start (no // concurrent peer handler exists to race the create-claim). // `runInput` is only present on the start()-enqueued message, // and `attempt === 1` (1-based) means this is the first @@ -608,30 +602,6 @@ export function workflowEntrypoint( !replayDivergence; span?.setAttributes(Attribute.WorkflowTurbo(turbo)); - // Turbo mode only: resolves once the backgrounded - // `run_started` has landed (or rejects if it failed). Threaded - // into handleSuspension and executeStep so no step/hook/wait - // write races ahead of the run's creation. Undefined outside - // turbo, where `run_started` is awaited up front. - let runReadyBarrier: Promise | undefined; - - // Order a terminal run write (run_completed / run_failed) after - // the backgrounded run_started in turbo mode — a no-step - // workflow can otherwise reach run_completed before the run - // exists. Best-effort: a barrier rejection is swallowed for - // ordering only; if run_started truly failed the terminal write - // surfaces the real error (run not found / gone) and the message - // redelivers. No-op outside turbo. - const awaitRunReady = async (): Promise => { - if (runReadyBarrier) { - try { - await runReadyBarrier; - } catch { - // intentional: ordering barrier only — see above. - } - } - }; - // Re-invoke the orchestrator. Outside turbo this returns // `{ timeoutSeconds }`, which makes the queue reschedule the // CURRENT delivery's message. In turbo that is a trap: the @@ -978,194 +948,84 @@ export function workflowEntrypoint( } : {}), }; - const recordRunStartedCreateStart = ( - skipPreload: boolean - ) => { - span?.addEvent('workflow.run_started.create.start', { - 'workflow.run_started.skip_preload': skipPreload, - }); - }; - - if (turbo && runInput) { - // Turbo: background `run_started` and synthesize the run - // entity locally so replay can begin without waiting for - // the round-trip. Safe here because this is the first - // delivery of the first invocation — start() created the - // run moments ago and no events have been written yet. The - // barrier is consumed by every downstream write (suspension - // handler, optimistic step_started, terminal run writes) so - // nothing is written before the run exists. - recordRunStartedCreateStart(true); - const startedPromise = world.events.create( + let startedRun: StartedWorkflowRun; + try { + span?.addEvent('workflow.run_started.create.start'); + const result = await world.events.create( runId, runStartedEvent, - // We background this purely as a write barrier and - // never read its preloaded events, so skip the - // run_started event-log preload. That trims the - // run_started request the chained first step_started - // waits on — shortening time-to-second-step — and the - // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } - ); - runReadyBarrier = startedPromise; - // Turbo backgrounds run_started, so the non-turbo assignment - // below never runs — thread the per-run event ceiling off the - // backgrounded response here instead. The guard re-checks - // maxEventsLimit every loop iteration, so a value that lands - // shortly after start still enforces well before a runaway - // log approaches the ceiling. - startedPromise.then( - (r) => { - const limit = clampMaxEvents(r?.maxEvents); - if (limit !== undefined) maxEventsLimit = limit; - }, - () => {} + { requestId } ); - // Attach a no-op rejection handler so an early failure - // never surfaces as an unhandledRejection before a consumer - // (await/then) is attached; consumers still observe it. - startedPromise.catch(() => {}); - // Skip the initial events.list: nothing has been written to - // the log yet on a first delivery (run_started is still in - // flight). An empty preload routes iteration 1 through - // the no-load preloaded branch; iteration 2 then takes the - // existing post-preloaded full reload to pick up a cursor - // without a spurious "cursor missing" warning. - preloadedEventLog = { events: [], cursor: null }; - const now = new Date(); - workflowRun = { - runId, - status: 'running', - deploymentId: runInput.deploymentId, - workflowName: runInput.workflowName, - specVersion: runInput.specVersion, - executionContext: runInput.executionContext, - input: runInput.input, - // Seed attributes from start() ride along in `runInput` - // (they live in `run_created`'s eventData, not separate - // `attr_set` events), so the synthesized snapshot carries - // them even though we skip the initial events.list. This - // is correct ONLY while attributes are write-only: - // there is no in-workflow read API today (see workflow.ts - // "structural until a read API is introduced"), so the - // empty preloaded log can't diverge on a read. If a read - // API is ever added it MUST read from this snapshot, not - // by replaying run_created/attr_set events — otherwise - // turbo's empty initial log would surface seed attributes - // as `{}` on the first delivery only. - attributes: runInput.attributes ?? {}, - startedAt: now, - createdAt: now, - updatedAt: now, - }; - workflowStartedAt = +now; - // See the `runStartedReceivedAtMs` declaration above: - // turbo synthesizes the run before the real - // `run_started` response lands, so anchor RSFS here - // rather than at an actual response instant. - runStartedReceivedAtMs = +now; - span?.setAttributes({ - ...Attribute.WorkflowRunStatus('running'), - ...Attribute.WorkflowStartedAt(workflowStartedAt), - }); - } else { - let startedRun: StartedWorkflowRun; - try { - recordRunStartedCreateStart(false); - const result = await world.events.create( - runId, - runStartedEvent, - { requestId } - ); - startedRun = result.run; - workflowRun = startedRun; - maxEventsLimit = clampMaxEvents(result.maxEvents); - // Anchors RSFS — see the declaration above. - runStartedReceivedAtMs = Date.now(); - - if (result.events?.length) { - let events = result.events; - let cursor = result.cursor ?? null; - if (result.hasMore) { - const loaded = await loadWorkflowRunEvents( - runId, - cursor ?? undefined - ); - if (cursor) { - const preloadedIds = new Set( - events.map((event) => event.eventId) - ); - events = events.concat( - loaded.events.filter( - (event) => !preloadedIds.has(event.eventId) - ) - ); - } else { - events = loaded.events; - } - cursor = loaded.cursor ?? cursor; - } - preloadedEventLog = { events, cursor }; - } - } catch (err) { - // Run was concurrently completed/failed/cancelled - if ( - EntityConflictError.is(err) || - RunExpiredError.is(err) - ) { - // EntityConflictError: run was concurrently - // completed/failed/cancelled during setup. - // RunExpiredError: run already in terminal state. - // In both cases, skip processing this message. - runtimeLogger.info( - 'Run already finished during setup, skipping', - { workflowRunId: runId, message: err.message } + startedRun = result.run; + workflowRun = startedRun; + maxEventsLimit = clampMaxEvents(result.maxEvents); + runStartedReceivedAtMs = Date.now(); + + if (result.events?.length) { + let events = result.events; + let cursor = result.cursor ?? null; + if (result.hasMore) { + const loaded = await loadWorkflowRunEvents( + runId, + cursor ?? undefined ); - return; - } else { - const errorCode = getWorkflowSetupErrorCode(err); - if (!errorCode) { - throw err; + if (cursor) { + const preloadedIds = new Set( + events.map((event) => event.eventId) + ); + events = events.concat( + loaded.events.filter( + (event) => !preloadedIds.has(event.eventId) + ) + ); + } else { + events = loaded.events; } - await recordFatalRunError({ - world, - workflowRun, - runId, - requestId, - err, - errorCode, - logMessage: - 'Fatal runtime error during workflow setup', - }); - return; + cursor = loaded.cursor ?? cursor; } + preloadedEventLog = { events, cursor }; } - workflowStartedAt = +startedRun.startedAt; - - span?.setAttributes({ - ...Attribute.WorkflowRunStatus(startedRun.status), - ...Attribute.WorkflowStartedAt(workflowStartedAt), - }); - - if (startedRun.status !== 'running') { - // Workflow has already completed or failed, so we can skip it + } catch (err) { + if ( + EntityConflictError.is(err) || + RunExpiredError.is(err) + ) { runtimeLogger.info( - 'Workflow already completed or failed, skipping', - { - workflowRunId: runId, - status: startedRun.status, - } + 'Run already finished during setup, skipping', + { workflowRunId: runId, message: err.message } ); - - // TODO: for `cancel`, we actually want to propagate a WorkflowCancelled event - // inside the workflow context so the user can gracefully exit. this is SIGTERM - // TODO: furthermore, there should be a timeout or a way to force cancel SIGKILL - // so that we actually exit here without replaying the workflow at all, in the case - // the replaying the workflow is itself failing. - return; } - } // end else (non-turbo run_started) + const errorCode = getWorkflowSetupErrorCode(err); + if (!errorCode) throw err; + await recordFatalRunError({ + world, + workflowRun, + runId, + requestId, + err, + errorCode, + logMessage: 'Fatal runtime error during workflow setup', + }); + return; + } + workflowStartedAt = +startedRun.startedAt; + + span?.setAttributes({ + ...Attribute.WorkflowRunStatus(startedRun.status), + ...Attribute.WorkflowStartedAt(workflowStartedAt), + }); + + if (startedRun.status !== 'running') { + runtimeLogger.info( + 'Workflow already completed or failed, skipping', + { + workflowRunId: runId, + status: startedRun.status, + } + ); + return; + } } // end if (!workflowRun) // Resolve the encryption key for this run's deployment. @@ -1189,7 +1049,6 @@ export function workflowEntrypoint( ); // Main replay loop - // biome-ignore lint/correctness/noConstantCondition: intentional loop while (true) { loopIteration++; @@ -1536,11 +1395,6 @@ export function workflowEntrypoint( events, encryptionKey, replayPayloadCache, - // Turbo: the end-of-run drain inside runWorkflow commits - // fire-and-forget `*_created` events before the terminal - // `awaitRunReady()` below, so gate those writes on the - // backgrounded run_started too. Undefined outside turbo. - runReadyBarrier, world.capabilities ); await payloadPrewarm; @@ -1558,10 +1412,6 @@ export function workflowEntrypoint( // result. The catch below lets PreconditionFailedError // propagate to the queue for re-invocation. try { - // Turbo: a workflow that finishes with no steps reaches - // here before the backgrounded run_started; order the - // terminal write after it so the run exists. - await awaitRunReady(); await world.events.create( runId, { @@ -1671,7 +1521,6 @@ export function workflowEntrypoint( span, requestId, eventLog: suspensionLog, - runReadyBarrier, }); } catch (suspensionError) { // A suspension create whose stale (412) rejection @@ -1711,9 +1560,6 @@ export function workflowEntrypoint( } ); try { - // Turbo: order the terminal write after the - // backgrounded run_started so the run exists. - await awaitRunReady(); await world.events.create( runId, { @@ -2254,16 +2100,13 @@ export function workflowEntrypoint( // step qualifies for TTFS/STSO measurement. Only the // batch's first step carries the tracking so a // parallel batch emits one sample per scheduling gap, - // not one per sibling. Turbo's synthesized run - // snapshot has a local-clock createdAt, so under - // turbo only the run-id ULID timestamp is trusted. + // not one per sibling. const latencyTracking = computeStepLatencyTracking({ events: cachedEvents ?? [], invocationStartedClean: invocationStartedClean === true, runCreatedAtMs: - runIdCreatedAt(runId) ?? - (turbo ? undefined : +workflowRun.createdAt), + runIdCreatedAt(runId) ?? +workflowRun.createdAt, runStartedReceivedAtMs, replayMs: replayDurationMs, preStepBlockingMs, @@ -2356,18 +2199,14 @@ export function workflowEntrypoint( // as in flight here and suppress the // immediate requeue (workflow#2780). ownerMessageId: metadata.messageId, - // Turbo: force optimistic start and hold the - // lazy step_started until the backgrounded - // run_started lands (the body still runs - // immediately). Both are undefined/false - // outside turbo. + // Turbo forces optimistic start on its first + // inline step. forceOptimisticStart, // Guard-enforced batches with an open hook // await the claim before running the body, so // a 412-fenced step never executes user code — // see suppressOptimisticStart above. suppressOptimisticStart, - runReadyBarrier, stateUpdatedAt: inlineClaimStateUpdatedAt, ...(stepIndex === 0 && s.lazyStepInput !== undefined && @@ -2766,9 +2605,6 @@ export function workflowEntrypoint( // also need the loaded event log, which is scoped to the // replay `try` above and not available in this catch. try { - // Turbo: order the terminal write after the - // backgrounded run_started so the run exists. - await awaitRunReady(); await world.events.create( runId, { diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 619c50f890..e30af271a6 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -284,10 +284,8 @@ export function isOptimisticInlineStartExplicitlyDisabled(): boolean { /** * Whether "turbo mode" is enabled. Turbo mode fast-paths the *first delivery of * the first invocation* of a run (detected by the entrypoint via `runInput` - * presence + `metadata.attempt === 1`): it backgrounds the `run_started` event - * creation, skips the initial event-log load (nothing has been written yet), - * and forces optimistic inline step start for that invocation — independent of - * `WORKFLOW_OPTIMISTIC_INLINE_START`. + * presence + `metadata.attempt === 1`) and forces optimistic inline step start + * for that invocation, independent of `WORKFLOW_OPTIMISTIC_INLINE_START`. * * Forcing optimistic start is safe here because the first delivery has no * concurrent peer handler to race the step create-claim, so a step body runs diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 2d22ff9b23..72db0ebb6c 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -167,15 +167,6 @@ export interface StepExecutorParams { * meaningful together with `lazyStepInput` (a brand-new lazy step). */ forceOptimisticStart?: boolean; - /** - * Turbo mode only: a promise that resolves once the backgrounded - * `run_started` has landed. When set, the lazy/optimistic `step_started` is - * chained on it so the step is never created before its run exists. The body - * still runs immediately against locally-synthesized state — only the network - * write waits — so the `run_started` round-trip overlaps the body. `undefined` - * outside turbo, where `run_started` was already awaited up front. - */ - runReadyBarrier?: Promise; /** * Latency telemetry (TTFS / STSO): eligibility and anchor timestamps decided * by the orchestrator. When set, this executor computes the final values @@ -297,13 +288,6 @@ export async function executeStep( // return `skipped` and never write the failure twice. if (params.lazyStepInput !== undefined) { try { - // Turbo: this lazy `step_started` must not precede the backgrounded - // `run_started`. Order it after the run-ready barrier (best-effort — - // a barrier rejection means the run doesn't exist, and the create - // below surfaces the real error). No-op outside turbo. - if (params.runReadyBarrier) { - await params.runReadyBarrier.catch(() => {}); - } await world.events.create(workflowRunId, { eventType: 'step_started', specVersion: SPEC_VERSION_CURRENT, @@ -533,42 +517,30 @@ export async function executeStep( }; if (optimisticStart) { - // Chain the lazy `step_started` on the run-ready barrier (turbo mode): - // the step can't be created before its run exists, but the body below - // runs immediately against synthesized state, so the `run_started` - // round-trip overlaps the body rather than blocking it. Outside turbo the - // barrier is undefined and this is a plain create. - const startedPromise = (params.runReadyBarrier ?? Promise.resolve()).then( - () => { - // Taken right before the create fires, not before the barrier — - // RSFS measures the run_started-to-POST stretch, and the barrier - // wait IS part of that stretch under turbo. - stepStartPostSentAtMs = Date.now(); - return world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), - }, - }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - params.stateUpdatedAt !== undefined - ? { stateUpdatedAt: params.stateUpdatedAt } - : undefined - ); - } + stepStartPostSentAtMs = Date.now(); + const startedPromise = world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, + }, + // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale + // (412) rejection surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + params.stateUpdatedAt !== undefined + ? { stateUpdatedAt: params.stateUpdatedAt } + : undefined ); optimisticStartSettled = startedPromise.then( () => ({ ok: true as const }), @@ -861,13 +833,6 @@ export async function executeStep( preCompletionOps, closureVars: hydratedInput.closureVars, encryptionKey, - // Turbo optimistic start runs this body before `run_started` is - // durable. Expose the barrier so a direct step-body world write - // (e.g. `setAttributes`) can order itself after the - // run exists. Undefined on the await path (run already durable). - runReadyBarrier: optimisticStart - ? params.runReadyBarrier - : undefined, }, () => stepFn.apply(thisVal, args) ); @@ -908,11 +873,7 @@ export async function executeStep( globalThis, false, false, - compression, - // Turbo optimistic start: a returned stream is piped to the server - // after the body but within this same op flush, so gate its first - // write on the run-ready barrier. Undefined on the await path. - optimisticStart ? params.runReadyBarrier : undefined + compression ); const durationMs = Date.now() - startTime; dehydrateSpan?.setAttributes({ diff --git a/packages/core/src/runtime/step-latency.ts b/packages/core/src/runtime/step-latency.ts index 0a4d9b08e3..1d4af28f72 100644 --- a/packages/core/src/runtime/step-latency.ts +++ b/packages/core/src/runtime/step-latency.ts @@ -61,14 +61,7 @@ export interface StepLatencyTracking { /** * Epoch ms the `run_started` response was received/parsed by the SDK. * Present only when the step qualifies for RSFS — the same eligibility as - * TTFS (see {@link computeStepLatencyTracking}), plus a recoverable - * anchor. In turbo mode, `run_started` is backgrounded rather than - * awaited, so this is stamped at the point the runtime synthesizes the - * run locally and begins replay instead of at the real response; the - * first step's start POST is still chained on the real `run_started` - * promise (see step-executor.ts), so RSFS still ends up measuring the - * genuine run_started-to-first-step-POST stretch even though the two - * halves overlap under turbo. + * TTFS (see {@link computeStepLatencyTracking}), plus a recoverable anchor. */ rsfsAnchorMs?: number; /** @@ -174,11 +167,7 @@ export function computeStepLatencyTracking(params: { invocationStartedClean: boolean; /** Epoch ms of run creation, if recoverable. Absent disqualifies TTFS. */ runCreatedAtMs: number | undefined; - /** - * Epoch ms the `run_started` response was received/parsed by the SDK (or, - * under turbo, the instant the run was synthesized locally — see - * {@link StepLatencyTracking.rsfsAnchorMs}). Absent disqualifies RSFS. - */ + /** Epoch ms the `run_started` response was received/parsed by the SDK. */ runStartedReceivedAtMs: number | undefined; /** * Wall-clock ms this suspension's `runWorkflow` call spent executing diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 921986d7bd..3993eaedf8 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -46,16 +46,6 @@ export interface SuspensionHandlerParams { * suspension) ensures a retry never re-issues an already-created event. */ eventLog?: MutableEventLog; - /** - * Turbo mode only: a promise that resolves once the backgrounded - * `run_started` has landed (the run exists). When present, every world write - * this suspension performs (`hook_created`, `wait_created`, eager overflow - * `step_created`, …) is gated on it so the write never races ahead of the - * run's creation. The pure inline hot path defers all of its steps and writes - * nothing here, so it never awaits this barrier. `undefined` outside turbo, - * where `run_started` was already awaited up front. - */ - runReadyBarrier?: Promise; } /** @@ -204,28 +194,9 @@ export async function handleSuspension({ span, requestId, eventLog, - runReadyBarrier, }: SuspensionHandlerParams): Promise { const runId = run.runId; - // Turbo mode: hold every world write below until the backgrounded - // `run_started` has *settled*, so we never write a step/hook/wait event for a - // run that does not exist yet. A no-op outside turbo (barrier undefined) and - // on the pure inline hot path, which defers all steps and writes nothing. - // Awaiting the same (usually already-settled) promise more than once is cheap. - // A barrier rejection is swallowed for ordering only: if `run_started` truly - // failed the run does not exist, so the subsequent write surfaces the real - // error (run not found / gone) and the message redelivers. - const ensureRunReady = async (): Promise => { - if (runReadyBarrier) { - try { - await runReadyBarrier; - } catch { - // intentional: ordering barrier only — see above. - } - } - }; - // Create an event with the optimistic-concurrency guard when the caller // supplied a loaded event log; otherwise create it directly (callers without // a replay snapshot, e.g. tests). The guard reloads + retries on a stale @@ -344,7 +315,6 @@ export async function handleSuspension({ if (hookItemsByToken.size > 0) { const hookPhaseStart = Date.now(); - await ensureRunReady(); await Promise.all( [...hookItemsByToken.values()].map(async (items) => { for (const queueItem of items) { @@ -404,7 +374,6 @@ export async function handleSuspension({ ); if (hooksNeedingAbort.length > 0) { - await ensureRunReady(); await Promise.all( hooksNeedingAbort.map(async (queueItem) => { try { @@ -558,7 +527,6 @@ export async function handleSuspension({ }, }; try { - await ensureRunReady(); await createGuarded(stepEvent, { requestId }); createdStepCorrelationIds.add(queueItem.correlationId); } catch (err) { @@ -591,7 +559,6 @@ export async function handleSuspension({ }, }; try { - await ensureRunReady(); await createGuarded(waitEvent, { requestId }); } catch (err) { if (EntityConflictError.is(err)) { @@ -613,7 +580,6 @@ export async function handleSuspension({ ops.push( (async () => { try { - await ensureRunReady(); await world.events.create( runId, { diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 661cd5bd9a..e7a8d5a79a 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -1046,16 +1046,7 @@ function recordStreamClose( } export class WorkflowServerWritableStream extends WritableStream { - /** - * @param runReadyBarrier Turbo mode only: a promise that resolves once the - * backgrounded `run_started` has landed. When the step body runs - * optimistically (before `run_started` is durable), the first chunk write to - * a brand-new stream would otherwise reach the World before the run exists - * and be rejected as run-not-found. Awaiting this once before the first - * flush/close orders the write after the run's creation. `undefined` outside - * turbo and on the await path, where the run was already durable. - */ - constructor(runId: string, name: string, runReadyBarrier?: Promise) { + constructor(runId: string, name: string) { if (typeof runId !== 'string') { throw new WorkflowRuntimeError( `"runId" must be a string, got "${typeof runId}"` @@ -1066,22 +1057,6 @@ export class WorkflowServerWritableStream extends WritableStream { } const worldPromise = getWorldLazy(); - // Hold the first server write until the run exists (turbo optimistic - // start). Awaited once, then cleared so later flushes pay nothing. The - // rejection is swallowed for ordering only: if `run_started` truly failed - // the run does not exist, so the write below surfaces the real error. - let pendingRunReady: Promise | undefined = runReadyBarrier; - const ensureRunReady = async (): Promise => { - if (pendingRunReady) { - try { - await pendingRunReady; - } catch { - // intentional: ordering barrier only — see above. - } - pendingRunReady = undefined; - } - }; - // ------------------------------------------------------------------ // Group-commit buffering. // @@ -1186,18 +1161,14 @@ export class WorkflowServerWritableStream extends WritableStream { * behavior, now independent of how the producer pipes. */ /** - * Send one group to the server: gate on run readiness, then one - * `writeMulti` (or sequential `write`s when the world lacks it). Emits - * the write-flush span; dwell is measured up to just before the RPC so a - * turbo run-ready barrier wait counts as buffer dwell, matching the - * pre-group-commit telemetry. + * Send one group to the server with one `writeMulti` call (or sequential + * `write`s when the world lacks it). */ const sendGroup = async ( group: Uint8Array[], bytes: number, groupT0: number | undefined ): Promise => { - await ensureRunReady(); const world = await worldPromise; const dispatchAt = Date.now(); if (typeof world.streams.writeMulti === 'function' && group.length > 1) { @@ -1401,11 +1372,6 @@ export class WorkflowServerWritableStream extends WritableStream { // closed — the server fences post-close writes. await drain(); - // A close with an empty buffer skips the dispatch path (and its - // barrier), but can itself be the first write to a brand-new - // stream — gate it too. - await ensureRunReady(); - const world = await worldPromise; const closeStart = Date.now(); await world.streams.close(runId, name); @@ -1710,13 +1676,7 @@ export function getExternalReducers( ops: Promise[], runId: string, cryptoKey: EncryptionKeyParam, - framedByteStreams = false, - // Turbo optimistic start: a nested `ReadableStream` found while serializing - // is piped to its own server stream independently of the outer sink, so its - // first chunk can race `run_started`. Thread the run-ready barrier into that - // sink so the write orders after the run exists. Undefined outside turbo / - // on the await path. - runReadyBarrier?: Promise + framedByteStreams = false ): Partial { return { ...getAllBaseReducers(global), @@ -1738,11 +1698,7 @@ export function getExternalReducers( const name = `strm_${streamId}`; const type = getStreamType(value); - const writable = new WorkflowServerWritableStream( - runId, - name, - runReadyBarrier - ); + const writable = new WorkflowServerWritableStream(runId, name); if (type === 'bytes') { if (framedByteStreams) { ops.push(value.pipeThrough(getByteFramingStream()).pipeTo(writable)); @@ -1759,8 +1715,7 @@ export function getExternalReducers( ops, runId, cryptoKey, - framedByteStreams, - runReadyBarrier + framedByteStreams ), cryptoKey ) @@ -1950,12 +1905,7 @@ function getStepReducers( ops: Promise[], runId: string, cryptoKey: EncryptionKeyParam, - framedByteStreams = false, - // Turbo optimistic start: a returned `ReadableStream` is piped to the server - // after the body but within the same op flush, so its first chunk can race - // `run_started`. Thread the run-ready barrier into the sink so that write - // orders after the run exists. Undefined outside turbo / on the await path. - runReadyBarrier?: Promise + framedByteStreams = false ): Partial { return { ...getAllBaseReducers(global), @@ -1991,11 +1941,7 @@ function getStepReducers( type = getStreamType(value); framing = type === 'bytes' && framedByteStreams ? 'framed-v1' : framing; - const writable = new WorkflowServerWritableStream( - runId, - name, - runReadyBarrier - ); + const writable = new WorkflowServerWritableStream(runId, name); if (type === 'bytes') { if (framing === 'framed-v1') { ops.push( @@ -2014,8 +1960,7 @@ function getStepReducers( ops, runId, cryptoKey, - framedByteStreams, - runReadyBarrier + framedByteStreams ), cryptoKey ) @@ -3383,23 +3328,12 @@ export async function dehydrateStepReturnValue( global: Record = globalThis, v1Compat = false, framedByteStreams = false, - compression = false, - // Turbo optimistic start: order the first chunk of a returned stream after - // the backgrounded `run_started`. Threaded into the step reducers' stream - // sink. Undefined outside turbo / on the await path. - runReadyBarrier?: Promise + compression = false ): Promise { if (v1Compat) { const str = stringify( value, - getStepReducers( - global, - ops, - runId, - key, - framedByteStreams, - runReadyBarrier - ) + getStepReducers(global, ops, runId, key, framedByteStreams) ); return revive(str); } @@ -3408,14 +3342,7 @@ export async function dehydrateStepReturnValue( const result = await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers( - getStepReducers( - global, - ops, - runId, - key, - framedByteStreams, - runReadyBarrier - ) + getStepReducers(global, ops, runId, key, framedByteStreams) ), compression, compressionStats, diff --git a/packages/core/src/set-attributes.test.ts b/packages/core/src/set-attributes.test.ts index 96c5861199..1a9e9b92ac 100644 --- a/packages/core/src/set-attributes.test.ts +++ b/packages/core/src/set-attributes.test.ts @@ -113,60 +113,6 @@ describe('setAttributes (host-side)', () => { ); }); - it('waits for runReadyBarrier before posting the event (turbo optimistic start)', async () => { - const order: string[] = []; - const create = vi.fn().mockImplementation(async () => { - order.push('create'); - return {}; - }); - globals[WORLD_CACHE] = { - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - }; - - let releaseBarrier!: () => void; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = () => { - order.push('barrier'); - resolve(); - }; - }); - - const call = contextStorage.run({ ...stepContext(), runReadyBarrier }, () => - setAttributes({ phase: 'ready' }) - ); - - // The body ran before run_started is durable: the write must not fire yet. - await Promise.resolve(); - expect(create).not.toHaveBeenCalled(); - - releaseBarrier(); - await call; - - // The attr_set create lands strictly after the run-ready barrier resolves. - expect(order).toEqual(['barrier', 'create']); - }); - - it('still posts when the runReadyBarrier rejects (write surfaces the real error)', async () => { - const create = vi.fn().mockResolvedValue({}); - globals[WORLD_CACHE] = { - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - }; - - const runReadyBarrier = Promise.reject(new Error('run_started failed')); - // Pre-attach a catch so the rejection never surfaces as unhandled. - runReadyBarrier.catch(() => {}); - - await contextStorage.run({ ...stepContext(), runReadyBarrier }, () => - setAttributes({ phase: 'ready' }) - ); - - // Barrier rejection is swallowed for ordering only — the write still fires - // and would surface a genuine run-not-found error from the World itself. - expect(create).toHaveBeenCalledTimes(1); - }); - it('keeps the deprecated experimental_setAttributes alias working', async () => { expect(experimental_setAttributes).toBe(setAttributes); }); diff --git a/packages/core/src/set-attributes.ts b/packages/core/src/set-attributes.ts index 072714a616..7eb7920306 100644 --- a/packages/core/src/set-attributes.ts +++ b/packages/core/src/set-attributes.ts @@ -35,20 +35,6 @@ export async function setAttributes( const changes = normalizeAttributeChanges(attrs, options); if (changes.length === 0) return; - // Turbo optimistic start runs the step body before the backgrounded - // `run_started` is durable. Order this `attr_set` after the run exists so it - // never reaches the World before the run does (which would be rejected as - // run-not-found). A no-op outside turbo (barrier undefined) and on the await - // path. The rejection is swallowed for ordering only: if `run_started` truly - // failed the run does not exist, so the create below surfaces the real error. - if (store.runReadyBarrier) { - try { - await store.runReadyBarrier; - } catch { - // intentional: ordering barrier only — see above. - } - } - const world = await getWorldLazy(); await world.events.create(runId, { eventType: 'attr_set', diff --git a/packages/core/src/step/context-storage.ts b/packages/core/src/step/context-storage.ts index 10b665bbcf..8fa82830e0 100644 --- a/packages/core/src/step/context-storage.ts +++ b/packages/core/src/step/context-storage.ts @@ -59,16 +59,6 @@ export type StepContext = { closureVars?: Record; encryptionKey?: CryptoKey; writables?: Map; - /** - * Turbo mode only: a promise that resolves once the backgrounded - * `run_started` has landed (the run exists). Set when the step body runs - * optimistically — before `run_started`/`step_started` are confirmed — so a - * direct step-body world write (e.g. `setAttributes`, which - * resolves to a host-side `attr_set` create) can gate on it and never race - * ahead of the run's creation. `undefined` outside turbo and on the await - * path, where `run_started` was already durable before the body ran. - */ - runReadyBarrier?: Promise; }; /** diff --git a/packages/core/src/step/writable-stream.ts b/packages/core/src/step/writable-stream.ts index 91b8534c26..5e3463b23e 100644 --- a/packages/core/src/step/writable-stream.ts +++ b/packages/core/src/step/writable-stream.ts @@ -78,18 +78,7 @@ export function getWritable( // version skew protection) is on this same SDK version, so byte-stream // framing is always safe here. const serialize = getSerializeStream( - // In turbo optimistic start the body runs before `run_started` is durable. - // Thread the run-ready barrier so that a nested ReadableStream written into - // this writable is piped to its own server stream only after the run - // exists. Undefined outside turbo / on the await path. - getExternalReducers( - globalThis, - ctx.ops, - runId, - ctx.encryptionKey, - true, - ctx.runReadyBarrier - ), + getExternalReducers(globalThis, ctx.ops, runId, ctx.encryptionKey, true), ctx.encryptionKey ); @@ -97,14 +86,7 @@ export function getWritable( // their writer lock, not only when the stream is explicitly closed. // Without this, Vercel functions hang until the runtime timeout because // .pipeTo() only resolves on stream close. - // In turbo optimistic start the body runs before `run_started` is durable; - // pass the run-ready barrier so the first server write orders after the run - // exists. Undefined outside turbo / on the await path (run already durable). - const serverWritable = new WorkflowServerWritableStream( - runId, - name, - ctx.runReadyBarrier - ); + const serverWritable = new WorkflowServerWritableStream(runId, name); const state = createFlushableState(); ctx.ops.push(state.promise); diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index d66faa9fb1..5cda61f41f 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -214,11 +214,10 @@ export const StepStsoMs = SemanticConvention('step.stso_ms'); /** * Client-measured run_started-to-first-step latency in milliseconds: the - * `run_started` response landing (or, under turbo, the local run synthesis - * instant) → this step's start POST being issued. A sub-window of ttfs that - * isolates replay overhead from the run-creation queue hop. Only present on - * the run's first step execution when it qualified for measurement (see - * runtime/step-latency.ts). + * `run_started` response landing → this step's start POST being issued. A + * sub-window of ttfs that isolates replay overhead from the run-creation queue + * hop. Only present on the run's first step execution when it qualified for + * measurement (see runtime/step-latency.ts). */ export const StepRsfsMs = SemanticConvention('step.rsfs_ms'); diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 2d903b94a7..8e8f299d5e 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -5433,13 +5433,7 @@ describe('runWorkflow', () => { } }, 30_000); - describe('turbo: end-of-run drain gates writes on runReadyBarrier', () => { - // A workflow that creates a fire-and-forget hook and then returns - // synchronously never suspends, so its `hook_created` event is committed by - // the end-of-run drain inside runWorkflow — *before* the runtime's terminal - // `awaitRunReady()`. In turbo (`run_started` backgrounded), that write must - // still be ordered after the run exists, so runWorkflow threads the barrier - // into the drain. These tests pin that gating. + describe('end-of-run drain', () => { const FIRE_AND_FORGET_HOOK = `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; async function workflow() { createHook({ token: 'fire-and-forget' }); @@ -5452,14 +5446,6 @@ describe('runWorkflow', () => { return 'done'; }`; - // `void sleep(...)` is the wait equivalent: it enqueues a wait_created the - // workflow never awaits, drained the same way as the hook above. - const FIRE_AND_FORGET_WAIT = `const sleep = globalThis[Symbol.for("WORKFLOW_SLEEP")]; - async function workflow() { - void sleep('1d'); - return 'done'; - }`; - async function makeRun(): Promise { const ops: Promise[] = []; return { @@ -5491,7 +5477,6 @@ describe('runWorkflow', () => { [], noEncryptionKey, undefined, - undefined, { hookRetention: { active: false } } ) ).rejects.toThrow( @@ -5499,130 +5484,7 @@ describe('runWorkflow', () => { ); }); - it('does not write hook_created until the runReadyBarrier resolves', async () => { - let releaseBarrier: () => void = () => {}; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = resolve; - }); - - const create = vi.fn(async () => ({ - event: { eventType: 'hook_created' as const }, - })); - setWorld({ - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - streams: { write: vi.fn(), close: vi.fn() }, - } as any); - - const runPromise = runWorkflow( - `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, - await makeRun(), - [], - noEncryptionKey, - undefined, - runReadyBarrier - ).then(() => 'completed' as const); - - // The body returns synchronously, but the drain's hook_created write — and - // therefore runWorkflow's own completion — must stay blocked behind the - // still-pending backgrounded run_started. A fixed-time race (not a fixed - // wait-then-assert) makes this robust to VM setup latency: the window only - // has to exceed the drain's own work, never a calibrated guess at it. - const winner = await Promise.race([ - runPromise, - new Promise<'pending'>((resolve) => - setTimeout(() => resolve('pending'), 250) - ), - ]); - expect(winner).toBe('pending'); - expect(create).not.toHaveBeenCalled(); - - // Once run_started lands, the drain proceeds and runWorkflow resolves. - releaseBarrier(); - expect(await runPromise).toBe('completed'); - - expect(create).toHaveBeenCalledTimes(1); - expect(create.mock.calls[0][1]).toMatchObject({ - eventType: 'hook_created', - }); - }); - - it('does not write wait_created until the runReadyBarrier resolves', async () => { - let releaseBarrier: () => void = () => {}; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = resolve; - }); - - const create = vi.fn(async () => ({ - event: { eventType: 'wait_created' as const }, - })); - setWorld({ - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - streams: { write: vi.fn(), close: vi.fn() }, - } as any); - - const runPromise = runWorkflow( - `${FIRE_AND_FORGET_WAIT}${getWorkflowTransformCode('workflow')}`, - await makeRun(), - [], - noEncryptionKey, - undefined, - runReadyBarrier - ).then(() => 'completed' as const); - - // Same gating as the hook case: a fire-and-forget wait drained at - // completion must not write wait_created before the backgrounded - // run_started lands. - const winner = await Promise.race([ - runPromise, - new Promise<'pending'>((resolve) => - setTimeout(() => resolve('pending'), 250) - ), - ]); - expect(winner).toBe('pending'); - expect(create).not.toHaveBeenCalled(); - - releaseBarrier(); - expect(await runPromise).toBe('completed'); - - expect(create).toHaveBeenCalledTimes(1); - expect(create.mock.calls[0][1]).toMatchObject({ - eventType: 'wait_created', - }); - }); - - it('still writes hook_created when the runReadyBarrier rejects (ordering only)', async () => { - const create = vi.fn(async () => ({ - event: { eventType: 'hook_created' as const }, - })); - setWorld({ - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - streams: { write: vi.fn(), close: vi.fn() }, - } as any); - - // A rejected barrier is swallowed for ordering: if run_started truly - // failed, the run does not exist and the write itself surfaces the error. - const rejected = Promise.reject(new Error('run_started failed')); - rejected.catch(() => {}); - - await runWorkflow( - `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, - await makeRun(), - [], - noEncryptionKey, - undefined, - rejected - ); - - expect(create).toHaveBeenCalledTimes(1); - expect(create.mock.calls[0][1]).toMatchObject({ - eventType: 'hook_created', - }); - }); - - it('writes hook_created without blocking when no barrier (non-turbo)', async () => { + it('writes fire-and-forget hooks before returning', async () => { const create = vi.fn(async () => ({ event: { eventType: 'hook_created' as const }, })); @@ -5632,8 +5494,6 @@ describe('runWorkflow', () => { streams: { write: vi.fn(), close: vi.fn() }, } as any); - // No barrier (the await path already awaited run_started up front): the - // drain writes immediately. await runWorkflow( `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, await makeRun(), diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4dfd069e4d..8cf97ffad3 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -68,12 +68,7 @@ async function drainPendingQueueItems( pendingQueue: Map, vmGlobalThis: typeof globalThis, workflowRun: WorkflowRun, - outcome: 'completed' | 'failed', - /** - * In turbo mode, gates final `*_created` writes on backgrounded - * `run_started`. Undefined when `run_started` is awaited. - */ - runReadyBarrier?: Promise + outcome: 'completed' | 'failed' ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -99,7 +94,6 @@ async function drainPendingQueueItems( suspension: synthesized, world, run: workflowRun, - runReadyBarrier, }); } catch (err) { runtimeLogger.warn( @@ -125,13 +119,6 @@ export async function runWorkflow( replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache( encryptionKey ), - /** - * Turbo mode only: resolves once the backgrounded `run_started` has landed. - * Threaded into the end-of-run drain so fire-and-forget `*_created` writes - * committed at workflow completion order after the run's creation. Undefined - * outside turbo, where `run_started` is awaited up front. - */ - runReadyBarrier?: Promise, /** * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. @@ -850,8 +837,7 @@ export async function runWorkflow( workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'completed', - runReadyBarrier + 'completed' ); return dehydrated; @@ -867,8 +853,7 @@ export async function runWorkflow( workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'failed', - runReadyBarrier + 'failed' ); throw err; diff --git a/packages/core/src/writable-stream-telemetry.test.ts b/packages/core/src/writable-stream-telemetry.test.ts index 81a939df6f..e3b6e52049 100644 --- a/packages/core/src/writable-stream-telemetry.test.ts +++ b/packages/core/src/writable-stream-telemetry.test.ts @@ -147,34 +147,6 @@ describe('WorkflowServerWritableStream write-flush telemetry', () => { } }); - it('counts the turbo run-ready barrier wait as buffer dwell', async () => { - let releaseBarrier!: () => void; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = resolve; - }); - - const stream = new WorkflowServerWritableStream( - 'run-123', - 'test-stream', - runReadyBarrier - ); - const writer = stream.getWriter(); - - await writer.write(new Uint8Array([1, 2, 3])); - // Hold the first dispatch on the barrier long enough to dominate the - // dwell (write() itself acks on buffer entry and does not wait). - await new Promise((r) => setTimeout(r, 50)); - releaseBarrier(); - await writer.close(); - - const [span] = await waitForSpans('workflow.stream.flush', 1); - expect(span).toBeDefined(); - const dwell = span.attributes[ - 'workflow.stream.flush.buffer_dwell_ms' - ] as number; - expect(dwell).toBeGreaterThanOrEqual(40); - }); - it('emits a workflow.stream.close span with the close RPC duration', async () => { const stream = new WorkflowServerWritableStream('run-123', 'test-stream'); const writer = stream.getWriter(); diff --git a/packages/core/src/writable-stream.test.ts b/packages/core/src/writable-stream.test.ts index 8dcbe253bd..233442e162 100644 --- a/packages/core/src/writable-stream.test.ts +++ b/packages/core/src/writable-stream.test.ts @@ -2,10 +2,7 @@ import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createFlushableState, flushablePipe } from './flushable-stream.js'; import { setWorld } from './runtime/world.js'; -import { - dehydrateStepReturnValue, - WorkflowServerWritableStream, -} from './serialization.js'; +import { WorkflowServerWritableStream } from './serialization.js'; /** * Poll until the expectation passes — replaces fixed sleeps for @@ -664,174 +661,6 @@ describe('WorkflowServerWritableStream', () => { }); }); - describe('runReadyBarrier (turbo optimistic start)', () => { - it('holds the first server write until the run-ready barrier resolves', async () => { - const order: string[] = []; - mockStreams.write.mockImplementation(async () => { - order.push('write'); - }); - - let releaseBarrier!: () => void; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = () => { - order.push('barrier'); - resolve(); - }; - }); - - const stream = new WorkflowServerWritableStream( - 'run-123', - 'test-stream', - runReadyBarrier - ); - const writer = stream.getWriter(); - await writer.write(new Uint8Array([1, 2, 3])); - - // The body wrote a chunk before run_started is durable: the commit - // window may fire, but the server write must not happen until the - // run exists. - await new Promise((r) => setTimeout(r, 30)); - expect(mockStreams.write).not.toHaveBeenCalled(); - - releaseBarrier(); - // close() drains: it completes only after the gated dispatch lands. - await writer.close(); - - // The chunk reaches the server strictly after the barrier resolves. - expect(order).toEqual(['barrier', 'write']); - }); - - it('only awaits the barrier once — later flushes are not gated', async () => { - const runReadyBarrier = Promise.resolve(); - const stream = new WorkflowServerWritableStream( - 'run-123', - 'test-stream', - runReadyBarrier - ); - const writer = stream.getWriter(); - - await writer.write(new Uint8Array([1])); - await writer.write(new Uint8Array([2])); - await writer.write(new Uint8Array([3])); - await writer.close(); - - // All three chunks are delivered (grouping may vary); the resolved - // barrier gated nothing after the first dispatch. - const delivered = [ - ...mockStreams.write.mock.calls.map( - (call: unknown[]) => (call[2] as Uint8Array)[0] - ), - ...mockStreams.writeMulti.mock.calls.flatMap((call: unknown[]) => - (call[2] as Uint8Array[]).map((c) => c[0]) - ), - ]; - expect(delivered.sort()).toEqual([1, 2, 3]); - }); - - it('still writes when the barrier rejects (write surfaces the real error)', async () => { - const runReadyBarrier = Promise.reject(new Error('run_started failed')); - runReadyBarrier.catch(() => {}); - - const stream = new WorkflowServerWritableStream( - 'run-123', - 'test-stream', - runReadyBarrier - ); - const writer = stream.getWriter(); - - await writer.write(new Uint8Array([1, 2, 3])); - await writer.close(); - - // Barrier rejection is swallowed for ordering only — the write still - // fires and would surface a genuine run-not-found error from the World. - expect(mockStreams.write).toHaveBeenCalledTimes(1); - }); - - it('gates the first write of a stream RETURNED from a turbo first step', async () => { - // Regression: getWritable()/setAttributes are gated while the step - // context is active, but a step that *returns* a fresh ReadableStream - // is serialized after the body via dehydrateStepReturnValue(), whose - // sink must also wait for run_started before the first chunk lands. - const order: string[] = []; - mockStreams.write.mockImplementation(async () => { - order.push('write'); - }); - - let releaseBarrier!: () => void; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = () => { - order.push('barrier'); - resolve(); - }; - }); - - const returned = new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array([1, 2, 3])); - controller.close(); - }, - }); - - const ops: Promise[] = []; - await dehydrateStepReturnValue( - returned, - 'run-123', - undefined, // encryption key - ops, - globalThis, - false, // v1Compat - false, // framedByteStreams - false, // compression - runReadyBarrier - ); - - // The pipe is queued but must not have written before the run exists. - await new Promise((r) => setTimeout(r, 30)); - expect(mockStreams.write).not.toHaveBeenCalled(); - - releaseBarrier(); - await Promise.all(ops); - - // The returned stream's first chunk reaches the server only after the - // run-ready barrier resolves. - expect(order[0]).toBe('barrier'); - expect(mockStreams.write).toHaveBeenCalled(); - }); - - it('gates a close that is itself the first write to a new stream', async () => { - const order: string[] = []; - mockStreams.close.mockImplementation(async () => { - order.push('close'); - }); - - let releaseBarrier!: () => void; - const runReadyBarrier = new Promise((resolve) => { - releaseBarrier = () => { - order.push('barrier'); - resolve(); - }; - }); - - const stream = new WorkflowServerWritableStream( - 'run-123', - 'test-stream', - runReadyBarrier - ); - const writer = stream.getWriter(); - - // Close with no chunks written: flush() short-circuits on the empty - // buffer, so close() must apply the barrier itself. - const closePromise = writer.close(); - await new Promise((r) => setTimeout(r, 30)); - expect(mockStreams.close).not.toHaveBeenCalled(); - - releaseBarrier(); - await closePromise; - - expect(order).toEqual(['barrier', 'close']); - }); - }); - describe('writeMulti batching via flushablePipe (equivalent to the native path)', () => { it('coalesces chunks that arrive during an in-flight write into one writeMulti', async () => { // Hold the first server write in flight so the chunks produced while it diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 7b21ff8d83..cfed551bab 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -17,6 +17,18 @@ import { import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; +function joinFrames(...frames: Uint8Array[]): Uint8Array { + const joined = new Uint8Array( + frames.reduce((length, frame) => length + frame.byteLength, 0) + ); + let offset = 0; + for (const frame of frames) { + joined.set(frame, offset); + offset += frame.byteLength; + } + return joined; +} + /** * The v4 client must preserve the typed-error contract of the v3 * `makeRequest` path — the workflow runtime branches on these types @@ -398,111 +410,82 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('forwards skipPreload in the run_started frame meta (turbo preload opt-out)', async () => { + it('requests and decodes the first event page for run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); - - // Decode the posted frame's CBOR meta block: - // u32_be(meta_len) || cbor_meta || u32_be(body_len) || body - let capturedMeta: Record | undefined; - const captureMeta = (rawBody: unknown) => { - const bytes = - typeof rawBody === 'string' - ? new TextEncoder().encode(rawBody) - : new Uint8Array(rawBody as ArrayBufferLike); - const metaLen = new DataView( - bytes.buffer, - bytes.byteOffset, - bytes.byteLength - ).getUint32(0, false); - capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< - string, - unknown - >; - }; + const input = new TextEncoder().encode('"workflow input"'); agent .get(origin) .intercept({ path: '/api/v4/runs/wrun_1/events/run_started', method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, }) .reply( 200, - (opts: { body?: unknown }) => { - captureMeta(opts.body); - return encode({ run: { runId: 'wrun_1', status: 'running' } }); - }, + joinFrames( + encodeFrame( + { + _result: 1, + event: { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + }, + run: { runId: 'wrun_1', status: 'running' }, + }, + new Uint8Array() + ), + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + }, + input + ), + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ) + ), { headers: { - 'x-wf-event-id': 'evnt_1', + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_2', 'x-wf-run-id': 'wrun_1', 'x-wf-created-at': '2026-06-10T00:00:00.000Z', }, } ); - await createWorkflowRunEventV4( + const result = await createWorkflowRunEventV4( { runId: 'wrun_1', eventType: 'run_started', specVersion: 5, - skipPreload: true, }, { token: 'test-token', dispatcher: agent } ); - expect(capturedMeta?.eventType).toBe('run_started'); - expect(capturedMeta?.skipPreload).toBe(true); - agent.assertNoPendingInterceptors(); - }); - - it('omits skipPreload from the frame meta when not set (default / old SDK parity)', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); - - let capturedMeta: Record | undefined; - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events/run_started', - method: 'POST', - }) - .reply( - 200, - (opts: { body?: unknown }) => { - const bytes = new Uint8Array(opts.body as ArrayBufferLike); - const metaLen = new DataView( - bytes.buffer, - bytes.byteOffset, - bytes.byteLength - ).getUint32(0, false); - capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< - string, - unknown - >; - return encode({ run: { runId: 'wrun_1', status: 'running' } }); - }, - { - headers: { - 'x-wf-event-id': 'evnt_1', - 'x-wf-run-id': 'wrun_1', - 'x-wf-created-at': '2026-06-10T00:00:00.000Z', - }, - } - ); - - await createWorkflowRunEventV4( - { runId: 'wrun_1', eventType: 'run_started', specVersion: 5 }, - { token: 'test-token', dispatcher: agent } - ); - - expect(capturedMeta?.eventType).toBe('run_started'); - expect('skipPreload' in (capturedMeta ?? {})).toBe(false); + expect(result.type).toBe('event-page'); + if (result.type !== 'event-page') throw new Error('expected event page'); + expect(result.body.run).toMatchObject({ status: 'running' }); + expect(result.page.events).toHaveLength(2); + expect(result.page.events[0].body).toEqual(input); + expect(result.page.next).toBe('eid:evnt_2'); + expect(result.page.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index f5422c81d9..0ebb9e11b5 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -189,11 +189,6 @@ export interface CreateEventV4Input { * other event types; older servers ignore it entirely (the runtime then * falls back to events.list). */ sinceCursor?: string; - /** Run-started preload opt-out. Turbo backgrounds run_started as a write - * barrier only and never reads the preloaded log, so it asks the server to - * skip the list+resolve. Acted on by the server only for run_started; - * older servers ignore it and preload as before. */ - skipPreload?: boolean; /** * Epoch ms (the ULID time of the latest event the runtime has loaded * during replay). Sent by replay-context creates so the backend can @@ -204,16 +199,13 @@ export interface CreateEventV4Input { stateUpdatedAt?: number; } -export interface CreateEventV4Result { +interface CreateEventV4ResultBase { eventId: string; runId: string; createdAt: string; /** - * Materialized-entity bag — CBOR-decoded from the response body. The - * server hands back the same shape v2/v3 use for EventResult so the - * adapter layer can drop these fields into its return value unchanged. - * Keys are unset when the event type doesn't materialize that entity - * kind. + * Materialized-entity bag decoded from CBOR or the leading result frame. + * Keys are unset when the event type doesn't materialize that entity kind. */ body: { event?: unknown; @@ -238,6 +230,13 @@ export interface CreateEventV4Result { }; } +export type CreateEventV4Result = + | (CreateEventV4ResultBase & { type: 'event' }) + | (CreateEventV4ResultBase & { + type: 'event-page'; + page: ListEventsV4Result; + }); + /** Build the CBOR meta map for a v4 POST frame. Drops undefined entries * so the wire shape matches what the server expects to see. */ function buildPostFrameMeta( @@ -293,7 +292,6 @@ function buildPostFrameMeta( meta.optimizations = input.optimizations; } if (input.sinceCursor !== undefined) meta.sinceCursor = input.sinceCursor; - if (input.skipPreload !== undefined) meta.skipPreload = input.skipPreload; if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } @@ -363,9 +361,8 @@ export function throwForErrorResponse( /** * POST /api/v4/runs/:runId/events/:eventType * - * Sends the full request as a single v4 frame and returns the event ids - * + materialized-entity bag from the CBOR response body. Throws on - * non-2xx. + * Sends the full request as a single v4 frame. A run_started response may + * include the first event page as frames; other responses use CBOR. * * The trailing `:eventType` path segment is an alias of the canonical * `/events` route: it exists purely so the event type is visible in @@ -382,6 +379,9 @@ export async function createWorkflowRunEventV4( const { baseUrl, headers: baseHeaders } = await getHttpConfig(config); const headers = new Headers(baseHeaders); headers.set('Content-Type', 'application/octet-stream'); + if (input.eventType === 'run_started') { + headers.set('Accept', V4_FRAME_CONTENT_TYPE); + } const frame = encodeFrame( buildPostFrameMeta(input), @@ -407,6 +407,32 @@ export async function createWorkflowRunEventV4( throw new Error('v4 createEvent: response missing required x-wf-* headers'); } + const contentType = response.headers.get('content-type'); + if (contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { + if (input.eventType !== 'run_started') { + throw new Error( + 'v4 createEvent: received an event page for a non-run_started event' + ); + } + const { result, ...page } = await consumeEventFrameStream( + response, + 'createEvent' + ); + if (!result) { + throw new Error( + 'v4 createEvent: frame stream ended without the result frame' + ); + } + return { + type: 'event-page', + eventId, + runId, + createdAt, + body: result, + page, + }; + } + // Decode the materialized-entity bag from the CBOR response body. const bodyBytes = new Uint8Array(await response.arrayBuffer()); const body = @@ -414,7 +440,7 @@ export async function createWorkflowRunEventV4( ? (decode(bodyBytes) as CreateEventV4Result['body']) : {}; - return { eventId, runId, createdAt, body }; + return { type: 'event', eventId, runId, createdAt, body }; } /** @@ -537,27 +563,14 @@ export interface ListEventsV4Result { hasMore?: boolean; } -/** - * Drive a v4 frame-stream list response into an in-memory page. Used by - * both the by-runId and by-correlationId list endpoints — the wire - * shape is identical, only the URL differs. - * - * `headers` come from the caller's single getHttpConfig resolution (the - * same call that produced the baseUrl in `url`) so each LIST resolves - * auth exactly once. - */ -async function consumeListFrameStream( - url: string, - headers: Headers, - config: APIConfig | undefined, +interface EventFrameStreamResult extends ListEventsV4Result { + result?: CreateEventV4ResultBase['body']; +} + +async function consumeEventFrameStream( + response: Response, opName: string -): Promise { - const response = await fetchV4( - url, - { method: 'GET', headers }, - config, - opName - ); +): Promise { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { throw new Error( @@ -565,15 +578,22 @@ async function consumeListFrameStream( ); } - // See getEventV4: fetch's web ReadableStream is async-iterable on Node; the - // cast only works around TS's lib type omitting the async iterator. const chunks = response.body as unknown as AsyncIterable; - const events: ListedEventV4[] = []; + let result: CreateEventV4ResultBase['body'] | undefined; let next: string | undefined; let hasMore: boolean | undefined; let sawEndSentinel = false; + for await (const frame of decodeFrames(chunks)) { + if (frame.meta._result === 1) { + if (result || events.length > 0 || frame.body.byteLength > 0) { + throw new Error(`v4 ${opName}: invalid result frame`); + } + const { _result, ...body } = frame.meta; + result = body; + continue; + } if (frame.meta._end === 1) { if (typeof frame.meta.next === 'string') next = frame.meta.next; if (typeof frame.meta.hasMore === 'boolean') hasMore = frame.meta.hasMore; @@ -586,12 +606,6 @@ async function consumeListFrameStream( }); } - // A LIST response always ends with the `{_end: 1}` sentinel frame. EOF - // without it means the response was truncated — and if the cut landed - // between two complete frames, decodeFrames alone can't tell. Returning - // the partial page here would surface as `hasMore: false` and silently - // drop events (replay correctness!), so fail loudly instead; the read - // is idempotent and safe for the caller to retry. if (!sawEndSentinel) { throw new Error( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + @@ -601,11 +615,40 @@ async function consumeListFrameStream( return { events, + ...(result ? { result } : {}), ...(next ? { next } : {}), ...(hasMore !== undefined ? { hasMore } : {}), }; } +/** + * Drive a v4 frame-stream list response into an in-memory page. Used by + * both the by-runId and by-correlationId list endpoints — the wire + * shape is identical, only the URL differs. + * + * `headers` come from the caller's single getHttpConfig resolution (the + * same call that produced the baseUrl in `url`) so each LIST resolves + * auth exactly once. + */ +async function consumeListFrameStream( + url: string, + headers: Headers, + config: APIConfig | undefined, + opName: string +): Promise { + const response = await fetchV4( + url, + { method: 'GET', headers }, + config, + opName + ); + const { result, ...page } = await consumeEventFrameStream(response, opName); + if (result) { + throw new Error(`v4 ${opName}: unexpected result frame`); + } + return page; +} + /** * Append the shared list params (pagination + ref behavior) to `sp`. * Shared by the runId and correlationId list query builders so both send diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index f78845fdaf..87118ed84c 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1,3 +1,4 @@ +import { gzipSync } from 'node:zlib'; import type { AnyEventRequest } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; @@ -33,6 +34,18 @@ function decodePostedMeta(rawBody: unknown): Record { return decode(bytes.subarray(4, 4 + metaLen)) as Record; } +function joinFrames(...frames: Uint8Array[]): Uint8Array { + const joined = new Uint8Array( + frames.reduce((length, frame) => length + frame.byteLength, 0) + ); + let offset = 0; + for (const frame of frames) { + joined.set(frame, offset); + offset += frame.byteLength; + } + return joined; +} + /** * Legacy (spec-version-1) runs predate event sourcing: the runtime still * posts hook_received (resumeHook) and wait_completed (wakeUpRun) for them @@ -629,6 +642,96 @@ describe('createWorkflowRunEvent response coercion', () => { agent.assertNoPendingInterceptors(); }); + it('hydrates a streamed run_started page in either structural event order', async () => { + const agent = mockAgent(); + const serializedInput = new TextEncoder().encode('"workflow input"'); + const compressedInput = gzipSync(serializedInput); + const input = new Uint8Array(4 + compressedInput.byteLength); + input.set(new TextEncoder().encode('gzip')); + input.set(compressedInput, 4); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + joinFrames( + encodeFrame( + { + _result: 1, + run: { + runId: 'wrun_1', + status: 'running', + startedAt: new Date('2026-06-10T00:00:01.000Z'), + }, + event: { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: new Date('2026-06-10T00:00:00.000Z'), + }, + }, + new Uint8Array() + ), + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: new Date('2026-06-10T00:00:00.000Z'), + specVersion: 5, + eventData: {}, + }, + new Uint8Array() + ), + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: new Date('2026-06-10T00:00:01.000Z'), + specVersion: 5, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'wf', + input: { _type: 'RemoteRef', value: 'dbrf:unused' }, + }, + }, + input + ), + encodeFrame({ _end: 1, next: 'eid:evnt_2' }, new Uint8Array()) + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const result = await createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 5 } as AnyEventRequest, + undefined, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events?.map((event) => event.eventType)).toEqual([ + 'run_started', + 'run_created', + ]); + expect(result.events?.[1].eventData?.input).toEqual(input); + expect(result.run?.input).toEqual(input); + expect(result.cursor).toBe('eid:evnt_2'); + expect(result.hasMore).toBe(true); + agent.assertNoPendingInterceptors(); + }); + it('threads the wait entity through to the EventResult', async () => { const agent = mockAgent(); agent diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index a2d223a882..868ee54c42 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -16,9 +16,8 @@ * * - POST request body is one v4 frame (meta + payload). The response * surfaces eventId/runId/createdAt as `x-wf-*` headers and carries - * the materialized EventResult (event/run/step/hook/wait/events/ - * cursor/hasMore) as a CBOR body — `remoteRefBehavior` in the frame - * meta still controls server-side ref resolution. + * a materialized EventResult. `run_started` negotiates a frame stream + * containing that result and the first event page; other events use CBOR. * - GET single event returns one v4 frame: the event entity in the * frame meta, the user payload bytes in the frame body. * - LIST events returns a stream of v4 frames terminated by a sentinel @@ -476,30 +475,44 @@ function decodeLegacyStructuredError(payload: Uint8Array): unknown { * * Both GET single-event and LIST use the same frame format: meta is the * full event entity with the payload field as a RefDescriptor, body is - * the resolved payload bytes (possibly empty). This helper splices the - * body bytes into `eventData[fieldName]`, normalizing any zstd wrapper - * back to the raw devalue-with-format-prefix Uint8Array the runtime's - * hydrate helpers (hydrateStepIO, hydrateRunError, …) consume. Stable-line - * structured errors are the exception: the backend stored those as CBOR, - * so they are decoded after checking that the payload is not a current - * format-prefixed serialized value. + * the resolved payload bytes (possibly empty). Read mode normalizes + * compression wrappers for display. Replay mode preserves the serialized + * bytes so the runtime's hydrate helpers own decompression and telemetry. + * Stable-line structured errors are decoded after checking that the payload + * is not a current format-prefixed serialized value. */ function buildEventFromV4( decoded: DecodedV4Event, payloadBody: Uint8Array, - resolveData: 'none' | 'all' + mode: 'none' | 'all' | 'replay' ): Event { const eventData = (decoded.eventData ?? {}) as Record; + let normalizePayload: boolean; + switch (mode) { + case 'none': + case 'all': + normalizePayload = true; + break; + case 'replay': + normalizePayload = false; + break; + default: { + const exhaustive: never = mode; + throw new Error(`Unsupported v4 event mode: ${exhaustive}`); + } + } if (payloadBody.byteLength > 0) { const payloadField = getEventDataPayloadField(decoded.eventType); - const normalizedPayload = normalizeSerializedData(payloadBody); - if (payloadField && normalizedPayload instanceof Uint8Array) { + const payload = normalizePayload + ? normalizeSerializedData(payloadBody) + : payloadBody; + if (payloadField && payload instanceof Uint8Array) { eventData[payloadField] = legacyStructuredErrorEventTypes.has( decoded.eventType ) - ? decodeLegacyStructuredError(normalizedPayload) - : normalizedPayload; + ? decodeLegacyStructuredError(payload) + : payload; } } @@ -526,11 +539,13 @@ function buildEventFromV4( : {}), }; - const event = coerceNormalizedEvent(raw); + const event = normalizePayload + ? coerceNormalizedEvent(raw) + : coerceEventDates(raw); // For resolveData='none', strip eventData entirely. Reuse the world- // side helper so behavior stays in sync with other backends. - return resolveData === 'none' ? stripEventDataRefs(event, 'none') : event; + return mode === 'none' ? stripEventDataRefs(event, 'none') : event; } // ============================================================================= @@ -706,11 +721,6 @@ async function createWorkflowRunEventInner( // step_completed/step_failed; older servers ignore it and the runtime // falls back to events.list. ...(params?.sinceCursor ? { sinceCursor: params.sinceCursor } : {}), - // Run-started preload opt-out: turbo backgrounds run_started as a write - // barrier only and never reads the preloaded log, so tell the server to - // skip the list+resolve. The server only acts on it for run_started; - // older servers ignore it and simply preload as before. - ...(params?.skipPreload ? { skipPreload: true } : {}), remoteRefBehavior, payload, ...meta, @@ -738,6 +748,38 @@ async function createWorkflowRunEventInner( // matching the v3 path's stripEventAndLegacyRefs behavior. const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; const body = result.body; + const streamedEvents = + result.type === 'event-page' + ? result.page.events.map(({ event, body }) => + buildEventFromV4(event, body, 'replay') + ) + : undefined; + const events = streamedEvents + ? streamedEvents.map((event) => stripEventDataRefs(event, resolveData)) + : body.events + ? (body.events as Record[]).map(coerceEventDates) + : undefined; + const cursor = result.type === 'event-page' ? result.page.next : body.cursor; + const hasMore = + result.type === 'event-page' + ? (result.page.hasMore ?? Boolean(result.page.next)) + : body.hasMore; + let run = body.run + ? deserializeError(body.run as Record) + : undefined; + if (run && streamedEvents) { + const runCreated = streamedEvents.find( + (event) => event.eventType === 'run_created' + ); + if (!runCreated) { + throw new Error('v4 createEvent: run_started page missing run_created'); + } + const input = (runCreated.eventData as Record).input; + const { inputRef: _inputRef, ...runData } = run as WorkflowRun & { + inputRef?: unknown; + }; + run = { ...runData, input } as WorkflowRun; + } return { event: body.event ? stripEventDataRefs( @@ -745,19 +787,15 @@ async function createWorkflowRunEventInner( resolveData ) : undefined, - run: body.run - ? deserializeError(body.run as Record) - : undefined, + run, step: body.step ? deserializeStep(body.step as Parameters[0]) : undefined, hook: body.hook as EventResult['hook'], wait: body.wait as EventResult['wait'], - events: body.events - ? (body.events as Record[]).map(coerceEventDates) - : undefined, - cursor: body.cursor ?? undefined, - hasMore: body.hasMore, + events, + cursor, + hasMore, // Lazy step start: thread the server's "I created the step on this call" // signal through so the owned-inline runtime path can gate body execution // on it. Absent from older servers → undefined → safe default. diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 6745875804..148ab095ca 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -742,23 +742,6 @@ export interface CreateEventParams { * delta is computed atomically against the same log the fetch would read. */ sinceCursor?: string; - /** - * Run-started preload opt-out (advisory). On a `run_started` write a World - * MAY preload the run's event log onto the {@link EventResult} - * (`events`/`cursor`/`hasMore`) so the runtime can skip its initial - * `events.list`. The turbo first invocation backgrounds `run_started` - * purely as a write barrier and never reads that preload, so it sets this - * to tell the World to skip the wasted list+resolve — trimming the - * `run_started` round-trip that the chained first `step_started` waits on. - * A World that ignores it (or doesn't preload) remains fully correct: the - * runtime falls back to `events.list` whenever it actually needs the log. - * Only honored for `run_started`; ignored for other event types. - * - * Named to match the World boundary, the wire frame meta, and the backend - * option end-to-end (cf. {@link sinceCursor}) so the single name greps - * across the SDK and the backend. - */ - skipPreload?: boolean; } /** From 471cd0fa44f939f4af7123163fbfdb598e682b8f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:44:49 -0700 Subject: [PATCH 2/3] fix: preserve turbo startup while streaming replay --- .changeset/stream-run-started-page.md | 4 +- packages/core/src/runtime-trace-mode.test.ts | 3 + packages/core/src/runtime.test.ts | 79 +++-- packages/core/src/runtime.ts | 310 +++++++++++++----- packages/core/src/runtime/constants.ts | 6 +- packages/core/src/runtime/step-executor.ts | 89 +++-- packages/core/src/runtime/step-latency.ts | 15 +- .../core/src/runtime/suspension-handler.ts | 34 ++ packages/core/src/serialization.ts | 97 +++++- packages/core/src/set-attributes.test.ts | 54 +++ packages/core/src/set-attributes.ts | 14 + packages/core/src/step/context-storage.ts | 10 + packages/core/src/step/writable-stream.ts | 22 +- .../src/telemetry/semantic-conventions.ts | 9 +- packages/core/src/workflow.test.ts | 144 +++++++- packages/core/src/workflow.ts | 21 +- .../src/writable-stream-telemetry.test.ts | 28 ++ packages/core/src/writable-stream.test.ts | 173 +++++++++- packages/world-vercel/src/events-v4.test.ts | 54 +++ packages/world-vercel/src/events-v4.ts | 11 +- packages/world-vercel/src/events.ts | 5 + packages/world/src/events.ts | 17 + 22 files changed, 1042 insertions(+), 157 deletions(-) diff --git a/.changeset/stream-run-started-page.md b/.changeset/stream-run-started-page.md index 270e239ebf..e14e9fde97 100644 --- a/.changeset/stream-run-started-page.md +++ b/.changeset/stream-run-started-page.md @@ -1,7 +1,5 @@ --- -'@workflow/core': patch -'@workflow/world': patch '@workflow/world-vercel': patch --- -Load the first replay page from the `run_started` response and await startup before Turbo replay. +Load the first replay page from non-Turbo `run_started` responses while preserving Turbo's preload opt-out. diff --git a/packages/core/src/runtime-trace-mode.test.ts b/packages/core/src/runtime-trace-mode.test.ts index 66d106271b..fbf5928b61 100644 --- a/packages/core/src/runtime-trace-mode.test.ts +++ b/packages/core/src/runtime-trace-mode.test.ts @@ -309,6 +309,9 @@ describe('workflowEntrypoint trace modes', () => { (e) => e.name === 'workflow.run_started.create.start' ); expect(runStartedCreateEvent).toBeDefined(); + expect( + runStartedCreateEvent?.attributes['workflow.run_started.skip_preload'] + ).toBe(false); // Queue-delivered invocation spans use the CONSUMER kind, matching // queue-delivered step.execute spans. diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index af1307bd82..aafec00075 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -1485,15 +1485,14 @@ describe('workflowEntrypoint turbo mode', () => { /** * Drives the handler with a first-invocation message (runInput present) at the * given delivery `attempt`. `runStartedGate`, when provided, holds the - * `run_started` create until released. `stepStartedGate` can separately hold - * the optimistic step claim. + * `run_started` create until released — its resolution pushes + * 'run_started_resolved' so tests can assert the body ran before or after it. */ async function driveTurbo(opts: { runId: string; attempt: number; source: string; runStartedGate?: Promise; - stepStartedGate?: Promise; }) { const { runId, attempt, source } = opts; const order = turboOrder; @@ -1529,8 +1528,6 @@ describe('workflowEntrypoint turbo mode', () => { } if (data.eventType === 'step_started') { order.push('step_started_called'); - if (opts.stepStartedGate) await opts.stepStartedGate; - order.push('step_started_resolved'); const d = data.eventData as { stepName?: string; input?: unknown }; if (d?.input !== undefined) { rec({ @@ -1600,41 +1597,40 @@ describe('workflowEntrypoint turbo mode', () => { return { handlerPromise, order, eventsCreate }; } - it('awaits run_started before forcing optimistic start on the first delivery', async () => { - let releaseRunStarted!: () => void; - const runStartedGate = new Promise((resolve) => { - releaseRunStarted = resolve; - }); - let releaseStepStarted!: () => void; - const stepStartedGate = new Promise((resolve) => { - releaseStepStarted = resolve; + it('backgrounds run_started and forces optimistic start on the first delivery', async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; }); const { handlerPromise, order, eventsCreate } = await driveTurbo({ runId: 'wrun_turbo_first', attempt: 1, source: oneStepWorkflow, - runStartedGate, - stepStartedGate, + runStartedGate: gate, }); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(order).not.toContain('body'); - expect(order).not.toContain('run_started_resolved'); - expect(order).not.toContain('step_started_called'); - - releaseRunStarted(); + // The body runs while run_started is still in flight — proving run_started + // was backgrounded AND optimistic start was forced (the env flag is off). + // The full VM replay leading up to the body can exceed vi.waitFor's default + // 1s timeout on slow CI runners (notably Windows), so widen it. await vi.waitFor(() => expect(order).toContain('body'), { timeout: 15_000, }); - expect(order.indexOf('run_started_resolved')).toBeLessThan( - order.indexOf('step_started_called') - ); - expect(order).not.toContain('step_started_resolved'); + expect(order).not.toContain('run_started_resolved'); + // The lazy step_started is chained on the run-ready barrier, so it is not + // even issued until run_started lands. + expect(order).not.toContain('step_started_called'); - releaseStepStarted(); + release(); const res = await handlerPromise; expect(res.status).toBe(204); + // After release: step_started fires, ordered strictly after run_started. + expect(order).toContain('step_started_called'); + expect(order.indexOf('run_started_resolved')).toBeLessThan( + order.indexOf('step_started_called') + ); + // run_started was created exactly once (idempotent first write). const runStartedCreates = eventsCreate.mock.calls.filter( (c) => (c[1] as any).eventType === 'run_started' ); @@ -1671,6 +1667,37 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('asks the World to skip the run_started preload only under turbo', async () => { + // The backgrounded run_started is used purely as a write barrier and its + // preloaded events are never read (preloadedEvents is forced to []), so + // turbo passes skipPreload to drop the wasted server-side + // list+resolve that the chained first step_started waits behind. + const turbo = await driveTurbo({ + runId: 'wrun_turbo_skip_preload', + attempt: 1, + source: oneStepWorkflow, + }); + expect((await turbo.handlerPromise).status).toBe(204); + const turboRunStarted = turbo.eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + expect((turboRunStarted?.[2] as any)?.skipPreload).toBe(true); + + // A redelivery (attempt > 1) is not turbo: it awaits run_started and + // consumes the preload to skip its initial events.list, so it must NOT ask + // the server to skip it. + const redeliver = await driveTurbo({ + runId: 'wrun_turbo_skip_preload_redeliver', + attempt: 2, + source: oneStepWorkflow, + }); + expect((await redeliver.handlerPromise).status).toBe(204); + const redeliverRunStarted = redeliver.eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 9be82ae629..82e0a9fc1a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -541,7 +541,7 @@ export function workflowEntrypoint( // or the run_started setup below. let workflowRun: WorkflowRun | undefined; // Server-supplied per-run event ceiling from the run_started - // response. Undefined means an older World supplied no limit. + // response. Undefined ⇒ no enforcement (older servers, turbo). let maxEventsLimit: number | undefined; let workflowStartedAt = -1; let preloadedEventLog: MutableEventLog | undefined; @@ -557,6 +557,10 @@ export function workflowEntrypoint( // Epoch ms the `run_started` response was received/parsed // by the SDK — anchors RSFS (run_started → first step's // start POST). Set once, in the run_started setup below. + // Under turbo, run_started is backgrounded rather than + // awaited, so this is stamped at the point the run is + // synthesized locally instead of the real response — see + // StepLatencyTracking.rsfsAnchorMs. let runStartedReceivedAtMs: number | undefined; // Wall-clock ms spent committing hook_created events before // the first step ran, accumulated across suspension passes @@ -571,7 +575,9 @@ export function workflowEntrypoint( let preStepBlockingBeforeAttrMs: number | undefined; // Turbo mode fast-paths the very first delivery of the very - // first invocation by forcing optimistic inline start (no + // first invocation, where it is provably safe to: background + // `run_started`, skip the initial event-log load (nothing has + // been written yet), and force optimistic inline start (no // concurrent peer handler exists to race the create-claim). // `runInput` is only present on the start()-enqueued message, // and `attempt === 1` (1-based) means this is the first @@ -602,6 +608,30 @@ export function workflowEntrypoint( !replayDivergence; span?.setAttributes(Attribute.WorkflowTurbo(turbo)); + // Turbo mode only: resolves once the backgrounded + // `run_started` has landed (or rejects if it failed). Threaded + // into handleSuspension and executeStep so no step/hook/wait + // write races ahead of the run's creation. Undefined outside + // turbo, where `run_started` is awaited up front. + let runReadyBarrier: Promise | undefined; + + // Order a terminal run write (run_completed / run_failed) after + // the backgrounded run_started in turbo mode — a no-step + // workflow can otherwise reach run_completed before the run + // exists. Best-effort: a barrier rejection is swallowed for + // ordering only; if run_started truly failed the terminal write + // surfaces the real error (run not found / gone) and the message + // redelivers. No-op outside turbo. + const awaitRunReady = async (): Promise => { + if (runReadyBarrier) { + try { + await runReadyBarrier; + } catch { + // intentional: ordering barrier only — see above. + } + } + }; + // Re-invoke the orchestrator. Outside turbo this returns // `{ timeoutSeconds }`, which makes the queue reschedule the // CURRENT delivery's message. In turbo that is a trap: the @@ -948,84 +978,194 @@ export function workflowEntrypoint( } : {}), }; - let startedRun: StartedWorkflowRun; - try { - span?.addEvent('workflow.run_started.create.start'); - const result = await world.events.create( + const recordRunStartedCreateStart = ( + skipPreload: boolean + ) => { + span?.addEvent('workflow.run_started.create.start', { + 'workflow.run_started.skip_preload': skipPreload, + }); + }; + + if (turbo && runInput) { + // Turbo: background `run_started` and synthesize the run + // entity locally so replay can begin without waiting for + // the round-trip. Safe here because this is the first + // delivery of the first invocation — start() created the + // run moments ago and no events have been written yet. The + // barrier is consumed by every downstream write (suspension + // handler, optimistic step_started, terminal run writes) so + // nothing is written before the run exists. + recordRunStartedCreateStart(true); + const startedPromise = world.events.create( runId, runStartedEvent, - { requestId } + // We background this purely as a write barrier and + // never read its preloaded events, so skip the + // run_started event-log preload. That trims the + // run_started request the chained first step_started + // waits on — shortening time-to-second-step — and the + // wasted list+resolve it would otherwise compute. + { requestId, skipPreload: true } ); - startedRun = result.run; - workflowRun = startedRun; - maxEventsLimit = clampMaxEvents(result.maxEvents); - runStartedReceivedAtMs = Date.now(); - - if (result.events?.length) { - let events = result.events; - let cursor = result.cursor ?? null; - if (result.hasMore) { - const loaded = await loadWorkflowRunEvents( - runId, - cursor ?? undefined - ); - if (cursor) { - const preloadedIds = new Set( - events.map((event) => event.eventId) - ); - events = events.concat( - loaded.events.filter( - (event) => !preloadedIds.has(event.eventId) - ) + runReadyBarrier = startedPromise; + // Turbo backgrounds run_started, so the non-turbo assignment + // below never runs — thread the per-run event ceiling off the + // backgrounded response here instead. The guard re-checks + // maxEventsLimit every loop iteration, so a value that lands + // shortly after start still enforces well before a runaway + // log approaches the ceiling. + startedPromise.then( + (r) => { + const limit = clampMaxEvents(r?.maxEvents); + if (limit !== undefined) maxEventsLimit = limit; + }, + () => {} + ); + // Attach a no-op rejection handler so an early failure + // never surfaces as an unhandledRejection before a consumer + // (await/then) is attached; consumers still observe it. + startedPromise.catch(() => {}); + // Skip the initial events.list: nothing has been written to + // the log yet on a first delivery (run_started is still in + // flight). An empty preload routes iteration 1 through + // the no-load preloaded branch; iteration 2 then takes the + // existing post-preloaded full reload to pick up a cursor + // without a spurious "cursor missing" warning. + preloadedEventLog = { events: [], cursor: null }; + const now = new Date(); + workflowRun = { + runId, + status: 'running', + deploymentId: runInput.deploymentId, + workflowName: runInput.workflowName, + specVersion: runInput.specVersion, + executionContext: runInput.executionContext, + input: runInput.input, + // Seed attributes from start() ride along in `runInput` + // (they live in `run_created`'s eventData, not separate + // `attr_set` events), so the synthesized snapshot carries + // them even though we skip the initial events.list. This + // is correct ONLY while attributes are write-only: + // there is no in-workflow read API today (see workflow.ts + // "structural until a read API is introduced"), so the + // empty preloaded log can't diverge on a read. If a read + // API is ever added it MUST read from this snapshot, not + // by replaying run_created/attr_set events — otherwise + // turbo's empty initial log would surface seed attributes + // as `{}` on the first delivery only. + attributes: runInput.attributes ?? {}, + startedAt: now, + createdAt: now, + updatedAt: now, + }; + workflowStartedAt = +now; + // See the `runStartedReceivedAtMs` declaration above: + // turbo synthesizes the run before the real + // `run_started` response lands, so anchor RSFS here + // rather than at an actual response instant. + runStartedReceivedAtMs = +now; + span?.setAttributes({ + ...Attribute.WorkflowRunStatus('running'), + ...Attribute.WorkflowStartedAt(workflowStartedAt), + }); + } else { + let startedRun: StartedWorkflowRun; + try { + recordRunStartedCreateStart(false); + const result = await world.events.create( + runId, + runStartedEvent, + { requestId } + ); + startedRun = result.run; + workflowRun = startedRun; + maxEventsLimit = clampMaxEvents(result.maxEvents); + // Anchors RSFS — see the declaration above. + runStartedReceivedAtMs = Date.now(); + + if (result.events?.length) { + let events = result.events; + let cursor = result.cursor ?? null; + if (result.hasMore) { + const loaded = await loadWorkflowRunEvents( + runId, + cursor ?? undefined ); - } else { - events = loaded.events; + if (cursor) { + const preloadedIds = new Set( + events.map((event) => event.eventId) + ); + events = events.concat( + loaded.events.filter( + (event) => !preloadedIds.has(event.eventId) + ) + ); + } else { + events = loaded.events; + } + cursor = loaded.cursor ?? cursor; + } + preloadedEventLog = { events, cursor }; + } + } catch (err) { + // Run was concurrently completed/failed/cancelled + if ( + EntityConflictError.is(err) || + RunExpiredError.is(err) + ) { + // EntityConflictError: run was concurrently + // completed/failed/cancelled during setup. + // RunExpiredError: run already in terminal state. + // In both cases, skip processing this message. + runtimeLogger.info( + 'Run already finished during setup, skipping', + { workflowRunId: runId, message: err.message } + ); + return; + } else { + const errorCode = getWorkflowSetupErrorCode(err); + if (!errorCode) { + throw err; } - cursor = loaded.cursor ?? cursor; + await recordFatalRunError({ + world, + workflowRun, + runId, + requestId, + err, + errorCode, + logMessage: + 'Fatal runtime error during workflow setup', + }); + return; } - preloadedEventLog = { events, cursor }; } - } catch (err) { - if ( - EntityConflictError.is(err) || - RunExpiredError.is(err) - ) { + workflowStartedAt = +startedRun.startedAt; + + span?.setAttributes({ + ...Attribute.WorkflowRunStatus(startedRun.status), + ...Attribute.WorkflowStartedAt(workflowStartedAt), + }); + + if (startedRun.status !== 'running') { + // Workflow has already completed or failed, so we can skip it runtimeLogger.info( - 'Run already finished during setup, skipping', - { workflowRunId: runId, message: err.message } + 'Workflow already completed or failed, skipping', + { + workflowRunId: runId, + status: startedRun.status, + } ); - return; - } - const errorCode = getWorkflowSetupErrorCode(err); - if (!errorCode) throw err; - await recordFatalRunError({ - world, - workflowRun, - runId, - requestId, - err, - errorCode, - logMessage: 'Fatal runtime error during workflow setup', - }); - return; - } - workflowStartedAt = +startedRun.startedAt; - span?.setAttributes({ - ...Attribute.WorkflowRunStatus(startedRun.status), - ...Attribute.WorkflowStartedAt(workflowStartedAt), - }); + // TODO: for `cancel`, we actually want to propagate a WorkflowCancelled event + // inside the workflow context so the user can gracefully exit. this is SIGTERM + // TODO: furthermore, there should be a timeout or a way to force cancel SIGKILL + // so that we actually exit here without replaying the workflow at all, in the case + // the replaying the workflow is itself failing. - if (startedRun.status !== 'running') { - runtimeLogger.info( - 'Workflow already completed or failed, skipping', - { - workflowRunId: runId, - status: startedRun.status, - } - ); - return; - } + return; + } + } // end else (non-turbo run_started) } // end if (!workflowRun) // Resolve the encryption key for this run's deployment. @@ -1049,6 +1189,7 @@ export function workflowEntrypoint( ); // Main replay loop + // biome-ignore lint/correctness/noConstantCondition: intentional loop while (true) { loopIteration++; @@ -1395,6 +1536,11 @@ export function workflowEntrypoint( events, encryptionKey, replayPayloadCache, + // Turbo: the end-of-run drain inside runWorkflow commits + // fire-and-forget `*_created` events before the terminal + // `awaitRunReady()` below, so gate those writes on the + // backgrounded run_started too. Undefined outside turbo. + runReadyBarrier, world.capabilities ); await payloadPrewarm; @@ -1412,6 +1558,10 @@ export function workflowEntrypoint( // result. The catch below lets PreconditionFailedError // propagate to the queue for re-invocation. try { + // Turbo: a workflow that finishes with no steps reaches + // here before the backgrounded run_started; order the + // terminal write after it so the run exists. + await awaitRunReady(); await world.events.create( runId, { @@ -1521,6 +1671,7 @@ export function workflowEntrypoint( span, requestId, eventLog: suspensionLog, + runReadyBarrier, }); } catch (suspensionError) { // A suspension create whose stale (412) rejection @@ -1560,6 +1711,9 @@ export function workflowEntrypoint( } ); try { + // Turbo: order the terminal write after the + // backgrounded run_started so the run exists. + await awaitRunReady(); await world.events.create( runId, { @@ -2100,13 +2254,16 @@ export function workflowEntrypoint( // step qualifies for TTFS/STSO measurement. Only the // batch's first step carries the tracking so a // parallel batch emits one sample per scheduling gap, - // not one per sibling. + // not one per sibling. Turbo's synthesized run + // snapshot has a local-clock createdAt, so under + // turbo only the run-id ULID timestamp is trusted. const latencyTracking = computeStepLatencyTracking({ events: cachedEvents ?? [], invocationStartedClean: invocationStartedClean === true, runCreatedAtMs: - runIdCreatedAt(runId) ?? +workflowRun.createdAt, + runIdCreatedAt(runId) ?? + (turbo ? undefined : +workflowRun.createdAt), runStartedReceivedAtMs, replayMs: replayDurationMs, preStepBlockingMs, @@ -2199,14 +2356,18 @@ export function workflowEntrypoint( // as in flight here and suppress the // immediate requeue (workflow#2780). ownerMessageId: metadata.messageId, - // Turbo forces optimistic start on its first - // inline step. + // Turbo: force optimistic start and hold the + // lazy step_started until the backgrounded + // run_started lands (the body still runs + // immediately). Both are undefined/false + // outside turbo. forceOptimisticStart, // Guard-enforced batches with an open hook // await the claim before running the body, so // a 412-fenced step never executes user code — // see suppressOptimisticStart above. suppressOptimisticStart, + runReadyBarrier, stateUpdatedAt: inlineClaimStateUpdatedAt, ...(stepIndex === 0 && s.lazyStepInput !== undefined && @@ -2605,6 +2766,9 @@ export function workflowEntrypoint( // also need the loaded event log, which is scoped to the // replay `try` above and not available in this catch. try { + // Turbo: order the terminal write after the + // backgrounded run_started so the run exists. + await awaitRunReady(); await world.events.create( runId, { diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index e30af271a6..619c50f890 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -284,8 +284,10 @@ export function isOptimisticInlineStartExplicitlyDisabled(): boolean { /** * Whether "turbo mode" is enabled. Turbo mode fast-paths the *first delivery of * the first invocation* of a run (detected by the entrypoint via `runInput` - * presence + `metadata.attempt === 1`) and forces optimistic inline step start - * for that invocation, independent of `WORKFLOW_OPTIMISTIC_INLINE_START`. + * presence + `metadata.attempt === 1`): it backgrounds the `run_started` event + * creation, skips the initial event-log load (nothing has been written yet), + * and forces optimistic inline step start for that invocation — independent of + * `WORKFLOW_OPTIMISTIC_INLINE_START`. * * Forcing optimistic start is safe here because the first delivery has no * concurrent peer handler to race the step create-claim, so a step body runs diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 72db0ebb6c..2d22ff9b23 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -167,6 +167,15 @@ export interface StepExecutorParams { * meaningful together with `lazyStepInput` (a brand-new lazy step). */ forceOptimisticStart?: boolean; + /** + * Turbo mode only: a promise that resolves once the backgrounded + * `run_started` has landed. When set, the lazy/optimistic `step_started` is + * chained on it so the step is never created before its run exists. The body + * still runs immediately against locally-synthesized state — only the network + * write waits — so the `run_started` round-trip overlaps the body. `undefined` + * outside turbo, where `run_started` was already awaited up front. + */ + runReadyBarrier?: Promise; /** * Latency telemetry (TTFS / STSO): eligibility and anchor timestamps decided * by the orchestrator. When set, this executor computes the final values @@ -288,6 +297,13 @@ export async function executeStep( // return `skipped` and never write the failure twice. if (params.lazyStepInput !== undefined) { try { + // Turbo: this lazy `step_started` must not precede the backgrounded + // `run_started`. Order it after the run-ready barrier (best-effort — + // a barrier rejection means the run doesn't exist, and the create + // below surfaces the real error). No-op outside turbo. + if (params.runReadyBarrier) { + await params.runReadyBarrier.catch(() => {}); + } await world.events.create(workflowRunId, { eventType: 'step_started', specVersion: SPEC_VERSION_CURRENT, @@ -517,30 +533,42 @@ export async function executeStep( }; if (optimisticStart) { - stepStartPostSentAtMs = Date.now(); - const startedPromise = world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), - }, - }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - params.stateUpdatedAt !== undefined - ? { stateUpdatedAt: params.stateUpdatedAt } - : undefined + // Chain the lazy `step_started` on the run-ready barrier (turbo mode): + // the step can't be created before its run exists, but the body below + // runs immediately against synthesized state, so the `run_started` + // round-trip overlaps the body rather than blocking it. Outside turbo the + // barrier is undefined and this is a plain create. + const startedPromise = (params.runReadyBarrier ?? Promise.resolve()).then( + () => { + // Taken right before the create fires, not before the barrier — + // RSFS measures the run_started-to-POST stretch, and the barrier + // wait IS part of that stretch under turbo. + stepStartPostSentAtMs = Date.now(); + return world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, + }, + // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale + // (412) rejection surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + params.stateUpdatedAt !== undefined + ? { stateUpdatedAt: params.stateUpdatedAt } + : undefined + ); + } ); optimisticStartSettled = startedPromise.then( () => ({ ok: true as const }), @@ -833,6 +861,13 @@ export async function executeStep( preCompletionOps, closureVars: hydratedInput.closureVars, encryptionKey, + // Turbo optimistic start runs this body before `run_started` is + // durable. Expose the barrier so a direct step-body world write + // (e.g. `setAttributes`) can order itself after the + // run exists. Undefined on the await path (run already durable). + runReadyBarrier: optimisticStart + ? params.runReadyBarrier + : undefined, }, () => stepFn.apply(thisVal, args) ); @@ -873,7 +908,11 @@ export async function executeStep( globalThis, false, false, - compression + compression, + // Turbo optimistic start: a returned stream is piped to the server + // after the body but within this same op flush, so gate its first + // write on the run-ready barrier. Undefined on the await path. + optimisticStart ? params.runReadyBarrier : undefined ); const durationMs = Date.now() - startTime; dehydrateSpan?.setAttributes({ diff --git a/packages/core/src/runtime/step-latency.ts b/packages/core/src/runtime/step-latency.ts index 1d4af28f72..0a4d9b08e3 100644 --- a/packages/core/src/runtime/step-latency.ts +++ b/packages/core/src/runtime/step-latency.ts @@ -61,7 +61,14 @@ export interface StepLatencyTracking { /** * Epoch ms the `run_started` response was received/parsed by the SDK. * Present only when the step qualifies for RSFS — the same eligibility as - * TTFS (see {@link computeStepLatencyTracking}), plus a recoverable anchor. + * TTFS (see {@link computeStepLatencyTracking}), plus a recoverable + * anchor. In turbo mode, `run_started` is backgrounded rather than + * awaited, so this is stamped at the point the runtime synthesizes the + * run locally and begins replay instead of at the real response; the + * first step's start POST is still chained on the real `run_started` + * promise (see step-executor.ts), so RSFS still ends up measuring the + * genuine run_started-to-first-step-POST stretch even though the two + * halves overlap under turbo. */ rsfsAnchorMs?: number; /** @@ -167,7 +174,11 @@ export function computeStepLatencyTracking(params: { invocationStartedClean: boolean; /** Epoch ms of run creation, if recoverable. Absent disqualifies TTFS. */ runCreatedAtMs: number | undefined; - /** Epoch ms the `run_started` response was received/parsed by the SDK. */ + /** + * Epoch ms the `run_started` response was received/parsed by the SDK (or, + * under turbo, the instant the run was synthesized locally — see + * {@link StepLatencyTracking.rsfsAnchorMs}). Absent disqualifies RSFS. + */ runStartedReceivedAtMs: number | undefined; /** * Wall-clock ms this suspension's `runWorkflow` call spent executing diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 3993eaedf8..921986d7bd 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -46,6 +46,16 @@ export interface SuspensionHandlerParams { * suspension) ensures a retry never re-issues an already-created event. */ eventLog?: MutableEventLog; + /** + * Turbo mode only: a promise that resolves once the backgrounded + * `run_started` has landed (the run exists). When present, every world write + * this suspension performs (`hook_created`, `wait_created`, eager overflow + * `step_created`, …) is gated on it so the write never races ahead of the + * run's creation. The pure inline hot path defers all of its steps and writes + * nothing here, so it never awaits this barrier. `undefined` outside turbo, + * where `run_started` was already awaited up front. + */ + runReadyBarrier?: Promise; } /** @@ -194,9 +204,28 @@ export async function handleSuspension({ span, requestId, eventLog, + runReadyBarrier, }: SuspensionHandlerParams): Promise { const runId = run.runId; + // Turbo mode: hold every world write below until the backgrounded + // `run_started` has *settled*, so we never write a step/hook/wait event for a + // run that does not exist yet. A no-op outside turbo (barrier undefined) and + // on the pure inline hot path, which defers all steps and writes nothing. + // Awaiting the same (usually already-settled) promise more than once is cheap. + // A barrier rejection is swallowed for ordering only: if `run_started` truly + // failed the run does not exist, so the subsequent write surfaces the real + // error (run not found / gone) and the message redelivers. + const ensureRunReady = async (): Promise => { + if (runReadyBarrier) { + try { + await runReadyBarrier; + } catch { + // intentional: ordering barrier only — see above. + } + } + }; + // Create an event with the optimistic-concurrency guard when the caller // supplied a loaded event log; otherwise create it directly (callers without // a replay snapshot, e.g. tests). The guard reloads + retries on a stale @@ -315,6 +344,7 @@ export async function handleSuspension({ if (hookItemsByToken.size > 0) { const hookPhaseStart = Date.now(); + await ensureRunReady(); await Promise.all( [...hookItemsByToken.values()].map(async (items) => { for (const queueItem of items) { @@ -374,6 +404,7 @@ export async function handleSuspension({ ); if (hooksNeedingAbort.length > 0) { + await ensureRunReady(); await Promise.all( hooksNeedingAbort.map(async (queueItem) => { try { @@ -527,6 +558,7 @@ export async function handleSuspension({ }, }; try { + await ensureRunReady(); await createGuarded(stepEvent, { requestId }); createdStepCorrelationIds.add(queueItem.correlationId); } catch (err) { @@ -559,6 +591,7 @@ export async function handleSuspension({ }, }; try { + await ensureRunReady(); await createGuarded(waitEvent, { requestId }); } catch (err) { if (EntityConflictError.is(err)) { @@ -580,6 +613,7 @@ export async function handleSuspension({ ops.push( (async () => { try { + await ensureRunReady(); await world.events.create( runId, { diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index e7a8d5a79a..661cd5bd9a 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -1046,7 +1046,16 @@ function recordStreamClose( } export class WorkflowServerWritableStream extends WritableStream { - constructor(runId: string, name: string) { + /** + * @param runReadyBarrier Turbo mode only: a promise that resolves once the + * backgrounded `run_started` has landed. When the step body runs + * optimistically (before `run_started` is durable), the first chunk write to + * a brand-new stream would otherwise reach the World before the run exists + * and be rejected as run-not-found. Awaiting this once before the first + * flush/close orders the write after the run's creation. `undefined` outside + * turbo and on the await path, where the run was already durable. + */ + constructor(runId: string, name: string, runReadyBarrier?: Promise) { if (typeof runId !== 'string') { throw new WorkflowRuntimeError( `"runId" must be a string, got "${typeof runId}"` @@ -1057,6 +1066,22 @@ export class WorkflowServerWritableStream extends WritableStream { } const worldPromise = getWorldLazy(); + // Hold the first server write until the run exists (turbo optimistic + // start). Awaited once, then cleared so later flushes pay nothing. The + // rejection is swallowed for ordering only: if `run_started` truly failed + // the run does not exist, so the write below surfaces the real error. + let pendingRunReady: Promise | undefined = runReadyBarrier; + const ensureRunReady = async (): Promise => { + if (pendingRunReady) { + try { + await pendingRunReady; + } catch { + // intentional: ordering barrier only — see above. + } + pendingRunReady = undefined; + } + }; + // ------------------------------------------------------------------ // Group-commit buffering. // @@ -1161,14 +1186,18 @@ export class WorkflowServerWritableStream extends WritableStream { * behavior, now independent of how the producer pipes. */ /** - * Send one group to the server with one `writeMulti` call (or sequential - * `write`s when the world lacks it). + * Send one group to the server: gate on run readiness, then one + * `writeMulti` (or sequential `write`s when the world lacks it). Emits + * the write-flush span; dwell is measured up to just before the RPC so a + * turbo run-ready barrier wait counts as buffer dwell, matching the + * pre-group-commit telemetry. */ const sendGroup = async ( group: Uint8Array[], bytes: number, groupT0: number | undefined ): Promise => { + await ensureRunReady(); const world = await worldPromise; const dispatchAt = Date.now(); if (typeof world.streams.writeMulti === 'function' && group.length > 1) { @@ -1372,6 +1401,11 @@ export class WorkflowServerWritableStream extends WritableStream { // closed — the server fences post-close writes. await drain(); + // A close with an empty buffer skips the dispatch path (and its + // barrier), but can itself be the first write to a brand-new + // stream — gate it too. + await ensureRunReady(); + const world = await worldPromise; const closeStart = Date.now(); await world.streams.close(runId, name); @@ -1676,7 +1710,13 @@ export function getExternalReducers( ops: Promise[], runId: string, cryptoKey: EncryptionKeyParam, - framedByteStreams = false + framedByteStreams = false, + // Turbo optimistic start: a nested `ReadableStream` found while serializing + // is piped to its own server stream independently of the outer sink, so its + // first chunk can race `run_started`. Thread the run-ready barrier into that + // sink so the write orders after the run exists. Undefined outside turbo / + // on the await path. + runReadyBarrier?: Promise ): Partial { return { ...getAllBaseReducers(global), @@ -1698,7 +1738,11 @@ export function getExternalReducers( const name = `strm_${streamId}`; const type = getStreamType(value); - const writable = new WorkflowServerWritableStream(runId, name); + const writable = new WorkflowServerWritableStream( + runId, + name, + runReadyBarrier + ); if (type === 'bytes') { if (framedByteStreams) { ops.push(value.pipeThrough(getByteFramingStream()).pipeTo(writable)); @@ -1715,7 +1759,8 @@ export function getExternalReducers( ops, runId, cryptoKey, - framedByteStreams + framedByteStreams, + runReadyBarrier ), cryptoKey ) @@ -1905,7 +1950,12 @@ function getStepReducers( ops: Promise[], runId: string, cryptoKey: EncryptionKeyParam, - framedByteStreams = false + framedByteStreams = false, + // Turbo optimistic start: a returned `ReadableStream` is piped to the server + // after the body but within the same op flush, so its first chunk can race + // `run_started`. Thread the run-ready barrier into the sink so that write + // orders after the run exists. Undefined outside turbo / on the await path. + runReadyBarrier?: Promise ): Partial { return { ...getAllBaseReducers(global), @@ -1941,7 +1991,11 @@ function getStepReducers( type = getStreamType(value); framing = type === 'bytes' && framedByteStreams ? 'framed-v1' : framing; - const writable = new WorkflowServerWritableStream(runId, name); + const writable = new WorkflowServerWritableStream( + runId, + name, + runReadyBarrier + ); if (type === 'bytes') { if (framing === 'framed-v1') { ops.push( @@ -1960,7 +2014,8 @@ function getStepReducers( ops, runId, cryptoKey, - framedByteStreams + framedByteStreams, + runReadyBarrier ), cryptoKey ) @@ -3328,12 +3383,23 @@ export async function dehydrateStepReturnValue( global: Record = globalThis, v1Compat = false, framedByteStreams = false, - compression = false + compression = false, + // Turbo optimistic start: order the first chunk of a returned stream after + // the backgrounded `run_started`. Threaded into the step reducers' stream + // sink. Undefined outside turbo / on the await path. + runReadyBarrier?: Promise ): Promise { if (v1Compat) { const str = stringify( value, - getStepReducers(global, ops, runId, key, framedByteStreams) + getStepReducers( + global, + ops, + runId, + key, + framedByteStreams, + runReadyBarrier + ) ); return revive(str); } @@ -3342,7 +3408,14 @@ export async function dehydrateStepReturnValue( const result = await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers( - getStepReducers(global, ops, runId, key, framedByteStreams) + getStepReducers( + global, + ops, + runId, + key, + framedByteStreams, + runReadyBarrier + ) ), compression, compressionStats, diff --git a/packages/core/src/set-attributes.test.ts b/packages/core/src/set-attributes.test.ts index 1a9e9b92ac..96c5861199 100644 --- a/packages/core/src/set-attributes.test.ts +++ b/packages/core/src/set-attributes.test.ts @@ -113,6 +113,60 @@ describe('setAttributes (host-side)', () => { ); }); + it('waits for runReadyBarrier before posting the event (turbo optimistic start)', async () => { + const order: string[] = []; + const create = vi.fn().mockImplementation(async () => { + order.push('create'); + return {}; + }); + globals[WORLD_CACHE] = { + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + }; + + let releaseBarrier!: () => void; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = () => { + order.push('barrier'); + resolve(); + }; + }); + + const call = contextStorage.run({ ...stepContext(), runReadyBarrier }, () => + setAttributes({ phase: 'ready' }) + ); + + // The body ran before run_started is durable: the write must not fire yet. + await Promise.resolve(); + expect(create).not.toHaveBeenCalled(); + + releaseBarrier(); + await call; + + // The attr_set create lands strictly after the run-ready barrier resolves. + expect(order).toEqual(['barrier', 'create']); + }); + + it('still posts when the runReadyBarrier rejects (write surfaces the real error)', async () => { + const create = vi.fn().mockResolvedValue({}); + globals[WORLD_CACHE] = { + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + }; + + const runReadyBarrier = Promise.reject(new Error('run_started failed')); + // Pre-attach a catch so the rejection never surfaces as unhandled. + runReadyBarrier.catch(() => {}); + + await contextStorage.run({ ...stepContext(), runReadyBarrier }, () => + setAttributes({ phase: 'ready' }) + ); + + // Barrier rejection is swallowed for ordering only — the write still fires + // and would surface a genuine run-not-found error from the World itself. + expect(create).toHaveBeenCalledTimes(1); + }); + it('keeps the deprecated experimental_setAttributes alias working', async () => { expect(experimental_setAttributes).toBe(setAttributes); }); diff --git a/packages/core/src/set-attributes.ts b/packages/core/src/set-attributes.ts index 7eb7920306..072714a616 100644 --- a/packages/core/src/set-attributes.ts +++ b/packages/core/src/set-attributes.ts @@ -35,6 +35,20 @@ export async function setAttributes( const changes = normalizeAttributeChanges(attrs, options); if (changes.length === 0) return; + // Turbo optimistic start runs the step body before the backgrounded + // `run_started` is durable. Order this `attr_set` after the run exists so it + // never reaches the World before the run does (which would be rejected as + // run-not-found). A no-op outside turbo (barrier undefined) and on the await + // path. The rejection is swallowed for ordering only: if `run_started` truly + // failed the run does not exist, so the create below surfaces the real error. + if (store.runReadyBarrier) { + try { + await store.runReadyBarrier; + } catch { + // intentional: ordering barrier only — see above. + } + } + const world = await getWorldLazy(); await world.events.create(runId, { eventType: 'attr_set', diff --git a/packages/core/src/step/context-storage.ts b/packages/core/src/step/context-storage.ts index 8fa82830e0..10b665bbcf 100644 --- a/packages/core/src/step/context-storage.ts +++ b/packages/core/src/step/context-storage.ts @@ -59,6 +59,16 @@ export type StepContext = { closureVars?: Record; encryptionKey?: CryptoKey; writables?: Map; + /** + * Turbo mode only: a promise that resolves once the backgrounded + * `run_started` has landed (the run exists). Set when the step body runs + * optimistically — before `run_started`/`step_started` are confirmed — so a + * direct step-body world write (e.g. `setAttributes`, which + * resolves to a host-side `attr_set` create) can gate on it and never race + * ahead of the run's creation. `undefined` outside turbo and on the await + * path, where `run_started` was already durable before the body ran. + */ + runReadyBarrier?: Promise; }; /** diff --git a/packages/core/src/step/writable-stream.ts b/packages/core/src/step/writable-stream.ts index 5e3463b23e..91b8534c26 100644 --- a/packages/core/src/step/writable-stream.ts +++ b/packages/core/src/step/writable-stream.ts @@ -78,7 +78,18 @@ export function getWritable( // version skew protection) is on this same SDK version, so byte-stream // framing is always safe here. const serialize = getSerializeStream( - getExternalReducers(globalThis, ctx.ops, runId, ctx.encryptionKey, true), + // In turbo optimistic start the body runs before `run_started` is durable. + // Thread the run-ready barrier so that a nested ReadableStream written into + // this writable is piped to its own server stream only after the run + // exists. Undefined outside turbo / on the await path. + getExternalReducers( + globalThis, + ctx.ops, + runId, + ctx.encryptionKey, + true, + ctx.runReadyBarrier + ), ctx.encryptionKey ); @@ -86,7 +97,14 @@ export function getWritable( // their writer lock, not only when the stream is explicitly closed. // Without this, Vercel functions hang until the runtime timeout because // .pipeTo() only resolves on stream close. - const serverWritable = new WorkflowServerWritableStream(runId, name); + // In turbo optimistic start the body runs before `run_started` is durable; + // pass the run-ready barrier so the first server write orders after the run + // exists. Undefined outside turbo / on the await path (run already durable). + const serverWritable = new WorkflowServerWritableStream( + runId, + name, + ctx.runReadyBarrier + ); const state = createFlushableState(); ctx.ops.push(state.promise); diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 5cda61f41f..d66faa9fb1 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -214,10 +214,11 @@ export const StepStsoMs = SemanticConvention('step.stso_ms'); /** * Client-measured run_started-to-first-step latency in milliseconds: the - * `run_started` response landing → this step's start POST being issued. A - * sub-window of ttfs that isolates replay overhead from the run-creation queue - * hop. Only present on the run's first step execution when it qualified for - * measurement (see runtime/step-latency.ts). + * `run_started` response landing (or, under turbo, the local run synthesis + * instant) → this step's start POST being issued. A sub-window of ttfs that + * isolates replay overhead from the run-creation queue hop. Only present on + * the run's first step execution when it qualified for measurement (see + * runtime/step-latency.ts). */ export const StepRsfsMs = SemanticConvention('step.rsfs_ms'); diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 8e8f299d5e..2d903b94a7 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -5433,7 +5433,13 @@ describe('runWorkflow', () => { } }, 30_000); - describe('end-of-run drain', () => { + describe('turbo: end-of-run drain gates writes on runReadyBarrier', () => { + // A workflow that creates a fire-and-forget hook and then returns + // synchronously never suspends, so its `hook_created` event is committed by + // the end-of-run drain inside runWorkflow — *before* the runtime's terminal + // `awaitRunReady()`. In turbo (`run_started` backgrounded), that write must + // still be ordered after the run exists, so runWorkflow threads the barrier + // into the drain. These tests pin that gating. const FIRE_AND_FORGET_HOOK = `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; async function workflow() { createHook({ token: 'fire-and-forget' }); @@ -5446,6 +5452,14 @@ describe('runWorkflow', () => { return 'done'; }`; + // `void sleep(...)` is the wait equivalent: it enqueues a wait_created the + // workflow never awaits, drained the same way as the hook above. + const FIRE_AND_FORGET_WAIT = `const sleep = globalThis[Symbol.for("WORKFLOW_SLEEP")]; + async function workflow() { + void sleep('1d'); + return 'done'; + }`; + async function makeRun(): Promise { const ops: Promise[] = []; return { @@ -5477,6 +5491,7 @@ describe('runWorkflow', () => { [], noEncryptionKey, undefined, + undefined, { hookRetention: { active: false } } ) ).rejects.toThrow( @@ -5484,7 +5499,130 @@ describe('runWorkflow', () => { ); }); - it('writes fire-and-forget hooks before returning', async () => { + it('does not write hook_created until the runReadyBarrier resolves', async () => { + let releaseBarrier: () => void = () => {}; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = resolve; + }); + + const create = vi.fn(async () => ({ + event: { eventType: 'hook_created' as const }, + })); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + streams: { write: vi.fn(), close: vi.fn() }, + } as any); + + const runPromise = runWorkflow( + `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, + await makeRun(), + [], + noEncryptionKey, + undefined, + runReadyBarrier + ).then(() => 'completed' as const); + + // The body returns synchronously, but the drain's hook_created write — and + // therefore runWorkflow's own completion — must stay blocked behind the + // still-pending backgrounded run_started. A fixed-time race (not a fixed + // wait-then-assert) makes this robust to VM setup latency: the window only + // has to exceed the drain's own work, never a calibrated guess at it. + const winner = await Promise.race([ + runPromise, + new Promise<'pending'>((resolve) => + setTimeout(() => resolve('pending'), 250) + ), + ]); + expect(winner).toBe('pending'); + expect(create).not.toHaveBeenCalled(); + + // Once run_started lands, the drain proceeds and runWorkflow resolves. + releaseBarrier(); + expect(await runPromise).toBe('completed'); + + expect(create).toHaveBeenCalledTimes(1); + expect(create.mock.calls[0][1]).toMatchObject({ + eventType: 'hook_created', + }); + }); + + it('does not write wait_created until the runReadyBarrier resolves', async () => { + let releaseBarrier: () => void = () => {}; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = resolve; + }); + + const create = vi.fn(async () => ({ + event: { eventType: 'wait_created' as const }, + })); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + streams: { write: vi.fn(), close: vi.fn() }, + } as any); + + const runPromise = runWorkflow( + `${FIRE_AND_FORGET_WAIT}${getWorkflowTransformCode('workflow')}`, + await makeRun(), + [], + noEncryptionKey, + undefined, + runReadyBarrier + ).then(() => 'completed' as const); + + // Same gating as the hook case: a fire-and-forget wait drained at + // completion must not write wait_created before the backgrounded + // run_started lands. + const winner = await Promise.race([ + runPromise, + new Promise<'pending'>((resolve) => + setTimeout(() => resolve('pending'), 250) + ), + ]); + expect(winner).toBe('pending'); + expect(create).not.toHaveBeenCalled(); + + releaseBarrier(); + expect(await runPromise).toBe('completed'); + + expect(create).toHaveBeenCalledTimes(1); + expect(create.mock.calls[0][1]).toMatchObject({ + eventType: 'wait_created', + }); + }); + + it('still writes hook_created when the runReadyBarrier rejects (ordering only)', async () => { + const create = vi.fn(async () => ({ + event: { eventType: 'hook_created' as const }, + })); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + streams: { write: vi.fn(), close: vi.fn() }, + } as any); + + // A rejected barrier is swallowed for ordering: if run_started truly + // failed, the run does not exist and the write itself surfaces the error. + const rejected = Promise.reject(new Error('run_started failed')); + rejected.catch(() => {}); + + await runWorkflow( + `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, + await makeRun(), + [], + noEncryptionKey, + undefined, + rejected + ); + + expect(create).toHaveBeenCalledTimes(1); + expect(create.mock.calls[0][1]).toMatchObject({ + eventType: 'hook_created', + }); + }); + + it('writes hook_created without blocking when no barrier (non-turbo)', async () => { const create = vi.fn(async () => ({ event: { eventType: 'hook_created' as const }, })); @@ -5494,6 +5632,8 @@ describe('runWorkflow', () => { streams: { write: vi.fn(), close: vi.fn() }, } as any); + // No barrier (the await path already awaited run_started up front): the + // drain writes immediately. await runWorkflow( `${FIRE_AND_FORGET_HOOK}${getWorkflowTransformCode('workflow')}`, await makeRun(), diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 8cf97ffad3..4dfd069e4d 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -68,7 +68,12 @@ async function drainPendingQueueItems( pendingQueue: Map, vmGlobalThis: typeof globalThis, workflowRun: WorkflowRun, - outcome: 'completed' | 'failed' + outcome: 'completed' | 'failed', + /** + * In turbo mode, gates final `*_created` writes on backgrounded + * `run_started`. Undefined when `run_started` is awaited. + */ + runReadyBarrier?: Promise ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -94,6 +99,7 @@ async function drainPendingQueueItems( suspension: synthesized, world, run: workflowRun, + runReadyBarrier, }); } catch (err) { runtimeLogger.warn( @@ -119,6 +125,13 @@ export async function runWorkflow( replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache( encryptionKey ), + /** + * Turbo mode only: resolves once the backgrounded `run_started` has landed. + * Threaded into the end-of-run drain so fire-and-forget `*_created` writes + * committed at workflow completion order after the run's creation. Undefined + * outside turbo, where `run_started` is awaited up front. + */ + runReadyBarrier?: Promise, /** * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. @@ -837,7 +850,8 @@ export async function runWorkflow( workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'completed' + 'completed', + runReadyBarrier ); return dehydrated; @@ -853,7 +867,8 @@ export async function runWorkflow( workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'failed' + 'failed', + runReadyBarrier ); throw err; diff --git a/packages/core/src/writable-stream-telemetry.test.ts b/packages/core/src/writable-stream-telemetry.test.ts index e3b6e52049..81a939df6f 100644 --- a/packages/core/src/writable-stream-telemetry.test.ts +++ b/packages/core/src/writable-stream-telemetry.test.ts @@ -147,6 +147,34 @@ describe('WorkflowServerWritableStream write-flush telemetry', () => { } }); + it('counts the turbo run-ready barrier wait as buffer dwell', async () => { + let releaseBarrier!: () => void; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = resolve; + }); + + const stream = new WorkflowServerWritableStream( + 'run-123', + 'test-stream', + runReadyBarrier + ); + const writer = stream.getWriter(); + + await writer.write(new Uint8Array([1, 2, 3])); + // Hold the first dispatch on the barrier long enough to dominate the + // dwell (write() itself acks on buffer entry and does not wait). + await new Promise((r) => setTimeout(r, 50)); + releaseBarrier(); + await writer.close(); + + const [span] = await waitForSpans('workflow.stream.flush', 1); + expect(span).toBeDefined(); + const dwell = span.attributes[ + 'workflow.stream.flush.buffer_dwell_ms' + ] as number; + expect(dwell).toBeGreaterThanOrEqual(40); + }); + it('emits a workflow.stream.close span with the close RPC duration', async () => { const stream = new WorkflowServerWritableStream('run-123', 'test-stream'); const writer = stream.getWriter(); diff --git a/packages/core/src/writable-stream.test.ts b/packages/core/src/writable-stream.test.ts index 233442e162..8dcbe253bd 100644 --- a/packages/core/src/writable-stream.test.ts +++ b/packages/core/src/writable-stream.test.ts @@ -2,7 +2,10 @@ import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createFlushableState, flushablePipe } from './flushable-stream.js'; import { setWorld } from './runtime/world.js'; -import { WorkflowServerWritableStream } from './serialization.js'; +import { + dehydrateStepReturnValue, + WorkflowServerWritableStream, +} from './serialization.js'; /** * Poll until the expectation passes — replaces fixed sleeps for @@ -661,6 +664,174 @@ describe('WorkflowServerWritableStream', () => { }); }); + describe('runReadyBarrier (turbo optimistic start)', () => { + it('holds the first server write until the run-ready barrier resolves', async () => { + const order: string[] = []; + mockStreams.write.mockImplementation(async () => { + order.push('write'); + }); + + let releaseBarrier!: () => void; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = () => { + order.push('barrier'); + resolve(); + }; + }); + + const stream = new WorkflowServerWritableStream( + 'run-123', + 'test-stream', + runReadyBarrier + ); + const writer = stream.getWriter(); + await writer.write(new Uint8Array([1, 2, 3])); + + // The body wrote a chunk before run_started is durable: the commit + // window may fire, but the server write must not happen until the + // run exists. + await new Promise((r) => setTimeout(r, 30)); + expect(mockStreams.write).not.toHaveBeenCalled(); + + releaseBarrier(); + // close() drains: it completes only after the gated dispatch lands. + await writer.close(); + + // The chunk reaches the server strictly after the barrier resolves. + expect(order).toEqual(['barrier', 'write']); + }); + + it('only awaits the barrier once — later flushes are not gated', async () => { + const runReadyBarrier = Promise.resolve(); + const stream = new WorkflowServerWritableStream( + 'run-123', + 'test-stream', + runReadyBarrier + ); + const writer = stream.getWriter(); + + await writer.write(new Uint8Array([1])); + await writer.write(new Uint8Array([2])); + await writer.write(new Uint8Array([3])); + await writer.close(); + + // All three chunks are delivered (grouping may vary); the resolved + // barrier gated nothing after the first dispatch. + const delivered = [ + ...mockStreams.write.mock.calls.map( + (call: unknown[]) => (call[2] as Uint8Array)[0] + ), + ...mockStreams.writeMulti.mock.calls.flatMap((call: unknown[]) => + (call[2] as Uint8Array[]).map((c) => c[0]) + ), + ]; + expect(delivered.sort()).toEqual([1, 2, 3]); + }); + + it('still writes when the barrier rejects (write surfaces the real error)', async () => { + const runReadyBarrier = Promise.reject(new Error('run_started failed')); + runReadyBarrier.catch(() => {}); + + const stream = new WorkflowServerWritableStream( + 'run-123', + 'test-stream', + runReadyBarrier + ); + const writer = stream.getWriter(); + + await writer.write(new Uint8Array([1, 2, 3])); + await writer.close(); + + // Barrier rejection is swallowed for ordering only — the write still + // fires and would surface a genuine run-not-found error from the World. + expect(mockStreams.write).toHaveBeenCalledTimes(1); + }); + + it('gates the first write of a stream RETURNED from a turbo first step', async () => { + // Regression: getWritable()/setAttributes are gated while the step + // context is active, but a step that *returns* a fresh ReadableStream + // is serialized after the body via dehydrateStepReturnValue(), whose + // sink must also wait for run_started before the first chunk lands. + const order: string[] = []; + mockStreams.write.mockImplementation(async () => { + order.push('write'); + }); + + let releaseBarrier!: () => void; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = () => { + order.push('barrier'); + resolve(); + }; + }); + + const returned = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + controller.close(); + }, + }); + + const ops: Promise[] = []; + await dehydrateStepReturnValue( + returned, + 'run-123', + undefined, // encryption key + ops, + globalThis, + false, // v1Compat + false, // framedByteStreams + false, // compression + runReadyBarrier + ); + + // The pipe is queued but must not have written before the run exists. + await new Promise((r) => setTimeout(r, 30)); + expect(mockStreams.write).not.toHaveBeenCalled(); + + releaseBarrier(); + await Promise.all(ops); + + // The returned stream's first chunk reaches the server only after the + // run-ready barrier resolves. + expect(order[0]).toBe('barrier'); + expect(mockStreams.write).toHaveBeenCalled(); + }); + + it('gates a close that is itself the first write to a new stream', async () => { + const order: string[] = []; + mockStreams.close.mockImplementation(async () => { + order.push('close'); + }); + + let releaseBarrier!: () => void; + const runReadyBarrier = new Promise((resolve) => { + releaseBarrier = () => { + order.push('barrier'); + resolve(); + }; + }); + + const stream = new WorkflowServerWritableStream( + 'run-123', + 'test-stream', + runReadyBarrier + ); + const writer = stream.getWriter(); + + // Close with no chunks written: flush() short-circuits on the empty + // buffer, so close() must apply the barrier itself. + const closePromise = writer.close(); + await new Promise((r) => setTimeout(r, 30)); + expect(mockStreams.close).not.toHaveBeenCalled(); + + releaseBarrier(); + await closePromise; + + expect(order).toEqual(['barrier', 'close']); + }); + }); + describe('writeMulti batching via flushablePipe (equivalent to the native path)', () => { it('coalesces chunks that arrive during an in-flight write into one writeMulti', async () => { // Hold the first server write in flight so the chunks produced while it diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index cfed551bab..4506a82e51 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -489,6 +489,60 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('keeps the CBOR response when run_started skips the preload', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: (headers) => headers.accept === '*/*', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ run: { runId: 'wrun_1', status: 'running' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_2', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const result = await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'run_started', + specVersion: 5, + skipPreload: true, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.skipPreload).toBe(true); + expect(result.type).toBe('event'); + expect(result.body.run).toMatchObject({ status: 'running' }); + agent.assertNoPendingInterceptors(); + }); + it('forwards stateUpdatedAt in the frame meta (precondition guard)', async () => { const origin = 'https://vercel-workflow.com'; const agent = new MockAgent(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 0ebb9e11b5..d2a49aa088 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -189,6 +189,11 @@ export interface CreateEventV4Input { * other event types; older servers ignore it entirely (the runtime then * falls back to events.list). */ sinceCursor?: string; + /** Run-started preload opt-out. Turbo backgrounds run_started as a write + * barrier only and never reads the preloaded log, so it asks the server to + * skip the list+resolve. Acted on by the server only for run_started; + * older servers ignore it and preload as before. */ + skipPreload?: boolean; /** * Epoch ms (the ULID time of the latest event the runtime has loaded * during replay). Sent by replay-context creates so the backend can @@ -292,6 +297,7 @@ function buildPostFrameMeta( meta.optimizations = input.optimizations; } if (input.sinceCursor !== undefined) meta.sinceCursor = input.sinceCursor; + if (input.skipPreload !== undefined) meta.skipPreload = input.skipPreload; if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } @@ -362,7 +368,8 @@ export function throwForErrorResponse( * POST /api/v4/runs/:runId/events/:eventType * * Sends the full request as a single v4 frame. A run_started response may - * include the first event page as frames; other responses use CBOR. + * include the first event page as frames; skipPreload and other events use + * CBOR. * * The trailing `:eventType` path segment is an alias of the canonical * `/events` route: it exists purely so the event type is visible in @@ -379,7 +386,7 @@ export async function createWorkflowRunEventV4( const { baseUrl, headers: baseHeaders } = await getHttpConfig(config); const headers = new Headers(baseHeaders); headers.set('Content-Type', 'application/octet-stream'); - if (input.eventType === 'run_started') { + if (input.eventType === 'run_started' && !input.skipPreload) { headers.set('Accept', V4_FRAME_CONTENT_TYPE); } diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 868ee54c42..635fc1e2d4 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -721,6 +721,11 @@ async function createWorkflowRunEventInner( // step_completed/step_failed; older servers ignore it and the runtime // falls back to events.list. ...(params?.sinceCursor ? { sinceCursor: params.sinceCursor } : {}), + // Run-started preload opt-out: turbo backgrounds run_started as a write + // barrier only and never reads the preloaded log, so tell the server to + // skip the list+resolve. The server only acts on it for run_started; + // older servers ignore it and simply preload as before. + ...(params?.skipPreload ? { skipPreload: true } : {}), remoteRefBehavior, payload, ...meta, diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 148ab095ca..6745875804 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -742,6 +742,23 @@ export interface CreateEventParams { * delta is computed atomically against the same log the fetch would read. */ sinceCursor?: string; + /** + * Run-started preload opt-out (advisory). On a `run_started` write a World + * MAY preload the run's event log onto the {@link EventResult} + * (`events`/`cursor`/`hasMore`) so the runtime can skip its initial + * `events.list`. The turbo first invocation backgrounds `run_started` + * purely as a write barrier and never reads that preload, so it sets this + * to tell the World to skip the wasted list+resolve — trimming the + * `run_started` round-trip that the chained first `step_started` waits on. + * A World that ignores it (or doesn't preload) remains fully correct: the + * runtime falls back to `events.list` whenever it actually needs the log. + * Only honored for `run_started`; ignored for other event types. + * + * Named to match the World boundary, the wire frame meta, and the backend + * option end-to-end (cf. {@link sinceCursor}) so the single name greps + * across the SDK and the backend. + */ + skipPreload?: boolean; } /** From c11d0701eb1f758be7b6d7107e571a5b89cac953 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:57:09 -0700 Subject: [PATCH 3/3] refactor(world-vercel): simplify run start stream --- .../world-vercel/src/events-retry.test.ts | 1 + packages/world-vercel/src/events-v4.test.ts | 19 +-- packages/world-vercel/src/events-v4.ts | 140 ++++++++++++------ packages/world-vercel/src/events.test.ts | 60 ++++---- packages/world-vercel/src/events.ts | 83 ++++++----- 5 files changed, 169 insertions(+), 134 deletions(-) diff --git a/packages/world-vercel/src/events-retry.test.ts b/packages/world-vercel/src/events-retry.test.ts index 68be2d0982..8de70fd2b2 100644 --- a/packages/world-vercel/src/events-retry.test.ts +++ b/packages/world-vercel/src/events-retry.test.ts @@ -38,6 +38,7 @@ const stepCompleted = () => ({ }); const v4Success = () => ({ + type: 'event' as const, eventId: 'evnt_1', runId: RUN_ID, createdAt: '2020-01-01T00:00:00.000Z', diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 4506a82e51..2781fa2e61 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import { EntityConflictError, RunExpiredError, @@ -17,18 +18,6 @@ import { import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; -function joinFrames(...frames: Uint8Array[]): Uint8Array { - const joined = new Uint8Array( - frames.reduce((length, frame) => length + frame.byteLength, 0) - ); - let offset = 0; - for (const frame of frames) { - joined.set(frame, offset); - offset += frame.byteLength; - } - return joined; -} - /** * The v4 client must preserve the typed-error contract of the v3 * `makeRequest` path — the workflow runtime branches on these types @@ -426,7 +415,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }) .reply( 200, - joinFrames( + Buffer.concat([ encodeFrame( { _result: 1, @@ -458,8 +447,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { encodeFrame( { _end: 1, next: 'eid:evnt_2', hasMore: false }, new Uint8Array() - ) - ), + ), + ]), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE, diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index d2a49aa088..34e88a6686 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -235,11 +235,19 @@ interface CreateEventV4ResultBase { }; } +type RequiredEventPageV4Result = Omit< + ListEventsV4Result, + 'next' | 'hasMore' +> & { + next: string; + hasMore: boolean; +}; + export type CreateEventV4Result = | (CreateEventV4ResultBase & { type: 'event' }) | (CreateEventV4ResultBase & { type: 'event-page'; - page: ListEventsV4Result; + page: RequiredEventPageV4Result; }); /** Build the CBOR meta map for a v4 POST frame. Drops undefined entries @@ -421,23 +429,26 @@ export async function createWorkflowRunEventV4( 'v4 createEvent: received an event page for a non-run_started event' ); } - const { result, ...page } = await consumeEventFrameStream( - response, - 'createEvent' - ); - if (!result) { - throw new Error( - 'v4 createEvent: frame stream ended without the result frame' - ); + const stream = await consumeEventFrameStream(response, 'createEvent'); + switch (stream.type) { + case 'events': + throw new Error( + 'v4 createEvent: frame stream ended without the result frame' + ); + case 'event-page': + return { + type: 'event-page', + eventId, + runId, + createdAt, + body: stream.body, + page: stream.page, + }; + default: { + const exhaustive: never = stream; + throw new Error(`v4 createEvent: unknown stream type ${exhaustive}`); + } } - return { - type: 'event-page', - eventId, - runId, - createdAt, - body: result, - page, - }; } // Decode the materialized-entity bag from the CBOR response body. @@ -570,8 +581,47 @@ export interface ListEventsV4Result { hasMore?: boolean; } -interface EventFrameStreamResult extends ListEventsV4Result { - result?: CreateEventV4ResultBase['body']; +type EventFrameStreamResult = + | { type: 'events'; page: ListEventsV4Result } + | { + type: 'event-page'; + body: CreateEventV4ResultBase['body']; + page: RequiredEventPageV4Result; + }; + +type EventFrameStreamState = + | { type: 'events'; events: ListedEventV4[] } + | { + type: 'event-page'; + body: CreateEventV4ResultBase['body']; + events: ListedEventV4[]; + }; + +function finishEventFrameStream( + state: EventFrameStreamState, + meta: Record, + opName: string +): EventFrameStreamResult { + const next = typeof meta.next === 'string' ? meta.next : undefined; + const hasMore = typeof meta.hasMore === 'boolean' ? meta.hasMore : undefined; + + switch (state.type) { + case 'events': + return { type: 'events', page: { events: state.events, next, hasMore } }; + case 'event-page': + if (next === undefined || hasMore === undefined) { + throw new Error(`v4 ${opName}: event page missing pagination state`); + } + return { + type: 'event-page', + body: state.body, + page: { events: state.events, next, hasMore }, + }; + default: { + const exhaustive: never = state; + throw new Error(`v4 ${opName}: unknown stream state ${exhaustive}`); + } + } } async function consumeEventFrameStream( @@ -586,46 +636,34 @@ async function consumeEventFrameStream( } const chunks = response.body as unknown as AsyncIterable; - const events: ListedEventV4[] = []; - let result: CreateEventV4ResultBase['body'] | undefined; - let next: string | undefined; - let hasMore: boolean | undefined; - let sawEndSentinel = false; + let state: EventFrameStreamState = { type: 'events', events: [] }; for await (const frame of decodeFrames(chunks)) { if (frame.meta._result === 1) { - if (result || events.length > 0 || frame.body.byteLength > 0) { + if ( + state.type !== 'events' || + state.events.length > 0 || + frame.body.byteLength > 0 + ) { throw new Error(`v4 ${opName}: invalid result frame`); } const { _result, ...body } = frame.meta; - result = body; + state = { type: 'event-page', body, events: state.events }; continue; } if (frame.meta._end === 1) { - if (typeof frame.meta.next === 'string') next = frame.meta.next; - if (typeof frame.meta.hasMore === 'boolean') hasMore = frame.meta.hasMore; - sawEndSentinel = true; - break; + return finishEventFrameStream(state, frame.meta, opName); } - events.push({ + state.events.push({ event: frame.meta as unknown as DecodedV4Event, body: frame.body, }); } - if (!sawEndSentinel) { - throw new Error( - `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + - `(${events.length} events read) — truncated response?` - ); - } - - return { - events, - ...(result ? { result } : {}), - ...(next ? { next } : {}), - ...(hasMore !== undefined ? { hasMore } : {}), - }; + throw new Error( + `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + + `(${state.events.length} events read) — truncated response?` + ); } /** @@ -649,11 +687,17 @@ async function consumeListFrameStream( config, opName ); - const { result, ...page } = await consumeEventFrameStream(response, opName); - if (result) { - throw new Error(`v4 ${opName}: unexpected result frame`); + const stream = await consumeEventFrameStream(response, opName); + switch (stream.type) { + case 'events': + return stream.page; + case 'event-page': + throw new Error(`v4 ${opName}: unexpected result frame`); + default: { + const exhaustive: never = stream; + throw new Error(`v4 ${opName}: unknown stream type ${exhaustive}`); + } } - return page; } /** diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 87118ed84c..c4479756b4 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import { gzipSync } from 'node:zlib'; import type { AnyEventRequest } from '@workflow/world'; import { decode, encode } from 'cbor-x'; @@ -34,18 +35,6 @@ function decodePostedMeta(rawBody: unknown): Record { return decode(bytes.subarray(4, 4 + metaLen)) as Record; } -function joinFrames(...frames: Uint8Array[]): Uint8Array { - const joined = new Uint8Array( - frames.reduce((length, frame) => length + frame.byteLength, 0) - ); - let offset = 0; - for (const frame of frames) { - joined.set(frame, offset); - offset += frame.byteLength; - } - return joined; -} - /** * Legacy (spec-version-1) runs predate event sourcing: the runtime still * posts hook_received (resumeHook) and wait_completed (wakeUpRun) for them @@ -642,7 +631,7 @@ describe('createWorkflowRunEvent response coercion', () => { agent.assertNoPendingInterceptors(); }); - it('hydrates a streamed run_started page in either structural event order', async () => { + it('hydrates streamed run input without decompressing replay events', async () => { const agent = mockAgent(); const serializedInput = new TextEncoder().encode('"workflow input"'); const compressedInput = gzipSync(serializedInput); @@ -658,7 +647,7 @@ describe('createWorkflowRunEvent response coercion', () => { }) .reply( 200, - joinFrames( + Buffer.concat([ encodeFrame( { _result: 1, @@ -668,10 +657,10 @@ describe('createWorkflowRunEvent response coercion', () => { startedAt: new Date('2026-06-10T00:00:01.000Z'), }, event: { - eventId: 'evnt_1', + eventId: 'evnt_2', runId: 'wrun_1', eventType: 'run_started', - createdAt: new Date('2026-06-10T00:00:00.000Z'), + createdAt: new Date('2026-06-10T00:00:01.000Z'), }, }, new Uint8Array() @@ -680,55 +669,58 @@ describe('createWorkflowRunEvent response coercion', () => { { eventId: 'evnt_1', runId: 'wrun_1', - eventType: 'run_started', + eventType: 'run_created', createdAt: new Date('2026-06-10T00:00:00.000Z'), specVersion: 5, - eventData: {}, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'wf', + input: { _type: 'RemoteRef', value: 'dbrf:unused' }, + }, }, - new Uint8Array() + input ), encodeFrame( { eventId: 'evnt_2', runId: 'wrun_1', - eventType: 'run_created', + eventType: 'run_started', createdAt: new Date('2026-06-10T00:00:01.000Z'), specVersion: 5, - eventData: { - deploymentId: 'dpl_1', - workflowName: 'wf', - input: { _type: 'RemoteRef', value: 'dbrf:unused' }, - }, + eventData: {}, }, - input + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() ), - encodeFrame({ _end: 1, next: 'eid:evnt_2' }, new Uint8Array()) - ), + ]), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE, - 'x-wf-event-id': 'evnt_1', + 'x-wf-event-id': 'evnt_2', 'x-wf-run-id': 'wrun_1', - 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + 'x-wf-created-at': '2026-06-10T00:00:01.000Z', }, } ); const result = await createWorkflowRunEvent( 'wrun_1', - { eventType: 'run_started', specVersion: 5 } as AnyEventRequest, + { eventType: 'run_started', specVersion: 5 }, undefined, { token: 'test-token', dispatcher: agent } ); expect(result.events?.map((event) => event.eventType)).toEqual([ - 'run_started', 'run_created', + 'run_started', ]); - expect(result.events?.[1].eventData?.input).toEqual(input); + expect(result.events?.[0].eventData?.input).toEqual(input); expect(result.run?.input).toEqual(input); expect(result.cursor).toBe('eid:evnt_2'); - expect(result.hasMore).toBe(true); + expect(result.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 635fc1e2d4..c686fa0e80 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -753,54 +753,18 @@ async function createWorkflowRunEventInner( // matching the v3 path's stripEventAndLegacyRefs behavior. const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; const body = result.body; - const streamedEvents = - result.type === 'event-page' - ? result.page.events.map(({ event, body }) => - buildEventFromV4(event, body, 'replay') - ) - : undefined; - const events = streamedEvents - ? streamedEvents.map((event) => stripEventDataRefs(event, resolveData)) - : body.events - ? (body.events as Record[]).map(coerceEventDates) - : undefined; - const cursor = result.type === 'event-page' ? result.page.next : body.cursor; - const hasMore = - result.type === 'event-page' - ? (result.page.hasMore ?? Boolean(result.page.next)) - : body.hasMore; - let run = body.run - ? deserializeError(body.run as Record) - : undefined; - if (run && streamedEvents) { - const runCreated = streamedEvents.find( - (event) => event.eventType === 'run_created' - ); - if (!runCreated) { - throw new Error('v4 createEvent: run_started page missing run_created'); - } - const input = (runCreated.eventData as Record).input; - const { inputRef: _inputRef, ...runData } = run as WorkflowRun & { - inputRef?: unknown; - }; - run = { ...runData, input } as WorkflowRun; - } - return { + const baseResult = { event: body.event ? stripEventDataRefs( coerceEventDates(body.event as Record), resolveData ) : undefined, - run, step: body.step ? deserializeStep(body.step as Parameters[0]) : undefined, hook: body.hook as EventResult['hook'], wait: body.wait as EventResult['wait'], - events, - cursor, - hasMore, // Lazy step start: thread the server's "I created the step on this call" // signal through so the owned-inline runtime path can gate body execution // on it. Absent from older servers → undefined → safe default. @@ -810,4 +774,49 @@ async function createWorkflowRunEventInner( ? { maxEvents: body.maxEvents } : {}), }; + + switch (result.type) { + case 'event': + return { + ...baseResult, + run: body.run + ? deserializeError(body.run as Record) + : undefined, + events: body.events + ? (body.events as Record[]).map(coerceEventDates) + : undefined, + cursor: body.cursor, + hasMore: body.hasMore, + }; + case 'event-page': { + const replayEvents = result.page.events.map(({ event, body }) => + buildEventFromV4(event, body, 'replay') + ); + const runCreated = replayEvents[0]; + if (runCreated?.eventType !== 'run_created') { + throw new Error( + 'v4 createEvent: run_started page must begin with run_created' + ); + } + if (!body.run) { + throw new Error('v4 createEvent: run_started result missing run'); + } + const { inputRef: _inputRef, ...run } = deserializeError< + WorkflowRun & { inputRef?: unknown } + >(body.run as Record); + return { + ...baseResult, + run: { ...run, input: runCreated.eventData.input }, + events: replayEvents.map((event) => + stripEventDataRefs(event, resolveData) + ), + cursor: result.page.next, + hasMore: result.page.hasMore, + }; + } + default: { + const exhaustive: never = result; + throw new Error(`v4 createEvent: unknown result type ${exhaustive}`); + } + } }