From 6c29237f2f065114cf026ab814222f058495012b Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:42:30 -0400 Subject: [PATCH 1/9] =?UTF-8?q?=F0=9F=90=9B=20Restore=20the=20durable=20ev?= =?UTF-8?q?al=20journal=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture.md | 41 ++-- packages/core/src/eval-handler.ts | 4 +- packages/core/tests/ephemeral-service.test.ts | 204 ++++++++++++------ packages/durable-streams/effect.ts | 26 +-- .../durable-streams/tests/durable-run.test.ts | 87 ++++++-- specs/executable-mdx-spec.md | 29 ++- 6 files changed, 250 insertions(+), 141 deletions(-) diff --git a/architecture.md b/architecture.md index 87eb1a68..c15a2122 100644 --- a/architecture.md +++ b/architecture.md @@ -133,9 +133,9 @@ The `@executablemd/workflow` package owns `WorkflowRun`, `useWorkflow()`, `getWorkflowRun()` and the Git capability. It depends on `@executablemd/core`, `@executablemd/durable-streams` and `@executablemd/runtime`, whose contextual `exec()` and `cwd()` the Git provider invokes; core never imports workflow or -Git. The CLI lifecycle is `xmd workflow start` and `xmd workflow resume`; the -durable lookup resumption requires is the run storage below. Ordinary `xmd run` -remains unchanged. +Git. The future CLI lifecycle is `xmd workflow start` and `xmd workflow resume`; +there is no workflow CLI execution branch yet. The durable lookup that resume +will require is the run storage below. Ordinary `xmd run` remains unchanged. ## Workflow run storage @@ -310,9 +310,10 @@ member value can. The command selects the environment; the document describes the procedure. `xmd run` uses the caller's current environment and makes no restoration -promise. `xmd workflow` creates a workflow run with one implicit root Workspace. -The same declarative components use contextual capabilities in both modes; the -workflow host supplies the stronger durability and authority boundary. +promise. The specified future `xmd workflow` command creates a workflow run with +one implicit root Workspace. The same declarative components use contextual +capabilities in both modes; the workflow host supplies the stronger durability +and authority boundary. Workspace identity belongs to the workflow run. A document path locates an immutable definition but does not identify a previous run. A host-generated or @@ -391,11 +392,11 @@ external branches, pushes, pull requests or provider state. ## Agent authority and generated XMD -An Agent under `xmd workflow` is read-only. The host enforces that ceiling in -the permission bridge, the provider-native sandbox and the filesystem view; a -document cannot raise it. A provider that cannot enforce the boundary fails -before Prompt execution. `xmd run` keeps its caller-selected Agent permission -behavior. +An Agent under the specified future `xmd workflow` command is read-only. The +host enforces that ceiling in the permission bridge, the provider-native sandbox +and the filesystem view; a document cannot raise it. A provider that cannot +enforce the boundary fails before Prompt execution. `xmd run` keeps its +caller-selected Agent permission behavior. Native Agent processes inspect disposable read-only materializations of the current logical Workspace root. Those views have no write-back path. An Agent @@ -673,10 +674,18 @@ replaced by a live one. Service attachment and `ephemeral eval` execute again during partial replay so the current process and middleware chain are reconstructed. A completed document replay returns its recorded result without expanding the document and -therefore starts no service. Workflow execution installs a non-delegating -`API.Service` denial provider: a workflow cannot reach an inherited host -adapter, because a run-owned durable service requires stable identity and -reconciliation rather than an execution-owned live process. +therefore starts no service. Ordinary durable eval transforms a block and +validates its declared exports against the live overlay before constructing its +durable effect. A collision therefore has no eval `Yield`; compatible partial +replay remains aligned, while retained history containing a now-incompatible +successful eval is rejected by the existing replay guard or divergence path. + +#390 provides and tests the non-delegating `useWorkflowServiceDenial()` provider. +#366 will install it in the future `xmd workflow start` and `xmd workflow resume` +scopes. No workflow CLI execution branch exists yet. The provider prevents a +workflow from reaching an inherited host adapter, because a run-owned durable +service requires stable identity and reconciliation rather than an +execution-owned live process. ## State ownership @@ -735,7 +744,7 @@ Status is measured against main. | `API.Service` / `startService()` | creates an authenticated, supervised loopback service attachment through a provider-neutral operation | built on main | | `service=` | publishes the attachment's endpoint into the live binding overlay for its invocation | built on main | | `ephemeral eval` | reconstructs live middleware and bindings without a journal entry | built on main | -| workflow service denial | prevents workflow documents from inheriting an ordinary host service adapter | built on main | +| `useWorkflowServiceDenial()` | provides and tests a non-delegating workflow service denial provider; #366 will install it in future start and resume scopes | built on main; no workflow CLI execution branch exists yet | | `xmd workflow start` / `xmd workflow resume` | starts or resumes a workflow run from the CLI | defined in `specs/workflow-workspace-spec.md`, unbuilt; the lookup it resumes through is built | | implicit workflow Workspace | retains provider-neutral filesystem, repository and attachment state by run ID | defined in `specs/workflow-workspace-spec.md`, unbuilt (#218) | | Repository / Worktree / transactional Git effects | compose named checkouts and publish local mutations with their journal result | defined in `specs/workflow-workspace-spec.md`, unbuilt | diff --git a/packages/core/src/eval-handler.ts b/packages/core/src/eval-handler.ts index ede9e05f..bc34d5fd 100644 --- a/packages/core/src/eval-handler.ts +++ b/packages/core/src/eval-handler.ts @@ -146,6 +146,7 @@ export const evalFactory: ModifierFactory = (_params) => (_args, _next) => }; const transformed = transformBlock(ctx.content, ctx.blockId, Object.keys(evalEnv.values)); + validateDurableExports(transformed.exports, liveEnvironment(evalEnv)); const bindings = serializeExports(evalEnv.values, transformed.imports); const result = (yield createDurableOperation( @@ -209,9 +210,6 @@ export const evalFactory: ModifierFactory = (_params) => (_args, _next) => return { value: exports as unknown as Json } as Json; }, - { - validate: () => validateDurableExports(transformed.exports, liveEnvironment(evalEnv)), - }, )) as unknown as { value: Json }; if (result.value && typeof result.value === "object") { diff --git a/packages/core/tests/ephemeral-service.test.ts b/packages/core/tests/ephemeral-service.test.ts index ba40e846..cac94123 100644 --- a/packages/core/tests/ephemeral-service.test.ts +++ b/packages/core/tests/ephemeral-service.test.ts @@ -1,14 +1,13 @@ import { beforeAll, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { resource, scoped, type Operation } from "effection"; -import { - InMemoryStream, - parseDurableEvent, - serializeDurableEvent, -} from "@executablemd/durable-streams"; +import { DivergenceError, InMemoryStream } from "@executablemd/durable-streams"; import { API, SERVICE_HOSTNAME } from "@executablemd/runtime"; import type { ServiceEndpoint } from "@executablemd/runtime"; import { useStubFs } from "@executablemd/runtime/test"; +import { applyModifiers } from "../src/component-api.ts"; +import { registerComponents } from "../src/components/registration.ts"; +import type { ComponentRegistration } from "../src/components/registration.ts"; import { collect } from "../src/collect.ts"; import { execute } from "../src/execute.ts"; import { useTempFileCompiler } from "../src/temp-file-compiler.ts"; @@ -49,6 +48,54 @@ function* useServiceStub( ); } +function* useTrackedExec(executions: string[]): Operation { + yield* API.Process.around({ + // deno-lint-ignore require-yield + *exec([options]) { + const script = (options.command[2] ?? "").trim(); + executions.push(script); + return { + exitCode: 0, + stdout: script.startsWith("echo ") ? `${script.slice(5)}\n` : "", + stderr: "", + }; + }, + }); +} + +function changingProvider(attachService: boolean): ComponentRegistration { + return { + name: "ChangingProvider", + origin: "ephemeral-service.test", + props: { type: "object", properties: {}, additionalProperties: false }, + *fn() { + if (attachService) { + yield* applyModifiers([{ name: "service", params: "server" }, { name: "exec" }], { + language: "bash", + content: "handshake-compatible-server", + blockId: "changing-service", + componentName: "ChangingProvider", + }); + } + yield* applyModifiers([{ name: "eval" }], { + language: "js", + content: 'const server = "durable";', + blockId: "changing-eval", + componentName: "ChangingProvider", + }); + return ""; + }, + }; +} + +function withoutClose(stream: InMemoryStream): InMemoryStream { + const events = stream.snapshot(); + if (events.at(-1)?.type !== "close") { + throw new Error("expected completed history"); + } + return new InMemoryStream(events.slice(0, -1)); +} + describe("ephemeral eval and attached-service bindings", () => { beforeAll(() => useTempFileCompiler()); @@ -267,12 +314,14 @@ done } }); - it("rejects a durable eval export that collides with an existing live binding", function* () { + it("prints a live durable-export collision, skips its Yield, and continues durably", function* () { const lifecycle = { starts: 0, stops: 0 }; const stream = new InMemoryStream(); + const executions: string[] = []; yield* useServiceStub(Object.freeze({ hostname: SERVICE_HOSTNAME, port: 40_100 }), lifecycle); + yield* useTrackedExec(executions); yield* useStubFs({ - "doc.md": ` + "doc.md": ` \`\`\`bash service=server exec handshake-compatible-server @@ -283,20 +332,21 @@ const partial = "must-not-commit"; const server = "must-not-execute"; \`\`\` - +\`\`\`bash exec +echo after-collision +\`\`\` `, }); - let failure: unknown; - try { + const output = String( yield* scoped(function* () { - yield* collect(yield* execute({ path: "doc.md", stream })); - }); - } catch (error) { - failure = error; - } + return yield* collect(yield* execute({ path: "doc.md", stream })); + }), + ); - expect(String(failure)).toContain("collides with a live binding"); + expect(output).toContain("collides with a live binding"); + expect(output).toContain("after-collision"); + expect(executions).toEqual(["echo after-collision"]); expect( stream .snapshot() @@ -305,81 +355,103 @@ const server = "must-not-execute"; expect(lifecycle).toEqual({ starts: 1, stops: 1 }); }); - it("rejects a colliding durable export before partial replay restoration", function* () { + it("prints the same collision on valid partial replay and restores the later effect", function* () { const endpoint = Object.freeze({ hostname: SERVICE_HOSTNAME, port: 40_101 }); const lifecycle = { starts: 0, stops: 0 }; - const legacy = new InMemoryStream(); + const executions: string[] = []; + const live = new InMemoryStream(); yield* useServiceStub(endpoint, lifecycle); + yield* useTrackedExec(executions); yield* useStubFs({ - "doc.md": ` + "doc.md": ` -\`\`\`bash service=other exec +\`\`\`bash service=server exec handshake-compatible-server \`\`\` \`\`\`js eval -const server = "legacy-durable-value"; +const partial = "must-not-commit"; +const server = "must-not-execute"; \`\`\` -tail - - +\`\`\`bash exec +echo after-collision +\`\`\` `, }); - yield* scoped(function* () { - yield* collect(yield* execute({ path: "doc.md", stream: legacy })); - }); - - const events = legacy.snapshot().map((event) => { - const parsed = parseDurableEvent( - serializeDurableEvent(event).replaceAll("service=other", "service=server"), - ); - if (!parsed.ok) { - throw parsed.error; - } - return parsed.value; - }); - const evalYield = events.findIndex( - (event) => event.type === "yield" && event.description.type === "eval", + const liveOutput = String( + yield* scoped(function* () { + return yield* collect(yield* execute({ path: "doc.md", stream: live })); + }), ); - expect(evalYield).toBeGreaterThan(-1); - const partial = new InMemoryStream(events.slice(0, evalYield + 1)); - const beforeEvalEvents = partial - .snapshot() - .filter((event) => event.type === "yield" && event.description.type === "eval").length; - yield* useStubFs({ - "doc.md": ` + expect(liveOutput).toContain("collides with a live binding"); + expect(executions).toEqual(["echo after-collision"]); -\`\`\`bash service=server exec -handshake-compatible-server -\`\`\` + const partial = withoutClose(live); + const before = partial.snapshot(); + const replayOutput = String( + yield* scoped(function* () { + return yield* collect(yield* execute({ path: "doc.md", stream: partial })); + }), + ); -\`\`\`js eval -const server = "legacy-durable-value"; -\`\`\` + expect(replayOutput).toContain("collides with a live binding"); + expect(replayOutput).toContain("after-collision"); + expect(executions).toEqual(["echo after-collision"]); + expect( + partial + .snapshot() + .filter((event) => event.type === "yield" && event.description.type === "eval").length, + ).toBe(0); + expect(partial.snapshot().slice(0, before.length)).toEqual(before); + expect(lifecycle).toEqual({ starts: 2, stops: 2 }); + }); -tail + it("rejects incompatible retained component history through divergence", function* () { + const lifecycle = { starts: 0, stops: 0 }; + const executions: string[] = []; + const history = new InMemoryStream(); + yield* useServiceStub(Object.freeze({ hostname: SERVICE_HOSTNAME, port: 40_102 }), lifecycle); + yield* useTrackedExec(executions); + yield* useStubFs({ + "doc.md": ` - +\`\`\`bash exec +echo after-provider +\`\`\` `, }); + yield* registerComponents([changingProvider(false)]); + expect(String(yield* collect(yield* execute({ path: "doc.md", stream: history })))).toContain( + "after-provider", + ); + expect(executions).toEqual(["echo after-provider"]); + const retained = withoutClose(history); + const before = retained.snapshot(); + const replayExecutions: string[] = []; let failure: unknown; - try { - yield* scoped(function* () { - yield* collect(yield* execute({ path: "doc.md", stream: partial })); - }); - } catch (error) { - failure = error; - } + yield* scoped(function* () { + yield* useTrackedExec(replayExecutions); + yield* registerComponents([changingProvider(true)]); + try { + yield* collect(yield* execute({ path: "doc.md", stream: retained })); + } catch (error) { + failure = error; + } + }); - expect(String(failure)).toContain("collides with a live binding"); + expect(failure).toBeInstanceOf(DivergenceError); + expect(String(failure)).not.toContain("collides with a live binding"); + expect(replayExecutions).toEqual([]); + expect(retained.snapshot().slice(0, before.length)).toEqual(before); expect( - partial + retained .snapshot() - .filter((event) => event.type === "yield" && event.description.type === "eval").length, - ).toBe(beforeEvalEvents); - expect(lifecycle).toEqual({ starts: 2, stops: 2 }); + .slice(before.length) + .filter((event) => event.type === "yield"), + ).toHaveLength(0); + expect(lifecycle).toEqual({ starts: 1, stops: 1 }); }); it("rejects ephemeral exports that collide with durable names before execution", function* () { diff --git a/packages/durable-streams/effect.ts b/packages/durable-streams/effect.ts index 3883d830..2455251d 100644 --- a/packages/durable-streams/effect.ts +++ b/packages/durable-streams/effect.ts @@ -81,7 +81,6 @@ function checkReplay( resolve: Resolve>, routine: CoroutineView, ctx: DurableContext, - validate?: () => void, ): ReplayResult { const entry = ctx.replayIndex.peekYield(ctx.coroutineId); @@ -145,16 +144,6 @@ function checkReplay( // All guards approved — consume the entry and advance cursor ctx.replayIndex.consumeYield(ctx.coroutineId); - try { - validate?.(); - } catch (error) { - resolve({ - ok: false, - error: error instanceof Error ? error : new Error(String(error)), - }); - return { path: "replayed", teardown: (exit) => exit(VOID_OK) }; - } - // Feed stored result synchronously resolve(protocolToEffection(entry.result)); return { path: "replayed", teardown: (exit) => exit(VOID_OK) }; @@ -304,13 +293,10 @@ export function createDurableEffect( * * @param desc Structured description for the journal and divergence detection * @param execute Returns an Operation to run during live execution - * @param options.validate Runs before live execution or replay restoration; a - * thrown error is not persisted as this operation's result */ export function createDurableOperation( desc: EffectDescription, execute: () => Operation, - options: { validate?: () => void } = {}, ): DurableEffect { return { description: `${desc.type}(${desc.name})`, @@ -321,22 +307,12 @@ export function createDurableOperation( routine, ): (resolve: Resolve>) => void { const ctx = routine.scope.expect(DurableCtx); - const replay = checkReplay(desc, resolve, routine, ctx, options.validate); + const replay = checkReplay(desc, resolve, routine, ctx); if (replay.path === "replayed") { return replay.teardown; } // ── LIVE PATH ── - try { - options.validate?.(); - } catch (error) { - resolve({ - ok: false, - error: error instanceof Error ? error : new Error(String(error)), - }); - return (exit) => exit(VOID_OK); - } - // Run the entire execute → capture → persist → resolve sequence // as a structured operation in the routine's scope. routine.scope.run(function* () { diff --git a/packages/durable-streams/tests/durable-run.test.ts b/packages/durable-streams/tests/durable-run.test.ts index 9b3a3608..af876ea2 100644 --- a/packages/durable-streams/tests/durable-run.test.ts +++ b/packages/durable-streams/tests/durable-run.test.ts @@ -8,8 +8,10 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; +import { spawn, withResolvers, type Operation } from "effection"; import { type DurableEvent, + type DurableStream, InMemoryStream, type Json, type Workflow, @@ -17,6 +19,30 @@ import { durableRun, } from "../mod.ts"; +class PausedYieldStream implements DurableStream { + readonly appendStarted = withResolvers(); + readonly allowAppend = withResolvers(); + readonly inner = new InMemoryStream(); + + constructor(readonly order: string[]) {} + + *readAll(): Operation { + return yield* this.inner.readAll(); + } + + *append(event: DurableEvent): Operation { + if (event.type === "yield") { + this.order.push("append:started"); + this.appendStarted.resolve(); + yield* this.allowAppend.operation; + } + yield* this.inner.append(event); + if (event.type === "yield") { + this.order.push("append:completed"); + } + } +} + /** Track which functions were actually called during live execution. */ function createCallTracker() { const calls: string[] = []; @@ -212,45 +238,64 @@ describe("durableRun", () => { }); it("persist-before-resume: generator does not advance until write completes", function* () { - const stream = new InMemoryStream(); const order: string[] = []; - - // Hook into append to track ordering - stream.onAppend = (event) => { - if (event.type === "yield") { - order.push(`persist:${event.type}`); - } - }; + const stream = new PausedYieldStream(order); + let resumed = false; function* workflow(): Workflow { yield* durableCall("step1", () => { order.push("execute:step1"); return Promise.resolve("one" as const); }); + resumed = true; order.push("resumed:after-step1"); - - yield* durableCall("step2", () => { - order.push("execute:step2"); - return Promise.resolve("two" as const); - }); - order.push("resumed:after-step2"); - return "done"; } - yield* durableRun(workflow, { stream }); + const task = yield* spawn(() => durableRun(workflow, { stream })); + yield* stream.appendStarted.operation; + expect(resumed).toBe(false); + expect(stream.inner.snapshot()).toHaveLength(0); + + stream.allowAppend.resolve(); + expect(yield* task).toBe("done"); - // Verify ordering: execute → persist → resume for each step expect(order).toEqual([ "execute:step1", - "persist:yield", + "append:started", + "append:completed", "resumed:after-step1", - "execute:step2", - "persist:yield", - "resumed:after-step2", ]); }); + it("append failure cannot resolve a live effect successfully without a Yield", function* () { + const stream = new InMemoryStream(); + const appendFailure = new Error("yield append failed"); + stream.injectFailure = appendFailure; + let resumed = false; + let failure: unknown; + + function* workflow(): Workflow { + yield* durableCall("step", () => Promise.resolve("completed" as const)); + resumed = true; + return "unjournaled-success"; + } + + try { + yield* durableRun(workflow, { stream }); + } catch (error) { + failure = error; + } + + expect(resumed).toBe(false); + expect(stream.snapshot()).toHaveLength(0); + expect(failure).toBeInstanceOf(AggregateError); + if (!(failure instanceof AggregateError)) { + throw new Error("expected append failure aggregation"); + } + expect(failure.errors[0]).toBe(appendFailure); + }); + it("actor handoff: Process B resumes from Process A's events", function* () { // Process A: execute first 2 steps then "crash" (we just take the events) const streamA = new InMemoryStream(); diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index 2ab311b4..3989bda0 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -1273,8 +1273,15 @@ projected caller content. The two namespaces may not overlap. `service=` validates the binding name and checks both environments before attaching a process. Ordinary durable -eval validates its declared exports against live names before execution or -replay restoration and before appending an eval event. `ephemeral eval` +eval first transforms the block to discover its declared exports, then validates +them against live names before constructing its durable effect. An invalid eval +is document validation: it never becomes a `DurableEffect` and appends no eval +`Yield`. A valid partial replay performs the same validation before yielding an +effect, so a diagnostic from the original run leaves later recorded effects +aligned. If retained history instead contains a successful eval `Yield` that the +current definition no longer yields, the existing replay guard, divergence and +stale-input rules reject that incompatible history; validation never consumes +the recorded result or substitutes an unjournaled error. `ephemeral eval` validates its exports against durable names before execution; it may atomically replace an existing live binding. A failed block publishes none of its exports. Values from the live overlay are never substituted into a durable effect's @@ -1459,8 +1466,8 @@ processor. Instead: #### What is journaled -`evalFactory` wraps execution in `createDurableOperation`. Diagnostic journal -shape: +After transformation and live-binding collision validation, `evalFactory` wraps +execution in `createDurableOperation`. Diagnostic journal shape: ```json { "type": "eval", "name": "eval:root:0", "language": "js" } @@ -1516,7 +1523,7 @@ run but are absent from the diagnostic trace. | `packages/cli/src/service-host.ts` | shared XMD service handshake observer and supervised host-process adapter | | `packages/cli/src/{deno,node,bun,compiled}-service.ts` | runtime-named service adapters for token, environment and stdio behavior | | `packages/cli/src/{deno,node,bun,compiled}.ts` | Entrypoints — each installs matching `API.Env` and `API.Service` adapters, then calls `runXmd` | -| `packages/workflow/src/service-denial.ts` | non-delegating workflow service denial middleware | +| `packages/workflow/src/service-denial.ts` | `useWorkflowServiceDenial()`, the tested non-delegating provider for future workflow start and resume scopes (#366) | | `packages/cli/src/file-stream.ts` | `FileStream` — JSONL-backed `DurableStream` implementation | Dependencies: `@effectionx/scope-eval`, `@effectionx/timebox`, @@ -5260,7 +5267,8 @@ workflow and returns a `DocumentExecution` handle. Options: - `componentDirs?` — component search directories (default: `["components", "."]`) - `modifiers?` — custom modifier factories registered alongside the - built-ins (`exec`, `silent`, `eval`, `persist`, `timeout`, `daemon`) + built-ins (`exec`, `silent`, `eval`, `ephemeral`, `persist`, `timeout`, + `daemon`, `service`) - `secretDetection?` — detect credentials before durable events persist (default: enabled) @@ -6712,12 +6720,13 @@ Defined in [Workflow runs](./workflow-spec.md) §9.4 and §9.6–§9.7. | R2 | Live overlay hidden from interpolation | `{server}` remains literal rather than becoming an endpoint string | | R3 | `ephemeral eval` executes during partial replay | Live bindings and middleware are reconstructed without a journal entry | | R4 | Service publication collision | A durable or live binding with the requested name refuses attachment before spawn | -| R5 | Durable export collides with a service | Live execution and partial replay both reject before execution or restoration and append no eval event | +| R5 | Durable export collides with a service | Live execution and valid partial replay diagnose the block before constructing an eval effect; a later durable effect executes or restores in alignment, and no eval `Yield` appears | | R6 | Ephemeral export collides with durable state | The block is rejected before execution and publishes no partial export | | R7 | Ephemeral update of a live binding | A later ephemeral block may atomically replace an existing live name | -| R8 | `when` accessible in eval block | `yield* when(fn)` retries until fn succeeds | -| R9 | `when` retries on throw | Inner function throws twice, then succeeds → `when` resolves | -| R10 | `when` propagates timeout | Inner function never succeeds → `when` throws after limit | +| R8 | Incompatible retained eval history | Genuine history from an earlier component definition fails fatally through the existing durability guard before restoring its now-incompatible eval result or running a later effect; no replacement `Yield` is appended | +| R9 | `when` accessible in eval block | `yield* when(fn)` retries until fn succeeds | +| R10 | `when` retries on throw | Inner function throws twice, then succeeds → `when` resolves | +| R11 | `when` propagates timeout | Inner function never succeeds → `when` throws after limit | ### Tier S — Provider component pattern (integration) From 47339f21753339dccf1586366e949367f0d9c605 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:44:06 -0400 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=90=9B=20Reject=20terminal=20replay?= =?UTF-8?q?=20divergence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture.md | 5 + packages/core/src/errors.ts | 6 +- packages/core/tests/ephemeral-service.test.ts | 59 +++++++++++- packages/core/tests/fatal-cause.test.ts | 2 + packages/durable-streams/README.md | 7 +- packages/durable-streams/errors.ts | 38 ++++++-- packages/durable-streams/mod.ts | 1 + packages/durable-streams/run.ts | 64 +++++++++---- packages/durable-streams/specs/DECISIONS.md | 28 +++--- .../specs/effection-integration.md | 95 +++++++++++-------- .../specs/protocol-specification.md | 16 +++- .../durable-streams/tests/divergence.test.ts | 35 +++++++ specs/executable-mdx-spec.md | 32 ++++--- 13 files changed, 284 insertions(+), 104 deletions(-) diff --git a/architecture.md b/architecture.md index c15a2122..15a38f7d 100644 --- a/architecture.md +++ b/architecture.md @@ -679,6 +679,11 @@ validates its declared exports against the live overlay before constructing its durable effect. A collision therefore has no eval `Yield`; compatible partial replay remains aligned, while retained history containing a now-incompatible successful eval is rejected by the existing replay guard or divergence path. +The durable root checks for unconsumed replay entries before appending either a +successful or failed `Close`. If collision handling terminates immediately, +`TerminalDivergenceError` retains the collision as its cause and the history +receives no terminal event; restoring the compatible definition can still +replay it. #390 provides and tests the non-delegating `useWorkflowServiceDenial()` provider. #366 will install it in the future `xmd workflow start` and `xmd workflow resume` diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts index 658e5b8f..ea26a795 100644 --- a/packages/core/src/errors.ts +++ b/packages/core/src/errors.ts @@ -3,8 +3,8 @@ import type { Context, Operation } from "effection"; import { ContinuePastCloseDivergenceError, DivergenceError, - EarlyReturnDivergenceError, StaleInputError, + TerminalDivergenceError, } from "@executablemd/durable-streams"; import { InvocationTeardownError } from "./invocation.ts"; import type { ErrorSegment } from "./types.ts"; @@ -186,7 +186,7 @@ export class ContentError extends Error { export type DurabilityFailure = | StaleInputError | DivergenceError - | EarlyReturnDivergenceError + | TerminalDivergenceError | ContinuePastCloseDivergenceError; /** A failure that ends the execution rather than becoming a printed error. */ @@ -287,7 +287,7 @@ function asDurabilityFailure(error: unknown): DurabilityFailure | undefined { if ( error instanceof StaleInputError || error instanceof DivergenceError || - error instanceof EarlyReturnDivergenceError || + error instanceof TerminalDivergenceError || error instanceof ContinuePastCloseDivergenceError ) { return error; diff --git a/packages/core/tests/ephemeral-service.test.ts b/packages/core/tests/ephemeral-service.test.ts index cac94123..c45dce84 100644 --- a/packages/core/tests/ephemeral-service.test.ts +++ b/packages/core/tests/ephemeral-service.test.ts @@ -1,7 +1,12 @@ import { beforeAll, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { resource, scoped, type Operation } from "effection"; -import { DivergenceError, InMemoryStream } from "@executablemd/durable-streams"; +import { + DivergenceError, + InMemoryStream, + serializeDurableEvent, + TerminalDivergenceError, +} from "@executablemd/durable-streams"; import { API, SERVICE_HOSTNAME } from "@executablemd/runtime"; import type { ServiceEndpoint } from "@executablemd/runtime"; import { useStubFs } from "@executablemd/runtime/test"; @@ -10,6 +15,7 @@ import { registerComponents } from "../src/components/registration.ts"; import type { ComponentRegistration } from "../src/components/registration.ts"; import { collect } from "../src/collect.ts"; import { execute } from "../src/execute.ts"; +import { LiveBindingCollisionError } from "../src/live-env.ts"; import { useTempFileCompiler } from "../src/temp-file-compiler.ts"; const SAMPLE = `--- @@ -96,6 +102,10 @@ function withoutClose(stream: InMemoryStream): InMemoryStream { return new InMemoryStream(events.slice(0, -1)); } +function serialized(stream: InMemoryStream): string { + return stream.snapshot().map(serializeDurableEvent).join(""); +} + describe("ephemeral eval and attached-service bindings", () => { beforeAll(() => useTempFileCompiler()); @@ -429,6 +439,7 @@ echo after-provider const retained = withoutClose(history); const before = retained.snapshot(); + const beforeBytes = serialized(retained); const replayExecutions: string[] = []; let failure: unknown; yield* scoped(function* () { @@ -444,14 +455,52 @@ echo after-provider expect(failure).toBeInstanceOf(DivergenceError); expect(String(failure)).not.toContain("collides with a live binding"); expect(replayExecutions).toEqual([]); + expect(retained.snapshot()).toEqual(before); + expect(serialized(retained)).toBe(beforeBytes); + expect(lifecycle).toEqual({ starts: 1, stops: 1 }); + }); + + it("rejects incompatible retained history when collision validation terminates the root", function* () { + const lifecycle = { starts: 0, stops: 0 }; + const history = new InMemoryStream(); + yield* useServiceStub(Object.freeze({ hostname: SERVICE_HOSTNAME, port: 40_103 }), lifecycle); + yield* useStubFs({ + "doc.md": ``, + }); + yield* registerComponents([changingProvider(false)]); + yield* collect(yield* execute({ path: "doc.md", stream: history })); + + const retained = withoutClose(history); + const before = retained.snapshot(); + const beforeBytes = serialized(retained); + let failure: unknown; + yield* scoped(function* () { + yield* registerComponents([changingProvider(true)]); + try { + yield* collect(yield* execute({ path: "doc.md", stream: retained })); + } catch (error) { + failure = error; + } + }); + + expect(failure).toBeInstanceOf(TerminalDivergenceError); + if (!(failure instanceof TerminalDivergenceError)) { + throw new Error("expected terminal divergence"); + } + expect(failure.cause).toBeInstanceOf(LiveBindingCollisionError); + expect(String(failure)).not.toContain("collides with a live binding"); + expect(retained.snapshot()).toEqual(before); + expect(serialized(retained)).toBe(beforeBytes); + expect(lifecycle).toEqual({ starts: 1, stops: 1 }); + + yield* collect(yield* execute({ path: "doc.md", stream: retained })); expect(retained.snapshot().slice(0, before.length)).toEqual(before); expect( retained .snapshot() - .slice(before.length) - .filter((event) => event.type === "yield"), - ).toHaveLength(0); - expect(lifecycle).toEqual({ starts: 1, stops: 1 }); + .filter((event) => event.type === "yield" && event.description.type === "eval"), + ).toHaveLength(1); + expect(retained.snapshot().at(-1)?.type).toBe("close"); }); it("rejects ephemeral exports that collide with durable names before execution", function* () { diff --git a/packages/core/tests/fatal-cause.test.ts b/packages/core/tests/fatal-cause.test.ts index 42c7fb35..ad23a455 100644 --- a/packages/core/tests/fatal-cause.test.ts +++ b/packages/core/tests/fatal-cause.test.ts @@ -15,6 +15,7 @@ import { DivergenceError, EarlyReturnDivergenceError, StaleInputError, + TerminalDivergenceError, } from "@executablemd/durable-streams"; import { InvocationTeardownError } from "../src/invocation.ts"; import { ContentError, DocumentationError, durabilityFailure, fatalCause } from "../src/errors.ts"; @@ -72,6 +73,7 @@ const DURABILITY_FAILURES: Array<() => Error> = [ { type: "eval", name: "eval:reached" }, ), () => new EarlyReturnDivergenceError("root", 2, 5), + () => new TerminalDivergenceError("root", 2, 5, { cause: new Error("document failed") }), () => new ContinuePastCloseDivergenceError("root", 5), ]; diff --git a/packages/durable-streams/README.md b/packages/durable-streams/README.md index 3af52d14..c2fd94ff 100644 --- a/packages/durable-streams/README.md +++ b/packages/durable-streams/README.md @@ -55,7 +55,7 @@ interface Yield { } ``` -**`Close`** is written when a coroutine terminates — whether it completed, threw an error, or was cancelled. Close events are load-bearing: they tell the runtime on restart which coroutines finished cleanly and which need re-execution. +**`Close`** is written when a coroutine terminates — whether it completed, threw an error, or was cancelled — after replay has consumed every retained entry. Close events are load-bearing: they tell the runtime on restart which coroutines finished cleanly and which need re-execution. A termination that leaves replay entries unconsumed is divergence and appends no `Close`. ### What goes into the journal @@ -292,12 +292,13 @@ During replay, every yielded effect is validated against its journal entry. Only // → DivergenceError ``` -Two additional terminal conditions are checked: +Three additional terminal conditions are checked: - **Generator finishes early**: the code returns before consuming all journal entries — effects were removed. +- **Generator fails early**: the code throws before consuming all journal entries — the ordinary failure cannot close over retained effects the current execution never reached. - **Generator continues past close**: the journal shows the coroutine closed, but the code keeps yielding — effects were added. -Both indicate the code has changed in a way that makes the stored history invalid. The solution for intentional code changes is `versionCheck`: +All indicate the code has changed in a way that makes the stored history invalid. Early return raises `EarlyReturnDivergenceError`; early failure raises `TerminalDivergenceError` with the ordinary failure as its cause. Neither appends a terminal `Close`, so a compatible definition can still replay the retained history. The solution for intentional code changes is `versionCheck`: ```typescript function* orderWorkflow(orderId: string): Workflow { diff --git a/packages/durable-streams/errors.ts b/packages/durable-streams/errors.ts index 9f595c2e..8dcd6918 100644 --- a/packages/durable-streams/errors.ts +++ b/packages/durable-streams/errors.ts @@ -66,20 +66,28 @@ export class DivergenceError extends Error { } /** - * Raised when the generator finishes (returns) while the replay index - * still has unconsumed entries for this coroutine. See spec §6.3. + * Raised when a workflow terminates while replay still has unconsumed entries. + * The retained journal describes effects that the current execution did not + * reach, so no terminal Close may be appended over that history. */ -export class EarlyReturnDivergenceError extends Error { - override name = "EarlyReturnDivergenceError"; +export class TerminalDivergenceError extends Error { + override name = "TerminalDivergenceError"; coroutineId: CoroutineId; consumedCount: number; totalCount: number; - constructor(coroutineId: CoroutineId, consumedCount: number, totalCount: number) { + constructor( + coroutineId: CoroutineId, + consumedCount: number, + totalCount: number, + options: { cause?: unknown; message?: string } = {}, + ) { super( - `Divergence: generator ${coroutineId} returned after ${consumedCount} yields, ` + - `but journal has ${totalCount} yield entries`, + options.message ?? + `Divergence: workflow ${coroutineId} terminated after ${consumedCount} yields, ` + + `but journal has ${totalCount} yield entries`, + { cause: options.cause }, ); this.coroutineId = coroutineId; this.consumedCount = consumedCount; @@ -87,6 +95,22 @@ export class EarlyReturnDivergenceError extends Error { } } +/** + * Raised when the generator finishes (returns) while the replay index + * still has unconsumed entries for this coroutine. See spec §6.3. + */ +export class EarlyReturnDivergenceError extends TerminalDivergenceError { + override name = "EarlyReturnDivergenceError"; + + constructor(coroutineId: CoroutineId, consumedCount: number, totalCount: number) { + super(coroutineId, consumedCount, totalCount, { + message: + `Divergence: generator ${coroutineId} returned after ${consumedCount} yields, ` + + `but journal has ${totalCount} yield entries`, + }); + } +} + /** * Raised when the journal has a Close event for a coroutine but the * generator has not finished after consuming all recorded yields. diff --git a/packages/durable-streams/mod.ts b/packages/durable-streams/mod.ts index 49d3765a..740380e4 100644 --- a/packages/durable-streams/mod.ts +++ b/packages/durable-streams/mod.ts @@ -46,6 +46,7 @@ export { EarlyReturnDivergenceError, MalformedDurableEventError, StaleInputError, + TerminalDivergenceError, } from "./errors.ts"; // Divergence API — pluggable policy for replay mismatches (DEC-031) diff --git a/packages/durable-streams/run.ts b/packages/durable-streams/run.ts index c42f3f80..fcc4b486 100644 --- a/packages/durable-streams/run.ts +++ b/packages/durable-streams/run.ts @@ -16,13 +16,35 @@ import { useScope } from "effection"; import type { Operation, Scope } from "effection"; import { DurableCtx } from "./context.ts"; -import { EarlyReturnDivergenceError } from "./errors.ts"; +import { + ContinuePastCloseDivergenceError, + DivergenceError, + EarlyReturnDivergenceError, + StaleInputError, + TerminalDivergenceError, +} from "./errors.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { ReplayIndex } from "./replay-index.ts"; import { deserializeError, serializeError } from "./serialize.ts"; import type { DurableStream } from "./stream.ts"; import type { Close, DurableEvent, Json, Workflow, WorkflowValue } from "./types.ts"; +function unconsumedReplay(replayIndex: ReplayIndex, coroutineId: string) { + if (replayIndex.isReplayDisabled(coroutineId)) { + return undefined; + } + return replayIndex.firstUnconsumed(); +} + +function isDurabilityFailure(error: Error): boolean { + return ( + error instanceof StaleInputError || + error instanceof DivergenceError || + error instanceof TerminalDivergenceError || + error instanceof ContinuePastCloseDivergenceError + ); +} + /** * Run the ReplayGuard check phase over all Yield events. * @@ -59,7 +81,8 @@ export interface DurableRunOptions { * 3. Runs the workflow — replayed effects resolve synchronously from * the index; live effects execute and persist before resuming. * 4. On completion, appends a Close event to the stream. - * 5. On error, appends a Close(err) event. + * 5. On error, appends a Close(err) event unless replay entries remain + * unconsumed, in which case it rejects the incompatible history unchanged. * * Returns the workflow's result value. * @@ -117,20 +140,13 @@ export function* durableRun( // yield* accepts it directly — no cast needed. const result: T = yield* workflow(); - // §6.3: Check for early return divergence. - // If the generator returned but the replay index has unconsumed yields, - // the workflow has diverged. Skip this check when replay has been - // disabled (run-live mode) — the workflow intentionally diverged and - // the Divergence API already approved it. - if (!replayIndex.isReplayDisabled(coroutineId)) { - const unconsumed = replayIndex.firstUnconsumed(); - if (unconsumed) { - throw new EarlyReturnDivergenceError( - unconsumed.coroutineId, - unconsumed.cursor, - unconsumed.totalYields, - ); - } + const unconsumed = unconsumedReplay(replayIndex, coroutineId); + if (unconsumed) { + throw new EarlyReturnDivergenceError( + unconsumed.coroutineId, + unconsumed.cursor, + unconsumed.totalYields, + ); } const closeEvent: Close = { @@ -143,10 +159,20 @@ export function* durableRun( return result; } catch (error) { - // Normalize the error once — use the same Error object for both the - // Close event and the rethrow so that live runs and replayed Close - // events carry identical error shapes. const primary = error instanceof Error ? error : new Error(String(error)); + const unconsumed = unconsumedReplay(replayIndex, coroutineId); + if (unconsumed) { + if (isDurabilityFailure(primary)) { + throw primary; + } + throw new TerminalDivergenceError( + unconsumed.coroutineId, + unconsumed.cursor, + unconsumed.totalYields, + { cause: primary }, + ); + } + const closeEvent: Close = { type: "close", coroutineId, diff --git a/packages/durable-streams/specs/DECISIONS.md b/packages/durable-streams/specs/DECISIONS.md index b768787d..19da56e8 100644 --- a/packages/durable-streams/specs/DECISIONS.md +++ b/packages/durable-streams/specs/DECISIONS.md @@ -135,23 +135,27 @@ Updated before completion of every phase and committed at the end of each phase. shape for `enter()` that will be aligned with Effection's exact Effect interface in Phase 1. -## DEC-008: Three distinct divergence error types +## DEC-008: Distinct divergence error types - **Phase:** 0 (Scaffolding) - **Date:** 2026-02-28 -- **Context:** Spec §6.2-6.3 defines three divergence conditions: description - mismatch, generator finishes early, generator continues past close. +- **Context:** Spec §6.2-6.3 defines divergence at effect matching and at every + terminal boundary: return with retained effects, failure with retained + effects, and continuation past a recorded close. - **Options considered:** 1. Single DivergenceError class with a `kind` field - 2. Three separate error classes -- **Decision:** Three classes: `DivergenceError`, `EarlyReturnDivergenceError`, - `ContinuePastCloseDivergenceError`. All share `name = "DivergenceError"`. -- **Rationale:** Each carries different diagnostic fields (expected/actual - descriptions vs. consumed/total counts). Separate classes enable precise - `instanceof` checks in tests while sharing the same error name for catch-all - handling. -- **Consequences:** Error handling code can match on the common name - `"DivergenceError"` or use instanceof for specific cases. + 2. Separate error classes with a common terminal base +- **Decision:** `DivergenceError` describes effect mismatch, + `TerminalDivergenceError` describes termination with unconsumed replay, + `EarlyReturnDivergenceError` specializes terminal divergence for a normal + return, and `ContinuePastCloseDivergenceError` describes continuation after a + recorded close. +- **Rationale:** Effect mismatch carries expected/actual descriptions, while + terminal divergence carries consumed/total counts and may retain the active + execution failure as its cause. The common terminal base keeps the existing + early-return API compatible while covering exceptional termination honestly. +- **Consequences:** No root `Close` is appended while replay entries remain + unconsumed. Error handling uses `instanceof` rather than matching names. ## DEC-009: Workflow = Generator, T, unknown> diff --git a/packages/durable-streams/specs/effection-integration.md b/packages/durable-streams/specs/effection-integration.md index 932229a7..8c8c11e3 100644 --- a/packages/durable-streams/specs/effection-integration.md +++ b/packages/durable-streams/specs/effection-integration.md @@ -946,8 +946,8 @@ use prototypal inheritance). Each child scope has its own `coroutineId` and Implemented in `run.ts`. Key implementation details beyond the original design: short-circuits on existing Close event (DEC-016), -checks for early-return divergence after workflow completes, and -emits Close(err) on exceptions. +checks terminal replay alignment before either successful or exceptional +closure, and emits Close(err) only when no replay entries remain unconsumed. The entry point creates a scope, builds the replay index, sets up the durable context, and runs the workflow: @@ -978,36 +978,44 @@ async function durableRun( scope.set(DurableCtx, { replayIndex, stream, coroutineId, childCounter: 0 }); try { - // Workflow is structurally assignable to Operation — no cast needed - const task = scope.run(workflow); - const result = await task; - - // §6.3: Check for early return divergence - const cursor = replayIndex.getCursor(coroutineId); - const totalYields = replayIndex.yieldCount(coroutineId); - if (cursor < totalYields) { - throw new EarlyReturnDivergenceError(coroutineId, cursor, totalYields); - } - - await stream.append({ - type: "close", - coroutineId, - result: { status: "ok", value: result as Json }, - }); - return result; - } catch (error) { - await stream.append({ - type: "close", - coroutineId, - result: { status: "err", error: serializeError(error) }, - }); - throw error; - } finally { try { - await destroy(); - } catch { - /* swallow scope cleanup errors */ + const result = await scope.run(workflow); + + const unconsumed = replayIndex.firstUnconsumed(); + if (unconsumed) { + throw new EarlyReturnDivergenceError( + unconsumed.coroutineId, + unconsumed.cursor, + unconsumed.totalYields, + ); + } + + await stream.append({ + type: "close", + coroutineId, + result: { status: "ok", value: result as Json }, + }); + return result; + } catch (error) { + const unconsumed = replayIndex.firstUnconsumed(); + if (unconsumed) { + if (isDurabilityFailure(error)) throw error; + throw new TerminalDivergenceError( + unconsumed.coroutineId, + unconsumed.cursor, + unconsumed.totalYields, + { cause: error }, + ); + } + await stream.append({ + type: "close", + coroutineId, + result: { status: "err", error: serializeError(error) }, + }); + throw error; } + } finally { + try { await destroy(); } catch { /* preserve the workflow outcome */ } } } ``` @@ -1028,7 +1036,13 @@ Key details: - **Early return divergence check.** After the workflow returns, checks if the replay index has unconsumed yields. If so, the generator finished - before replaying all journal entries — the code has changed (§6.3). + before replaying all journal entries — the code has changed (§6.3). The + check is outside the ordinary failure-to-Close path, so it appends no Close. + +- **Exceptional terminal divergence check.** Before an ordinary error becomes + `Close(err)`, checks the same replay state. Unconsumed entries produce + `TerminalDivergenceError`, with the ordinary error as its cause, and leave the + retained stream unchanged. An already-active durability failure is preserved. - **Swallowing destroy errors.** If the workflow threw, `destroy()` may also throw "halted". The `try { await destroy() } catch {}` in finally @@ -1121,7 +1135,7 @@ Key validations: | Persist-before-resume strategy | ✅ Resolved | Strategy B — async append + deferred resolve (DEC-017) | | Serialization boundary | ✅ Resolved | `T extends Json` type constraint (DEC-018) | | DurableStream interface | ✅ Resolved | `readAll()` + `append()`, InMemoryStream for tests, HttpDurableStream for production | -| Terminal divergence detection | ✅ Resolved | Both cases implemented with 3 error classes (DEC-008) | +| Terminal divergence detection | ✅ Resolved | All terminal paths use the `TerminalDivergenceError` family (DEC-008) | | durableSpawn implementation | ✅ Resolved | Operations using Effection's native spawn/all/race | | HTTP backend adapter | ✅ Resolved | Raw fetch writes, promise chain serialization, epoch fencing (DEC-026–029) | | Batch persistence | ⏳ Deferred | Optimization for concurrent children, not blocking correctness. See §15.1 | @@ -1323,20 +1337,25 @@ with the buffer flushed at the end of each reduce cycle. ### 12.5 Terminal divergence detection (resolved) -Both cases from §6.3 are implemented and tested (DEC-008): +All cases from §6.3 are implemented and tested (DEC-008): 1. **Generator finishes early.** Detected in `durableRun()` after the workflow returns — if `cursor < totalYields`, throws - `EarlyReturnDivergenceError`. Tested in divergence test 9 and 13. + `EarlyReturnDivergenceError` without appending a `Close`. Tested in + divergence test 9 and 13. + +2. **Generator fails early.** Detected in `durableRun()` before an exceptional + termination is journaled. Unconsumed replay entries raise + `TerminalDivergenceError`, preserve the original failure as the cause and + append no `Close`. Tested in divergence test 13b. -2. **Generator continues past close.** Detected in +3. **Generator continues past close.** Detected in `createDurableEffect.enter()` — when `peekYield()` returns undefined but `hasClose()` returns true, throws `ContinuePastCloseDivergenceError`. Tested in divergence test 14. -Three distinct error classes share `name = "DivergenceError"` for -catch-all handling but carry different diagnostic fields for precise -`instanceof` checks. +The terminal errors carry consumed and total counts; exceptional termination +also carries the execution failure as its cause. ### 12.6 Durable `each()` — design and implementation plan diff --git a/packages/durable-streams/specs/protocol-specification.md b/packages/durable-streams/specs/protocol-specification.md index fe7f3f95..94d2817b 100644 --- a/packages/durable-streams/specs/protocol-specification.md +++ b/packages/durable-streams/specs/protocol-specification.md @@ -549,21 +549,28 @@ divergence cases into run-live behavior for controlled migrations. ### 6.3 Terminal divergence cases -Beyond per-effect matching, the reducer detects two additional -divergence conditions: +Beyond per-effect matching, the reducer detects terminal divergence before it +appends a root `Close`: **Generator finishes early.** The generator returns `{ done: true }` while the replay index still has unconsumed entries for this coroutine. This means the current code produces fewer effects than the recorded run — effects were removed without a version gate. +**Generator fails early.** The generator throws while the replay index still +has unconsumed entries. The ordinary execution error does not describe a +completed replay of the retained history. The runtime raises +`TerminalDivergenceError`, retaining the execution error as its cause, and +appends neither that ordinary error nor any root `Close`. + **Journal exhausted with close but generator continues.** The replay index has a `Close` event for this coroutine but the generator has not finished after consuming all recorded yields. This means the current code produces more effects than the recorded run — effects were added without a version gate. -Both cases raise `DivergenceError`. +All terminal divergence cases raise a durability error and leave the retained +journal unchanged. ### 6.4 What is NOT checked @@ -1043,7 +1050,8 @@ These tests MUST pass for the protocol to be considered implemented. | 10 | **Reordered steps** | Record with v1. Replay with v2 that swaps two effects. | `DivergenceError` at first swapped position. | | 11 | **Type mismatch** | Record a `call` effect. Replay code yields `sleep` at same position. | `DivergenceError` citing type mismatch. | | 12 | **Name mismatch** | Record `call("fetchOrder")`. Replay yields `call("chargeCard")`. | `DivergenceError` citing name mismatch. | -| 13 | **Generator finishes early** | Record stream with 5 yields + close. Replay code produces only 3 yields then returns. | `DivergenceError`: generator completed with unconsumed journal entries. | +| 13 | **Generator finishes early** | Record a partial stream with 5 yields and no root close. Replay code produces only 3 yields then returns. | `EarlyReturnDivergenceError`; no root `Close` is appended. | +| 13b | **Generator fails early** | Record a partial stream with a retained yield and no root close. Replay code throws before reaching it. | `TerminalDivergenceError`; the original error is its cause and no root `Close` is appended. | | 14 | **Generator continues past close** | Record stream with close after 3 yields. Replay code produces 5 yields. | `DivergenceError`: journal shows close but generator hasn't finished. | ### Tier 3 — Structured concurrency diff --git a/packages/durable-streams/tests/divergence.test.ts b/packages/durable-streams/tests/divergence.test.ts index fdff2c45..49d56588 100644 --- a/packages/durable-streams/tests/divergence.test.ts +++ b/packages/durable-streams/tests/divergence.test.ts @@ -266,6 +266,7 @@ describe("divergence detection", () => { ]; const stream = new InMemoryStream(events); + const before = stream.snapshot(); try { yield* durableRun( function* (): Workflow { @@ -284,6 +285,40 @@ describe("divergence detection", () => { expect(e.totalCount).toBe(3); } } + expect(stream.snapshot()).toEqual(before); + }); + + it("generator failure cannot close over unconsumed replay entries", function* () { + const original = new Error("current workflow failed"); + const events: DurableEvent[] = [ + { + type: "yield", + coroutineId: "root", + description: { type: "call", name: "removed" }, + result: { status: "ok", value: "recorded" }, + }, + ]; + const stream = new InMemoryStream(events); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + throw original; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) { + throw new Error("expected terminal divergence"); + } + expect(failure.name).toBe("TerminalDivergenceError"); + expect(failure.cause).toBe(original); + expect(stream.snapshot()).toEqual(events); }); it("continues past close — journal has Close but generator keeps yielding (completed workflow stays completed)", function* () { diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index 3989bda0..d0e60775 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -1281,7 +1281,12 @@ effect, so a diagnostic from the original run leaves later recorded effects aligned. If retained history instead contains a successful eval `Yield` that the current definition no longer yields, the existing replay guard, divergence and stale-input rules reject that incompatible history; validation never consumes -the recorded result or substitutes an unjournaled error. `ephemeral eval` +the recorded result or substitutes an unjournaled error. If collision handling +terminates the document before another durable effect is reached, the durable +root detects the unconsumed eval `Yield` before writing `Close(err)`, raises +`TerminalDivergenceError` with the collision as its cause, and leaves the +retained journal unchanged. A compatible definition can still replay it. +`ephemeral eval` validates its exports against durable names before execution; it may atomically replace an existing live binding. A failed block publishes none of its exports. Values from the live overlay are never substituted into a durable effect's @@ -4448,14 +4453,15 @@ than convert — including through a teardown aggregate, which must not launder it into an ordinary failure. Ordinary failures inside a `` are unaffected and remain printed errors. -**Divergence errors are rethrown on the same terms.** A `DivergenceError`, an -`EarlyReturnDivergenceError`, and a `ContinuePastCloseDivergenceError` all say -the journal no longer describes this run, so a generic catch that converted one -into a printed error would let expansion continue to the *next* durable operation — -whose own mismatch is then the failure reported, at a position that has nothing -to do with where the journal actually stopped describing the run. Every one of -these is discovered through the same cycle-safe cause traversal, so what the -caller receives is the failure itself rather than the wrapper it travelled in. +**Divergence errors are rethrown on the same terms.** A `DivergenceError`, a +`TerminalDivergenceError` or its `EarlyReturnDivergenceError` specialization, +and a `ContinuePastCloseDivergenceError` all say the journal no longer describes +this run, so a generic catch that converted one into a printed error would let +expansion continue to the *next* durable operation — whose own mismatch is then +the failure reported, at a position that has nothing to do with where the +journal actually stopped describing the run. Every one of these is discovered +through the same cycle-safe cause traversal, so what the caller receives is the +failure itself rather than the wrapper it travelled in. **Durability discovery traverses the whole cause graph.** No wrapper keeps a durability failure from being found, including a content failure a component @@ -6417,13 +6423,13 @@ visible warning blocks, gather into a separate error report). | FA6 | Both at once | A fatal error is still found when the wrapper holding it is itself cyclic | | FA7 | Documentation failures | A `DocumentationError` is discovered the same way | | FA8 | Ordinary errors are unaffected | A cyclic ordinary error is printed and the next block still runs | -| FA9 | Every durability failure | `StaleInputError`, `DivergenceError`, `EarlyReturnDivergenceError`, and `ContinuePastCloseDivergenceError` are each discovered as fatal, bare and wrapped | -| FA10 | Precedence, either order | Each of the four outranks a `DocumentationError` in an `AggregateError`, whichever comes first | +| FA9 | Every durability failure | `StaleInputError`, `DivergenceError`, `TerminalDivergenceError`, `EarlyReturnDivergenceError`, and `ContinuePastCloseDivergenceError` are each discovered as fatal, bare and wrapped | +| FA10 | Precedence, either order | Each of the five outranks a `DocumentationError` in an `AggregateError`, whichever comes first | | FA11 | Precedence through a teardown | The same holds for an `InvocationTeardownError`'s stage failures | | FA12 | Precedence at any depth | Nesting either one deeper than the other does not change the answer | | FA13 | No durability failure | A `DocumentationError` is reported when the graph holds none, and `durabilityFailure` finds nothing | | FA14 | Precedence with a cycle | A mixed graph that is also cyclic still reports the durability failure | -| FA15 | A content failure hides nothing fatal | A durability failure beneath a `ContentError` — set by a subclass and by assignment — is found by both `durabilityFailure` and `fatalCause`, for each of the four kinds | +| FA15 | A content failure hides nothing fatal | A durability failure beneath a `ContentError` — set by a subclass and by assignment — is found by both `durabilityFailure` and `fatalCause`, for each of the five kinds | | FA16 | Wherever the content failure sits | The same holds beneath an ordinary cause, inside an `AggregateError`, inside an `InvocationTeardownError`, and through all three at once | | FA17 | No resurrection | A `DocumentationError` a component recovered from is not reported as the outward failure, while the same one reached without crossing a content failure still is | | FA18 | Precedence behind a content failure | A durability failure beneath a recovered content failure outranks a documentation failure, in either wrapper order | @@ -6723,7 +6729,7 @@ Defined in [Workflow runs](./workflow-spec.md) §9.4 and §9.6–§9.7. | R5 | Durable export collides with a service | Live execution and valid partial replay diagnose the block before constructing an eval effect; a later durable effect executes or restores in alignment, and no eval `Yield` appears | | R6 | Ephemeral export collides with durable state | The block is rejected before execution and publishes no partial export | | R7 | Ephemeral update of a live binding | A later ephemeral block may atomically replace an existing live name | -| R8 | Incompatible retained eval history | Genuine history from an earlier component definition fails fatally through the existing durability guard before restoring its now-incompatible eval result or running a later effect; no replacement `Yield` is appended | +| R8 | Incompatible retained eval history | Genuine history from an earlier component definition fails fatally before restoring its now-incompatible eval result; effect-level mismatch and immediate root termination both append neither replacement `Yield` nor root `Close`, leave the retained prefix byte-for-byte unchanged, and permit the compatible definition to replay it | | R9 | `when` accessible in eval block | `yield* when(fn)` retries until fn succeeds | | R10 | `when` retries on throw | Inner function throws twice, then succeeds → `when` resolves | | R11 | `when` propagates timeout | Inner function never succeeds → `when` throws after limit | From ec3c14c7dd49d02bf51b97ee66d8904bbc4b0065 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:34:40 -0400 Subject: [PATCH 3/9] fix durable terminal journal alignment --- architecture.md | 15 +- packages/core/src/errors.ts | 3 + packages/core/tests/fatal-cause.test.ts | 2 + packages/durable-streams/README.md | 9 +- packages/durable-streams/combinators.ts | 76 ++++- packages/durable-streams/context.ts | 2 + packages/durable-streams/durability.ts | 77 +++++ packages/durable-streams/effect.ts | 8 +- packages/durable-streams/errors.ts | 15 + packages/durable-streams/guard.ts | 45 ++- packages/durable-streams/mod.ts | 1 + packages/durable-streams/replay-index.ts | 45 ++- packages/durable-streams/run.ts | 67 ++-- packages/durable-streams/specs/DECISIONS.md | 17 +- .../specs/effection-integration.md | 209 +++++-------- .../specs/protocol-specification.md | 24 +- .../durable-streams/tests/durable-run.test.ts | 123 +++++++- .../tests/guard-stream.test.ts | 39 ++- .../tests/replay-guard.test.ts | 11 +- .../tests/terminal-boundary.test.ts | 293 ++++++++++++++++++ specs/executable-mdx-spec.md | 20 +- 21 files changed, 872 insertions(+), 229 deletions(-) create mode 100644 packages/durable-streams/durability.ts create mode 100644 packages/durable-streams/tests/terminal-boundary.test.ts diff --git a/architecture.md b/architecture.md index 15a38f7d..b0bef8aa 100644 --- a/architecture.md +++ b/architecture.md @@ -626,7 +626,12 @@ through middleware; waiting itself is never raised. ### 8. Durability failures are outside the model A durability failure (§6.11) says the journal no longer describes the document -execution. No middleware sees it; it is never the document's own outcome. +execution. No middleware sees it; it is never the document's own outcome. A +backing-journal append failure is the same kind of boundary failure: it +preserves the adapter error as its cause, resumes no consumer with an +unpersisted success, and never triggers a compensating `Close`. A +pre-persistence policy rejection remains the policy's ordinary document +failure; the guarded stream marks that boundary before the backing append. ## Attempts @@ -679,8 +684,12 @@ validates its declared exports against the live overlay before constructing its durable effect. A collision therefore has no eval `Yield`; compatible partial replay remains aligned, while retained history containing a now-incompatible successful eval is rejected by the existing replay guard or divergence path. -The durable root checks for unconsumed replay entries before appending either a -successful or failed `Close`. If collision handling terminates immediately, +Every root and child termination checks its retained subtree before appending a +`Close`. The replay index tracks which coroutine identities the current +definition claims, so a completed retained child is not mistaken for aligned +history merely because it already has a `Close`. Durability failures take +precedence over ordinary termination even after the last retained `Yield` was +consumed. If collision handling terminates immediately, `TerminalDivergenceError` retains the collision as its cause and the history receives no terminal event; restoring the compatible definition can still replay it. diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts index ea26a795..aa9f508b 100644 --- a/packages/core/src/errors.ts +++ b/packages/core/src/errors.ts @@ -3,6 +3,7 @@ import type { Context, Operation } from "effection"; import { ContinuePastCloseDivergenceError, DivergenceError, + DurablePersistenceError, StaleInputError, TerminalDivergenceError, } from "@executablemd/durable-streams"; @@ -186,6 +187,7 @@ export class ContentError extends Error { export type DurabilityFailure = | StaleInputError | DivergenceError + | DurablePersistenceError | TerminalDivergenceError | ContinuePastCloseDivergenceError; @@ -287,6 +289,7 @@ function asDurabilityFailure(error: unknown): DurabilityFailure | undefined { if ( error instanceof StaleInputError || error instanceof DivergenceError || + error instanceof DurablePersistenceError || error instanceof TerminalDivergenceError || error instanceof ContinuePastCloseDivergenceError ) { diff --git a/packages/core/tests/fatal-cause.test.ts b/packages/core/tests/fatal-cause.test.ts index ad23a455..ba375fa8 100644 --- a/packages/core/tests/fatal-cause.test.ts +++ b/packages/core/tests/fatal-cause.test.ts @@ -13,6 +13,7 @@ import { scoped } from "effection"; import { ContinuePastCloseDivergenceError, DivergenceError, + DurablePersistenceError, EarlyReturnDivergenceError, StaleInputError, TerminalDivergenceError, @@ -75,6 +76,7 @@ const DURABILITY_FAILURES: Array<() => Error> = [ () => new EarlyReturnDivergenceError("root", 2, 5), () => new TerminalDivergenceError("root", 2, 5, { cause: new Error("document failed") }), () => new ContinuePastCloseDivergenceError("root", 5), + () => new DurablePersistenceError("yield", new Error("journal unavailable")), ]; describe("Tier FA — Fatal error discovery", () => { diff --git a/packages/durable-streams/README.md b/packages/durable-streams/README.md index c2fd94ff..61d7dc33 100644 --- a/packages/durable-streams/README.md +++ b/packages/durable-streams/README.md @@ -55,7 +55,7 @@ interface Yield { } ``` -**`Close`** is written when a coroutine terminates — whether it completed, threw an error, or was cancelled — after replay has consumed every retained entry. Close events are load-bearing: they tell the runtime on restart which coroutines finished cleanly and which need re-execution. A termination that leaves replay entries unconsumed is divergence and appends no `Close`. +**`Close`** is written when a coroutine terminates — whether it completed, threw an error, or was cancelled — after its retained subtree is aligned. Alignment requires every retained yield to be consumed and every retained child coroutine to be claimed by the current definition. Close events are load-bearing: they tell the runtime on restart which coroutines finished cleanly and which need re-execution. An unaligned termination is divergence and appends no `Close`. ### What goes into the journal @@ -278,6 +278,12 @@ The transition from replay to live happens **per-coroutine**, not globally. In a This is the protocol's most critical invariant: **the `Yield` event must be durably written to the stream before `iterator.next()` is called**. If the process crashes between an effect resolving and the journal write completing, the effect will be re-executed on the next run — which is safe, because the generator hasn't advanced past that point yet. +If the backing-stream write fails, the run raises `DurablePersistenceError` +with the adapter error as its cause. It does not resume the effect successfully +or write a compensating `Close(err)`. The same rule applies when a terminal +`Close` cannot be persisted. A `guardDurableStream` gate rejection remains the +gate's ordinary failure and may produce a separately admitted `Close(err)`. + Violating this invariant (advancing the generator before the write) creates an unrecoverable gap: the journal would be missing an entry, and replay would feed the wrong result to a subsequent effect. --- @@ -297,6 +303,7 @@ Three additional terminal conditions are checked: - **Generator finishes early**: the code returns before consuming all journal entries — effects were removed. - **Generator fails early**: the code throws before consuming all journal entries — the ordinary failure cannot close over retained effects the current execution never reached. - **Generator continues past close**: the journal shows the coroutine closed, but the code keeps yielding — effects were added. +- **Completed child is abandoned**: retained child history has a `Close`, but the current definition never claims that coroutine identity. All indicate the code has changed in a way that makes the stored history invalid. Early return raises `EarlyReturnDivergenceError`; early failure raises `TerminalDivergenceError` with the ordinary failure as its cause. Neither appends a terminal `Close`, so a compatible definition can still replay the retained history. The solution for intentional code changes is `versionCheck`: diff --git a/packages/durable-streams/combinators.ts b/packages/durable-streams/combinators.ts index f28a364b..30876ca3 100644 --- a/packages/durable-streams/combinators.ts +++ b/packages/durable-streams/combinators.ts @@ -28,7 +28,13 @@ import { } from "effection"; import type { Operation, Task } from "effection"; import { type DurableContext, DurableCtx } from "./context.ts"; +import { + activeDurabilityFailure, + appendDurableEvent, + rememberDurabilityFailure, +} from "./durability.ts"; import { ephemeral } from "./ephemeral.ts"; +import { EarlyReturnDivergenceError, TerminalDivergenceError } from "./errors.ts"; import { deserializeError, serializeError } from "./serialize.ts"; import type { Close, Json, Workflow, WorkflowValue } from "./types.ts"; @@ -53,6 +59,7 @@ function* runDurableChild( parentCtx: DurableContext, ): Operation { const { replayIndex, stream } = parentCtx; + replayIndex.claim(childId); // Short-circuit: child already completed in a previous run. // NOTE: Replay guard validation is not bypassed here — the check phase @@ -87,19 +94,40 @@ function* runDurableChild( // Set child's DurableContext on this scope const scope = yield* useScope(); - scope.set(DurableCtx, { + parentCtx.durability ??= {}; + const childCtx: DurableContext = { replayIndex, stream, coroutineId: childId, childCounter: 0, - }); + durability: parentCtx.durability, + }; + scope.set(DurableCtx, childCtx); let closeEvent: Close | undefined; + let suppressClose = false; yield* ensure(function* () { + if (suppressClose || activeDurabilityFailure(childCtx)) { + return; + } + // closeEvent still undefined means the child was cancelled before the // normal-return or catch path ran. if (!closeEvent) { + const unaligned = replayIndex.firstUnaligned(childId); + if (unaligned) { + const failure = new TerminalDivergenceError( + unaligned.coroutineId, + unaligned.cursor, + unaligned.totalYields, + { + message: `Divergence: coroutine ${childId} was cancelled before retained history was exhausted`, + }, + ); + rememberDurabilityFailure(childCtx, failure); + throw failure; + } closeEvent = { type: "close", coroutineId: childId, @@ -110,7 +138,7 @@ function* runDurableChild( // Don't re-emit a Close event if one already exists in the journal // (e.g., a cancelled child being replayed via suspend()). if (!replayIndex.hasClose(childId)) { - yield* stream.append(closeEvent!); + yield* appendDurableEvent(childCtx, closeEvent); } }); @@ -119,6 +147,24 @@ function* runDurableChild( // DurableCtx from the scope, so they'll use childId. const result: T = yield* childWorkflow(); + const durabilityFailure = activeDurabilityFailure(childCtx); + if (durabilityFailure) { + suppressClose = true; + throw durabilityFailure; + } + + const unaligned = replayIndex.firstUnaligned(childId); + if (unaligned) { + suppressClose = true; + const failure = new EarlyReturnDivergenceError( + unaligned.coroutineId, + unaligned.cursor, + unaligned.totalYields, + ); + rememberDurabilityFailure(childCtx, failure); + throw failure; + } + closeEvent = { type: "close", coroutineId: childId, @@ -127,16 +173,36 @@ function* runDurableChild( return result; } catch (error) { + const primary = error instanceof Error ? error : new Error(String(error)); + const durabilityFailure = activeDurabilityFailure(childCtx, primary); + if (durabilityFailure) { + suppressClose = true; + throw durabilityFailure; + } + + const unaligned = replayIndex.firstUnaligned(childId); + if (unaligned) { + suppressClose = true; + const failure = new TerminalDivergenceError( + unaligned.coroutineId, + unaligned.cursor, + unaligned.totalYields, + { cause: primary }, + ); + rememberDurabilityFailure(childCtx, failure); + throw failure; + } + closeEvent = { type: "close", coroutineId: childId, result: { status: "err", - error: serializeError(error instanceof Error ? error : new Error(String(error))), + error: serializeError(primary), }, }; - throw error; + throw primary; } } diff --git a/packages/durable-streams/context.ts b/packages/durable-streams/context.ts index 9b2d96a4..6c210fe4 100644 --- a/packages/durable-streams/context.ts +++ b/packages/durable-streams/context.ts @@ -20,6 +20,8 @@ export interface DurableContext { coroutineId: CoroutineId; /** Counter for assigning child IDs. */ childCounter: number; + /** Protocol failure shared by the root and every durable child. */ + durability?: { failure?: Error }; } /** diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts new file mode 100644 index 00000000..1332b796 --- /dev/null +++ b/packages/durable-streams/durability.ts @@ -0,0 +1,77 @@ +import type { Operation } from "effection"; +import type { DurableContext } from "./context.ts"; +import { + ContinuePastCloseDivergenceError, + DivergenceError, + DurablePersistenceError, + StaleInputError, + TerminalDivergenceError, +} from "./errors.ts"; +import { isDurableEventRejection, unwrapDurableEventRejection } from "./guard.ts"; +import type { DurableEvent } from "./types.ts"; + +export function findDurabilityFailure(error: unknown): Error | undefined { + const visited = new Set(); + const pending: unknown[] = [error]; + + while (pending.length > 0) { + const current = pending.shift(); + if (visited.has(current)) { + continue; + } + visited.add(current); + + if ( + current instanceof DurablePersistenceError || + current instanceof StaleInputError || + current instanceof DivergenceError || + current instanceof TerminalDivergenceError || + current instanceof ContinuePastCloseDivergenceError + ) { + return current; + } + if (current instanceof AggregateError) { + pending.push(...current.errors); + } + if (current instanceof Error && current.cause !== undefined) { + pending.push(current.cause); + } + } + + return undefined; +} + +export function rememberDurabilityFailure(ctx: DurableContext, error: Error): Error { + ctx.durability ??= {}; + ctx.durability.failure ??= error; + return ctx.durability.failure; +} + +export function activeDurabilityFailure(ctx: DurableContext, error?: unknown): Error | undefined { + if (ctx.durability?.failure) { + return ctx.durability.failure; + } + const failure = findDurabilityFailure(error); + if (failure) { + return rememberDurabilityFailure(ctx, failure); + } + return undefined; +} + +export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): Operation { + try { + yield* ctx.stream.append(event); + } catch (error) { + if (isDurableEventRejection(error)) { + const rejection = unwrapDurableEventRejection(error); + const failure = findDurabilityFailure(rejection); + if (failure) { + rememberDurabilityFailure(ctx, failure); + } + throw rejection; + } + const failure = new DurablePersistenceError(event.type, error); + rememberDurabilityFailure(ctx, failure); + throw failure; + } +} diff --git a/packages/durable-streams/effect.ts b/packages/durable-streams/effect.ts index 2455251d..f120ccf0 100644 --- a/packages/durable-streams/effect.ts +++ b/packages/durable-streams/effect.ts @@ -23,6 +23,7 @@ import type { Operation } from "effection"; import { type DurableContext, DurableCtx } from "./context.ts"; import { Divergence } from "./divergence.ts"; +import { appendDurableEvent, rememberDurabilityFailure } from "./durability.ts"; import { StaleInputError } from "./errors.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { protocolToEffection, serializeError } from "./serialize.ts"; @@ -110,6 +111,7 @@ function checkReplay( ]); if (decision.type === "throw") { + rememberDurabilityFailure(ctx, decision.error); resolve({ ok: false, error: decision.error }); return { path: "replayed", teardown: (exit) => exit(VOID_OK) }; } @@ -137,6 +139,7 @@ function checkReplay( coroutineId: ctx.coroutineId, description: desc, }); + rememberDurabilityFailure(ctx, error); resolve({ ok: false, error }); return { path: "replayed", teardown: (exit) => exit(VOID_OK) }; } @@ -161,6 +164,7 @@ function checkReplay( ]); if (decision.type === "throw") { + rememberDurabilityFailure(ctx, decision.error); resolve({ ok: false, error: decision.error }); return { path: "replayed", teardown: (exit) => exit(VOID_OK) }; } @@ -227,7 +231,7 @@ export function createDurableEffect( // down, the append is cancelled. routine.scope.run(function* () { try { - yield* ctx.stream.append(event); + yield* appendDurableEvent(ctx, event); resolve(protocolToEffection(result)); } catch (err) { resolve({ @@ -333,7 +337,7 @@ export function createDurableOperation( }; try { - yield* ctx.stream.append(event); + yield* appendDurableEvent(ctx, event); resolve(protocolToEffection(result)); } catch (err) { resolve({ diff --git a/packages/durable-streams/errors.ts b/packages/durable-streams/errors.ts index 8dcd6918..129f6ce5 100644 --- a/packages/durable-streams/errors.ts +++ b/packages/durable-streams/errors.ts @@ -4,6 +4,21 @@ import type { CoroutineId, EffectDescription } from "./types.ts"; +/** + * Raised when a durable event cannot be persisted. + * + * Persistence failures are protocol failures, not workflow outcomes. The + * adapter error remains available as the cause, and no compensating Close is + * written over the unpersisted event. + */ +export class DurablePersistenceError extends Error { + override name = "DurablePersistenceError"; + + constructor(eventType: "yield" | "close", cause: unknown) { + super(`Failed to persist durable ${eventType} event`, { cause }); + } +} + /** * Raised when a persisted record does not describe a `DurableEvent`. * diff --git a/packages/durable-streams/guard.ts b/packages/durable-streams/guard.ts index 3fec4c92..8160f442 100644 --- a/packages/durable-streams/guard.ts +++ b/packages/durable-streams/guard.ts @@ -18,6 +18,45 @@ import type { Operation } from "effection"; import type { DurableStream } from "./stream.ts"; import type { DurableEvent } from "./types.ts"; +const EVENT_REJECTION = Symbol.for("@effectionx/durable-streams/event-rejection"); + +class WrappedDurableEventRejection extends Error { + constructor(readonly rejection: unknown) { + super(rejection instanceof Error ? rejection.message : String(rejection), { + cause: rejection, + }); + } +} + +function markEventRejection(error: unknown): Error { + const rejection = error instanceof Error ? error : new Error(String(error)); + if ( + Reflect.defineProperty(rejection, EVENT_REJECTION, { + value: true, + configurable: false, + enumerable: false, + writable: false, + }) + ) { + return rejection; + } + return new WrappedDurableEventRejection(error); +} + +export function isDurableEventRejection(error: unknown): boolean { + if (error instanceof WrappedDurableEventRejection) { + return true; + } + if ((typeof error !== "object" || error === null) && typeof error !== "function") { + return false; + } + return Reflect.get(error, EVENT_REJECTION) === true; +} + +export function unwrapDurableEventRejection(error: unknown): unknown { + return error instanceof WrappedDurableEventRejection ? error.rejection : error; +} + /** * A check that runs before a durable event is persisted. * @@ -50,7 +89,11 @@ export function guardDurableStream(stream: DurableStream, gate: DurableEventGate // The gate sees a copy so "inspect or reject" is enforced rather than // merely documented: the backend always receives the event the effect // produced, whatever the gate did to the one it was handed. - yield* gate(structuredClone(event)); + try { + yield* gate(structuredClone(event)); + } catch (error) { + throw markEventRejection(error); + } yield* stream.append(event); }, }; diff --git a/packages/durable-streams/mod.ts b/packages/durable-streams/mod.ts index 740380e4..5cbf605e 100644 --- a/packages/durable-streams/mod.ts +++ b/packages/durable-streams/mod.ts @@ -43,6 +43,7 @@ export type { HttpDurableStreamHandle, HttpDurableStreamOptions } from "./http-s export { ContinuePastCloseDivergenceError, DivergenceError, + DurablePersistenceError, EarlyReturnDivergenceError, MalformedDurableEventError, StaleInputError, diff --git a/packages/durable-streams/replay-index.ts b/packages/durable-streams/replay-index.ts index 521f96f1..0d8e0e95 100644 --- a/packages/durable-streams/replay-index.ts +++ b/packages/durable-streams/replay-index.ts @@ -18,6 +18,8 @@ export class ReplayIndex { private closes = new Map(); /** Coroutines where replay has been disabled (run-live mode). */ private disabled = new Set(); + /** Retained coroutine identities reached by the current definition. */ + private claimed = new Set(); constructor(events: DurableEvent[]) { for (const event of events) { @@ -54,6 +56,21 @@ export class ReplayIndex { return this.disabled.has(coroutineId); } + /** Mark a retained coroutine identity as reached by the current run. */ + claim(coroutineId: CoroutineId): void { + this.claimed.add(coroutineId); + if (!this.closes.has(coroutineId)) { + return; + } + + const retainedIds = new Set([...this.yields.keys(), ...this.closes.keys()]); + for (const retainedId of retainedIds) { + if (retainedId.startsWith(`${coroutineId}.`)) { + this.claimed.add(retainedId); + } + } + } + /** * Returns the next unconsumed yield for this coroutine, * or undefined if the cursor is past the end or replay is disabled. @@ -113,30 +130,34 @@ export class ReplayIndex { return false; } - /** - * Return the first non-disabled coroutine with unconsumed yields. - * - * NOTE: Closed coroutines are skipped because their yields were consumed - * by the child's own replay path (via runDurableChild). This means - * orphaned children (recorded in the journal but never spawned in the - * current run) are not detected here. Orphan detection requires tracking - * which coroutine IDs were visited during the current run, which is a - * future enhancement. - */ - firstUnconsumed(): + /** Return the first retained coroutine not aligned with the current subtree. */ + firstUnaligned(subtreeId: CoroutineId): | { coroutineId: CoroutineId; cursor: number; totalYields: number; } | undefined { - for (const [coroutineId, entries] of this.yields.entries()) { + if (this.disabled.has(subtreeId)) { + return undefined; + } + + const coroutineIds = new Set([...this.yields.keys(), ...this.closes.keys()]); + for (const coroutineId of coroutineIds) { + if (coroutineId !== subtreeId && !coroutineId.startsWith(`${subtreeId}.`)) { + continue; + } if (this.disabled.has(coroutineId)) { continue; } if (this.closes.has(coroutineId)) { + if (!this.claimed.has(coroutineId)) { + const entries = this.yields.get(coroutineId) ?? []; + return { coroutineId, cursor: 0, totalYields: entries.length }; + } continue; } + const entries = this.yields.get(coroutineId) ?? []; const cursor = this.cursors.get(coroutineId) ?? 0; if (cursor < entries.length) { return { coroutineId, cursor, totalYields: entries.length }; diff --git a/packages/durable-streams/run.ts b/packages/durable-streams/run.ts index fcc4b486..edc3ff59 100644 --- a/packages/durable-streams/run.ts +++ b/packages/durable-streams/run.ts @@ -16,33 +16,16 @@ import { useScope } from "effection"; import type { Operation, Scope } from "effection"; import { DurableCtx } from "./context.ts"; -import { - ContinuePastCloseDivergenceError, - DivergenceError, - EarlyReturnDivergenceError, - StaleInputError, - TerminalDivergenceError, -} from "./errors.ts"; +import { activeDurabilityFailure, appendDurableEvent } from "./durability.ts"; +import { EarlyReturnDivergenceError, TerminalDivergenceError } from "./errors.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { ReplayIndex } from "./replay-index.ts"; import { deserializeError, serializeError } from "./serialize.ts"; import type { DurableStream } from "./stream.ts"; import type { Close, DurableEvent, Json, Workflow, WorkflowValue } from "./types.ts"; -function unconsumedReplay(replayIndex: ReplayIndex, coroutineId: string) { - if (replayIndex.isReplayDisabled(coroutineId)) { - return undefined; - } - return replayIndex.firstUnconsumed(); -} - -function isDurabilityFailure(error: Error): boolean { - return ( - error instanceof StaleInputError || - error instanceof DivergenceError || - error instanceof TerminalDivergenceError || - error instanceof ContinuePastCloseDivergenceError - ); +function unalignedReplay(replayIndex: ReplayIndex, coroutineId: string) { + return replayIndex.firstUnaligned(coroutineId); } /** @@ -81,8 +64,8 @@ export interface DurableRunOptions { * 3. Runs the workflow — replayed effects resolve synchronously from * the index; live effects execute and persist before resuming. * 4. On completion, appends a Close event to the stream. - * 5. On error, appends a Close(err) event unless replay entries remain - * unconsumed, in which case it rejects the incompatible history unchanged. + * 5. Before any Close, rejects durability failures and retained coroutine + * history the current definition did not align with. * * Returns the workflow's result value. * @@ -107,12 +90,14 @@ export function* durableRun( // is already installed by the caller before yield*-ing into durableRun. const scope = yield* useScope(); - scope.set(DurableCtx, { + const ctx = { replayIndex, stream, coroutineId, childCounter: 0, - }); + durability: {}, + }; + scope.set(DurableCtx, ctx); // ── REPLAY GUARD: Check phase ── // Run before the workflow starts. Middleware can yield* for I/O (hash @@ -134,13 +119,19 @@ export function* durableRun( throw new Error("Workflow was cancelled"); } } + replayIndex.claim(coroutineId); try { // Workflow is structurally assignable to Operation, so // yield* accepts it directly — no cast needed. const result: T = yield* workflow(); - const unconsumed = unconsumedReplay(replayIndex, coroutineId); + const durabilityFailure = activeDurabilityFailure(ctx); + if (durabilityFailure) { + throw durabilityFailure; + } + + const unconsumed = unalignedReplay(replayIndex, coroutineId); if (unconsumed) { throw new EarlyReturnDivergenceError( unconsumed.coroutineId, @@ -155,16 +146,17 @@ export function* durableRun( result: { status: "ok", value: result as Json }, }; - yield* stream.append(closeEvent); + yield* appendDurableEvent(ctx, closeEvent); return result; } catch (error) { const primary = error instanceof Error ? error : new Error(String(error)); - const unconsumed = unconsumedReplay(replayIndex, coroutineId); + const durabilityFailure = activeDurabilityFailure(ctx, primary); + if (durabilityFailure) { + throw durabilityFailure; + } + const unconsumed = unalignedReplay(replayIndex, coroutineId); if (unconsumed) { - if (isDurabilityFailure(primary)) { - throw primary; - } throw new TerminalDivergenceError( unconsumed.coroutineId, unconsumed.cursor, @@ -183,12 +175,15 @@ export function* durableRun( }; try { - yield* stream.append(closeEvent); - } catch (appendError) { - const appendFailure = - appendError instanceof Error ? appendError : new Error(String(appendError)); + yield* appendDurableEvent(ctx, closeEvent); + } catch (closeError) { + const closeDurabilityFailure = activeDurabilityFailure(ctx, closeError); + if (closeDurabilityFailure) { + throw closeDurabilityFailure; + } + const closeFailure = closeError instanceof Error ? closeError : new Error(String(closeError)); throw new AggregateError( - [primary, appendFailure], + [primary, closeFailure], "Workflow failed and Close append also failed", ); } diff --git a/packages/durable-streams/specs/DECISIONS.md b/packages/durable-streams/specs/DECISIONS.md index 19da56e8..04eae00b 100644 --- a/packages/durable-streams/specs/DECISIONS.md +++ b/packages/durable-streams/specs/DECISIONS.md @@ -154,8 +154,10 @@ Updated before completion of every phase and committed at the end of each phase. terminal divergence carries consumed/total counts and may retain the active execution failure as its cause. The common terminal base keeps the existing early-return API compatible while covering exceptional termination honestly. -- **Consequences:** No root `Close` is appended while replay entries remain - unconsumed. Error handling uses `instanceof` rather than matching names. +- **Consequences:** No root or child `Close` is appended while its retained + subtree is unaligned. Reaching a completed child claims its retained subtree; + abandoning that child is terminal divergence. Error handling uses + `instanceof` rather than matching names. ## DEC-009: Workflow = Generator, T, unknown> @@ -312,16 +314,17 @@ Updated before completion of every phase and committed at the end of each phase. - **Date:** 2026-02-28 - **Context:** The spec §5 defines the persist-before-resume invariant with three strategies. Need to choose one for the Effection integration. -- **Decision:** Strategy B — the effect's `enter()` calls `stream.append(event)` - and places `resolve()` inside the `.then()` callback. The generator does - not advance until the durable write completes. +- **Decision:** Strategy B — the effect's live operation appends the event and + calls `resolve()` only after that operation completes. The generator does not + advance successfully until the durable write completes. - **Rationale:** This is the natural fit for Effection's async resolve model. The reducer waits for `resolve()` to be called, so deferring it until after the append guarantees persist-before-resume. Verified by the ordering test (execute → persist → resume for each step). - **Consequences:** Live execution has one async hop per effect (the stream - append). During replay, `resolve()` is called synchronously — zero async - overhead. + append). A rejected backing append raises `DurablePersistenceError`, retains + the adapter error as its cause, and cannot produce a compensating `Close`. + During replay, `resolve()` is called synchronously — zero async overhead. ## DEC-018: durableCall constrains T extends Json for serializability diff --git a/packages/durable-streams/specs/effection-integration.md b/packages/durable-streams/specs/effection-integration.md index 8c8c11e3..298fb84a 100644 --- a/packages/durable-streams/specs/effection-integration.md +++ b/packages/durable-streams/specs/effection-integration.md @@ -458,8 +458,10 @@ function createDurableEffect(desc: EffectDescription, execute: Executor): Dur `resolve()` is called inside the `.then()` callback of the stream append. The generator does not advance until the durable write completes. This is the spec's "Strategy B: buffered write with deferred resume." If the -append rejects, the error is delivered through Effection's normal error -channel to avoid hanging the generator. +append rejects, `DurablePersistenceError` is delivered through Effection's +error channel and recorded in the shared durable context. Root and child +terminal handling therefore cannot reinterpret it as a workflow failure or +append a compensating `Close(err)`. **Transparency (§4.3).** During replay, `resolve()` is called synchronously with the stored result (converted via `protocolToEffection()`). The reducer @@ -660,66 +662,28 @@ This is an `Operation` meant to run inside a `spawn()`: ```typescript function* runDurableChild( - childWorkflow: () => Workflow | Operation, + childWorkflow: () => Workflow, childId: string, parentCtx: DurableContext, ): Operation { - const { replayIndex, stream } = parentCtx; - - // Short-circuit: child already completed in a previous run - if (replayIndex.hasClose(childId)) { - const closeEvent = replayIndex.getClose(childId)!; - if (closeEvent.result.status === "ok") { - return closeEvent.result.value as T; - } else if (closeEvent.result.status === "err") { - throw deserializeError(closeEvent.result.error); - } else { - // Cancelled in previous run — suspend until parent cancels us - yield* suspend(); - return undefined as T; // unreachable - } - } + parentCtx.replayIndex.claim(childId); - // Set child's DurableContext - const scope = yield* useScope(); - scope.set(DurableCtx, { - replayIndex, - stream, - coroutineId: childId, - childCounter: 0, - }); + if (parentCtx.replayIndex.hasClose(childId)) { + return yield* restoreChildClose(childId, parentCtx); + } - let closeEvent: Close | undefined; + const childCtx = installChildContext(childId, parentCtx); try { - const result: T = yield* childWorkflow(); - closeEvent = { - type: "close", - coroutineId: childId, - result: { status: "ok", value: result as Json }, - }; + const result = yield* childWorkflow(); + throwIfDurabilityFailed(childCtx); + assertSubtreeAligned(childCtx.replayIndex, childId, "return"); + yield* appendChildClose(childCtx, closeOk(childId, result)); return result; } catch (error) { - closeEvent = { - type: "close", - coroutineId: childId, - result: { - status: "err", - error: serializeError(error instanceof Error ? error : new Error(String(error))), - }, - }; + throwIfDurabilityFailed(childCtx, error); + assertSubtreeAligned(childCtx.replayIndex, childId, "error", error); + yield* appendChildClose(childCtx, closeError(childId, error)); throw error; - } finally { - if (!closeEvent) { - closeEvent = { - type: "close", - coroutineId: childId, - result: { status: "cancelled" }, - }; - } - // Don't re-emit if journal already has this Close - if (!replayIndex.hasClose(childId)) { - yield* call(() => stream.append(closeEvent!)); - } } } ``` @@ -727,25 +691,26 @@ function* runDurableChild( Key design decisions in this helper: **Short-circuit on existing Close.** If the journal already has a Close for -this child, `runDurableChild` never runs the workflow. For `ok` and `err`, +this child, `runDurableChild` claims the child identity but never runs the +workflow. For `ok` and `err`, it returns/throws immediately. For `cancelled`, it uses `yield* suspend()` (see §8.4). -**Close events via try/catch/finally.** The `closeEvent` variable starts -undefined. Normal completion sets it in the try block. Errors set it in -catch. If both are skipped (cancellation via `iterator.return()`), it -stays undefined and finally assigns `cancelled`. This covers all three -terminal states with plain JavaScript. +**The same terminal policy as the root.** Normal return, ordinary error, and +cancellation all check the child's retained subtree before a `Close`. A stale +input, divergence, terminal divergence, or journal persistence failure +suppresses the child close and propagates to the root without becoming a root +close. **No re-emission guard.** The `if (!replayIndex.hasClose(childId))` check in finally prevents writing a duplicate Close when replaying a child that was already completed. Without this, a fully-replayed child that short- circuits would emit a second Close event. -**`yield* call()` for stream append.** The finally block uses Effection's -`call()` to await the stream append within generator context, rather than -raw `await`. This keeps the code within Effection's structured concurrency -model. +**Shared persistence state.** Root and child contexts share the first active +durability failure. If a child append fails while sibling teardown runs, no +sibling can serialize the persistence failure as its own cancellation or +ordinary error outcome. ### 8.2 durableSpawn @@ -947,76 +912,29 @@ use prototypal inheritance). Each child scope has its own `coroutineId` and Implemented in `run.ts`. Key implementation details beyond the original design: short-circuits on existing Close event (DEC-016), checks terminal replay alignment before either successful or exceptional -closure, and emits Close(err) only when no replay entries remain unconsumed. +closure, and emits `Close(err)` only for an ordinary workflow failure after the +whole retained coroutine tree is aligned. Durability and persistence failures +never become terminal events. The entry point creates a scope, builds the replay index, sets up the durable context, and runs the workflow: ```typescript -interface DurableRunOptions { - stream: DurableStream; - coroutineId?: string; -} - -async function durableRun( - workflow: () => Workflow | Operation, - options: DurableRunOptions, -): Promise { - const { stream, coroutineId = "root" } = options; - const events = await stream.readAll(); - const replayIndex = new ReplayIndex(events); - - // Short-circuit: root already completed in a previous run - if (replayIndex.hasClose(coroutineId)) { - const closeEvent = replayIndex.getClose(coroutineId)!; - if (closeEvent.result.status === "ok") return closeEvent.result.value as T; - if (closeEvent.result.status === "err") throw deserializeError(closeEvent.result.error); - throw new Error("Workflow was cancelled"); - } - - const [scope, destroy] = createScope(); - scope.set(DurableCtx, { replayIndex, stream, coroutineId, childCounter: 0 }); - - try { - try { - const result = await scope.run(workflow); - - const unconsumed = replayIndex.firstUnconsumed(); - if (unconsumed) { - throw new EarlyReturnDivergenceError( - unconsumed.coroutineId, - unconsumed.cursor, - unconsumed.totalYields, - ); - } - - await stream.append({ - type: "close", - coroutineId, - result: { status: "ok", value: result as Json }, - }); - return result; - } catch (error) { - const unconsumed = replayIndex.firstUnconsumed(); - if (unconsumed) { - if (isDurabilityFailure(error)) throw error; - throw new TerminalDivergenceError( - unconsumed.coroutineId, - unconsumed.cursor, - unconsumed.totalYields, - { cause: error }, - ); - } - await stream.append({ - type: "close", - coroutineId, - result: { status: "err", error: serializeError(error) }, - }); - throw error; - } - } finally { - try { await destroy(); } catch { /* preserve the workflow outcome */ } - } +const ctx = { replayIndex, stream, coroutineId, childCounter: 0, durability: {} }; +scope.set(DurableCtx, ctx); +replayIndex.claim(coroutineId); + +try { + const result = yield* workflow(); + throwIfDurabilityFailed(ctx); + assertSubtreeAligned(replayIndex, coroutineId, "return"); + yield* appendDurableEvent(ctx, closeOk(coroutineId, result)); + return result; +} catch (error) { + throwIfDurabilityFailed(ctx, error); + assertSubtreeAligned(replayIndex, coroutineId, "error", error); + yield* appendDurableEvent(ctx, closeError(coroutineId, error)); + throw error; } ``` @@ -1034,19 +952,23 @@ Key details: where users pass a raw Operation at the top level. Structural compatibility means no cast is needed at the `scope.run()` call site. -- **Early return divergence check.** After the workflow returns, checks - if the replay index has unconsumed yields. If so, the generator finished - before replaying all journal entries — the code has changed (§6.3). The - check is outside the ordinary failure-to-Close path, so it appends no Close. +- **Subtree alignment.** The root claims its own identity, and every durable + child claims its identity before using a retained `Close`. Before root + termination, the index rejects unconsumed yields and retained completed + children that the current definition never claimed. - **Exceptional terminal divergence check.** Before an ordinary error becomes - `Close(err)`, checks the same replay state. Unconsumed entries produce + `Close(err)`, checks the same subtree state. Unaligned entries produce `TerminalDivergenceError`, with the ordinary error as its cause, and leave the - retained stream unchanged. An already-active durability failure is preserved. + retained stream unchanged. An already-active durability failure is preserved + even if it consumed the last retained yield. -- **Swallowing destroy errors.** If the workflow threw, `destroy()` may - also throw "halted". The `try { await destroy() } catch {}` in finally - prevents masking the original error. +- **Backing persistence is outside workflow outcomes.** Every backing append + failure records a shared `DurablePersistenceError` before it reaches root or + child terminal handling. A failed yield never resumes successfully, and a + failed successful close is not retried as `Close(err)`. A + `guardDurableStream` gate rejection is marked separately and remains the + gate's ordinary workflow failure. --- @@ -1345,7 +1267,7 @@ All cases from §6.3 are implemented and tested (DEC-008): divergence test 9 and 13. 2. **Generator fails early.** Detected in `durableRun()` before an exceptional - termination is journaled. Unconsumed replay entries raise + termination is journaled. Unaligned replay entries raise `TerminalDivergenceError`, preserve the original failure as the cause and append no `Close`. Tested in divergence test 13b. @@ -1354,8 +1276,15 @@ All cases from §6.3 are implemented and tested (DEC-008): but `hasClose()` returns true, throws `ContinuePastCloseDivergenceError`. Tested in divergence test 14. -The terminal errors carry consumed and total counts; exceptional termination -also carries the execution failure as its cause. +4. **Completed child is abandoned.** Each child identity is claimed before its + retained `Close` is restored. Root and child termination scan the whole + retained subtree, so a completed child absent from the current definition + raises terminal divergence and appends no `Close`. + +The same checks run at child termination. The terminal errors carry consumed +and total counts; exceptional termination also carries the execution failure +as its cause. An already-active durability failure bypasses ordinary terminal +serialization even when no unconsumed yield remains. ### 12.6 Durable `each()` — design and implementation plan diff --git a/packages/durable-streams/specs/protocol-specification.md b/packages/durable-streams/specs/protocol-specification.md index 94d2817b..09ce96d2 100644 --- a/packages/durable-streams/specs/protocol-specification.md +++ b/packages/durable-streams/specs/protocol-specification.md @@ -458,6 +458,13 @@ the protocol. > resume the generator. Violation of this invariant creates > unrecoverable replay gaps. +If backing persistence fails, the implementation MUST raise a durability +failure that retains the backend error as its cause. It MUST NOT resume the +consumer with an unjournaled success or append a compensating `Close(err)`. A +failed terminal `Close` append follows the same rule. A pre-persistence gate +rejection is distinct: the rejected event does not reach the backend, and the +gate's ordinary failure may be recorded by a separately admitted `Close(err)`. + ### 5.1 Why this is a hard invariant If the generator advances past a yield point whose resolution is not in @@ -550,7 +557,7 @@ divergence cases into run-live behavior for controlled migrations. ### 6.3 Terminal divergence cases Beyond per-effect matching, the reducer detects terminal divergence before it -appends a root `Close`: +appends any root or child `Close`: **Generator finishes early.** The generator returns `{ done: true }` while the replay index still has unconsumed entries for this coroutine. @@ -563,14 +570,22 @@ completed replay of the retained history. The runtime raises `TerminalDivergenceError`, retaining the execution error as its cause, and appends neither that ordinary error nor any root `Close`. +**Retained child is not claimed.** A retained child coroutine has completed, +but the current definition terminates its parent without reaching that child +identity. Its `Close` proves only that the earlier child completed; it does not +make the current coroutine tree aligned. + **Journal exhausted with close but generator continues.** The replay index has a `Close` event for this coroutine but the generator has not finished after consuming all recorded yields. This means the current code produces more effects than the recorded run — effects were added without a version gate. -All terminal divergence cases raise a durability error and leave the retained -journal unchanged. +Terminal alignment covers the whole terminating coroutine subtree. All +terminal divergence cases raise a durability error and leave the retained +journal unchanged. A durability failure discovered after the last replay entry +was consumed remains a durability failure and is never serialized as +`Close(err)`. ### 6.4 What is NOT checked @@ -1039,6 +1054,7 @@ These tests MUST pass for the protocol to be considered implemented. | 4 | **Crash at position N** | Provide first N events from golden stream. | First N effects replayed (not re-executed); remaining execute live; same result. | | 5 | **Crash after last effect** | Provide all `Yield` events but no `Close` events. | All effects replayed; close events written; same result. | | 6 | **Persist-before-resume verification** | Inject crash between effect resolution and `iterator.next()`. | On resume, the resolved effect is in the stream; no replay gap; no duplicate execution. | +| 6b | **Fail-once persistence** | Fail the first `Yield` append and, separately, the successful root `Close` append. | The adapter failure remains the cause; no compensating `Close` is appended and an unpersisted effect never resumes successfully. | | 7 | **Actor handoff** | Process A writes first N events, terminates. Process B reads stream, resumes. | B replays N events (none re-executed), continues live; correct result. | ### Tier 2 — Divergence detection @@ -1052,6 +1068,7 @@ These tests MUST pass for the protocol to be considered implemented. | 12 | **Name mismatch** | Record `call("fetchOrder")`. Replay yields `call("chargeCard")`. | `DivergenceError` citing name mismatch. | | 13 | **Generator finishes early** | Record a partial stream with 5 yields and no root close. Replay code produces only 3 yields then returns. | `EarlyReturnDivergenceError`; no root `Close` is appended. | | 13b | **Generator fails early** | Record a partial stream with a retained yield and no root close. Replay code throws before reaching it. | `TerminalDivergenceError`; the original error is its cause and no root `Close` is appended. | +| 13c | **Completed child removed** | Record a child `Yield` and `Close`, omit the root `Close`, then remove the child from the current definition. | Root return and root failure both reject the unchanged prefix; the compatible definition still replays it. | | 14 | **Generator continues past close** | Record stream with close after 3 yields. Replay code produces 5 yields. | `DivergenceError`: journal shows close but generator hasn't finished. | ### Tier 3 — Structured concurrency @@ -1067,6 +1084,7 @@ These tests MUST pass for the protocol to be considered implemented. | 21 | **Error boundary** | Child error caught by parent `try/catch`. | Parent catches error; siblings NOT cancelled; execution continues. | | 22 | **Race — winner cancels losers** | `race([op1, op2])`, op1 wins. | op2 receives `Close(cancelled)`; race returns op1's result. | | 23 | **Race replay with partial loser** | Replay race where loser partially executed. | Loser's partial yields replayed, then cancelled at correct point. | +| 23b | **Child durability failure** | Make replay stale or divergent inside a retained child. | Neither the child nor root appends `Close`; the exact durability failure escapes and the prefix remains unchanged. | ### Tier 4 — Deterministic identity diff --git a/packages/durable-streams/tests/durable-run.test.ts b/packages/durable-streams/tests/durable-run.test.ts index af876ea2..833f1839 100644 --- a/packages/durable-streams/tests/durable-run.test.ts +++ b/packages/durable-streams/tests/durable-run.test.ts @@ -11,10 +11,12 @@ import { expect } from "@executablemd/test-support/expect"; import { spawn, withResolvers, type Operation } from "effection"; import { type DurableEvent, + DurablePersistenceError, type DurableStream, InMemoryStream, type Json, type Workflow, + durableAction, durableCall, durableRun, } from "../mod.ts"; @@ -43,6 +45,34 @@ class PausedYieldStream implements DurableStream { } } +class FailOnceStream implements DurableStream { + readonly inner: InMemoryStream; + private failed = false; + + constructor( + readonly failure: Error, + events: DurableEvent[] = [], + ) { + this.inner = new InMemoryStream(events); + } + + *readAll(): Operation { + return yield* this.inner.readAll(); + } + + *append(event: DurableEvent): Operation { + if (!this.failed) { + this.failed = true; + throw this.failure; + } + yield* this.inner.append(event); + } + + snapshot(): DurableEvent[] { + return this.inner.snapshot(); + } +} + /** Track which functions were actually called during live execution. */ function createCallTracker() { const calls: string[] = []; @@ -269,9 +299,8 @@ describe("durableRun", () => { }); it("append failure cannot resolve a live effect successfully without a Yield", function* () { - const stream = new InMemoryStream(); const appendFailure = new Error("yield append failed"); - stream.injectFailure = appendFailure; + const stream = new FailOnceStream(appendFailure); let resumed = false; let failure: unknown; @@ -289,11 +318,93 @@ describe("durableRun", () => { expect(resumed).toBe(false); expect(stream.snapshot()).toHaveLength(0); - expect(failure).toBeInstanceOf(AggregateError); - if (!(failure instanceof AggregateError)) { - throw new Error("expected append failure aggregation"); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(appendFailure); + }); + + it("a failed successful root Close append does not write a compensating Close", function* () { + const appendFailure = new Error("root close append failed"); + const stream = new FailOnceStream(appendFailure); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + return "done"; + }, + { stream }, + ); + } catch (error) { + failure = error; } - expect(failure.errors[0]).toBe(appendFailure); + + expect(stream.snapshot()).toEqual([]); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(appendFailure); + }); + + it("callback effect append failure cannot become a workflow Close", function* () { + const appendFailure = new Error("action yield append failed"); + const stream = new FailOnceStream(appendFailure); + let resumed = false; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + yield* durableAction("step", (resolve) => { + resolve("completed"); + return () => {}; + }); + resumed = true; + return "unjournaled-success"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(resumed).toBe(false); + expect(stream.snapshot()).toEqual([]); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(appendFailure); + }); + + it("caught persistence failure still prevents root termination", function* () { + const appendFailure = new Error("caught yield append failed"); + const stream = new FailOnceStream(appendFailure); + let caught = false; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("step", () => Promise.resolve("completed")); + } catch { + caught = true; + } + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(caught).toBe(true); + expect(failure).toBeInstanceOf(DurablePersistenceError); + expect(stream.snapshot()).toEqual([]); }); it("actor handoff: Process B resumes from Process A's events", function* () { diff --git a/packages/durable-streams/tests/guard-stream.test.ts b/packages/durable-streams/tests/guard-stream.test.ts index 23af19dc..4a8c5c1a 100644 --- a/packages/durable-streams/tests/guard-stream.test.ts +++ b/packages/durable-streams/tests/guard-stream.test.ts @@ -154,22 +154,23 @@ describe("guardDurableStream", () => { it("keeps a rejected event out of an in-memory backend", function* () { const timeline: string[] = []; + const rejection = new Error("rejected"); const guarded = guardDurableStream( recordingStream(timeline), // deno-lint-ignore require-yield function* () { - throw new Error("rejected"); + throw rejection; }, ); - let failure: Error | undefined; + let failure: unknown; try { yield* guarded.append(EVENT); } catch (error) { - failure = error instanceof Error ? error : new Error(String(error)); + failure = error; } - expect(failure?.message).toBe("rejected"); + expect(failure).toBe(rejection); expect(backendAppends(timeline)).toEqual([]); }); @@ -199,6 +200,36 @@ describe("guardDurableStream", () => { expect(yield* exists(journalPath)).toBe(false); }); + it("keeps gate rejection distinct from backing-stream failure", function* () { + const backend = new InMemoryStream(); + const rejection = new Error("policy rejected the event"); + const guarded = guardDurableStream( + backend, + // deno-lint-ignore require-yield + function* (event) { + if (event.type === "yield") { + throw rejection; + } + }, + ); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + return yield* durableCall("step", waited(0, "value")); + }, + { stream: guarded }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBe(rejection); + expect(backend.snapshot()).toHaveLength(1); + expect(backend.snapshot()[0]?.type).toBe("close"); + }); + it("hands the gate a copy, so mutating it cannot change what is persisted", function* () { const backend = new InMemoryStream(); const original = structuredClone(EVENT); diff --git a/packages/durable-streams/tests/replay-guard.test.ts b/packages/durable-streams/tests/replay-guard.test.ts index 4863cb2c..4baf074c 100644 --- a/packages/durable-streams/tests/replay-guard.test.ts +++ b/packages/durable-streams/tests/replay-guard.test.ts @@ -195,6 +195,10 @@ describe("replay guard", () => { }, ]; const stream = new InMemoryStream(events); + const before = stream.snapshot(); + const stale = new StaleInputError( + "File changed: ./test.txt (recorded: abc123, current: def456)", + ); // Cache simulates current file having a DIFFERENT hash const cache = new Map([["./test.txt", "def456"]]); @@ -212,9 +216,7 @@ describe("replay guard", () => { if (currentSHA && currentSHA !== recordedHash) { return { outcome: "error", - error: new StaleInputError( - `File changed: ${filePath} (recorded: ${recordedHash}, current: ${currentSHA})`, - ), + error: stale, }; } } @@ -236,10 +238,11 @@ describe("replay guard", () => { ); throw new Error("expected StaleInputError"); } catch (e) { - expect(e).toBeInstanceOf(StaleInputError); + expect(e).toBe(stale); expect((e as Error).message).toContain("File changed"); expect((e as Error).message).toContain("./test.txt"); } + expect(stream.snapshot()).toEqual(before); }); it("multiple guards — error from any guard halts replay", function* () { diff --git a/packages/durable-streams/tests/terminal-boundary.test.ts b/packages/durable-streams/tests/terminal-boundary.test.ts new file mode 100644 index 00000000..2503a23e --- /dev/null +++ b/packages/durable-streams/tests/terminal-boundary.test.ts @@ -0,0 +1,293 @@ +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import type { Operation } from "effection"; +import { + ContinuePastCloseDivergenceError, + DivergenceError, + type DurableEvent, + EarlyReturnDivergenceError, + InMemoryStream, + ReplayGuard, + StaleInputError, + TerminalDivergenceError, + type Workflow, + durableAll, + durableCall, + durableRun, + serializeDurableEvent, +} from "../mod.ts"; + +function expectUnchanged(stream: InMemoryStream, before: DurableEvent[]): void { + const after = stream.snapshot(); + expect(after).toEqual(before); + expect(after.map(serializeDurableEvent).join("")).toBe( + before.map(serializeDurableEvent).join(""), + ); +} + +function* recordCompletedChild(): Operation { + const stream = new InMemoryStream(); + yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* () { + return yield* durableCall("child-step", () => Promise.resolve("recorded")); + }, + ]); + return value; + }, + { stream }, + ); + return stream.snapshot().filter((event) => event.coroutineId !== "root"); +} + +function* replayCompletedChild(stream: InMemoryStream): Operation { + return yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* () { + return yield* durableCall("child-step", () => Promise.resolve("not-executed")); + }, + ]); + return value; + }, + { stream }, + ); +} + +function* recordTwoStepChild(): Operation { + const stream = new InMemoryStream(); + yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* () { + yield* durableCall("child-first", () => Promise.resolve("first")); + return yield* durableCall("child-second", () => Promise.resolve("second")); + }, + ]); + return value; + }, + { stream }, + ); + return stream.snapshot().filter((event) => event.type === "yield"); +} + +describe("durable terminal boundary", () => { + it("claiming a completed child aligns its retained descendants", function* () { + const golden = new InMemoryStream(); + let executions = 0; + const nestedWorkflow = function* (): Workflow { + const [outer] = yield* durableAll([ + function* () { + const [inner] = yield* durableAll([ + function* () { + return yield* durableCall("nested-step", () => { + executions++; + return Promise.resolve("recorded"); + }); + }, + ]); + return inner; + }, + ]); + return outer; + }; + + yield* durableRun(nestedWorkflow, { stream: golden }); + const retained = golden.snapshot().filter((event) => event.coroutineId !== "root"); + const replay = new InMemoryStream(retained); + + expect(yield* durableRun(nestedWorkflow, { stream: replay })).toBe("recorded"); + expect(executions).toBe(1); + expect(replay.snapshot().at(-1)?.coroutineId).toBe("root"); + }); + + it("rejects a normal root return that abandons a completed child", function* () { + const retained = yield* recordCompletedChild(); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + let failure: unknown; + + try { + yield* durableRun( + // deno-lint-ignore require-yield + function* (): Workflow { + return "child-removed"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(EarlyReturnDivergenceError); + expectUnchanged(stream, before); + + const compatible = new InMemoryStream(retained); + expect(yield* replayCompletedChild(compatible)).toBe("recorded"); + expect(compatible.snapshot().at(-1)?.coroutineId).toBe("root"); + }); + + it("rejects a root error that abandons a completed child", function* () { + const retained = yield* recordCompletedChild(); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + const workflowFailure = new Error("current root failed"); + let failure: unknown; + + try { + yield* durableRun( + // deno-lint-ignore require-yield + function* (): Workflow { + throw workflowFailure; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(TerminalDivergenceError); + if (!(failure instanceof TerminalDivergenceError)) { + throw new Error("expected terminal divergence"); + } + expect(failure.cause).toBe(workflowFailure); + expectUnchanged(stream, before); + }); + + it("does not close a child or root over stale child history", function* () { + const completed = yield* recordCompletedChild(); + const retained = completed.filter((event) => event.type === "yield"); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + const stale = new StaleInputError("child input changed"); + + yield* ReplayGuard.around({ + *check([event], next) { + return yield* next(event); + }, + decide() { + return { outcome: "error", error: stale }; + }, + }); + + let failure: unknown; + try { + yield* replayCompletedChild(stream); + } catch (error) { + failure = error; + } + + expect(failure).toBe(stale); + expectUnchanged(stream, before); + }); + + it("does not close a child or root over divergent child history", function* () { + const completed = yield* recordCompletedChild(); + const retained = completed.filter((event) => event.type === "yield"); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* () { + return yield* durableCall("changed-child-step", () => + Promise.resolve("not-executed"), + ); + }, + ]); + return value; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(DivergenceError); + expectUnchanged(stream, before); + }); + + it("checks child alignment before a successful child Close", function* () { + const retained = yield* recordTwoStepChild(); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* () { + yield* durableCall("child-first", () => Promise.resolve("not-executed")); + return "child-finished-early"; + }, + ]); + return value; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(EarlyReturnDivergenceError); + expectUnchanged(stream, before); + }); + + it("checks child alignment before a failed child Close", function* () { + const retained = yield* recordTwoStepChild(); + const stream = new InMemoryStream(retained); + const before = stream.snapshot(); + const childFailure = new Error("child failed early"); + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + const [value] = yield* durableAll([ + function* (): Workflow { + yield* durableCall("child-first", () => Promise.resolve("not-executed")); + throw childFailure; + }, + ]); + return value; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(TerminalDivergenceError); + if (!(failure instanceof TerminalDivergenceError)) { + throw new Error("expected terminal divergence"); + } + expect(failure.cause).toBe(childFailure); + expectUnchanged(stream, before); + }); + + it("does not serialize continue-past-close as a document outcome", function* () { + const stream = new InMemoryStream(); + const divergence = new ContinuePastCloseDivergenceError("root.0", 1); + let failure: unknown; + + try { + yield* durableRun( + // deno-lint-ignore require-yield + function* (): Workflow { + throw divergence; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBe(divergence); + expect(stream.snapshot()).toEqual([]); + }); +}); diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index d0e60775..2685a1d0 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -1283,9 +1283,11 @@ current definition no longer yields, the existing replay guard, divergence and stale-input rules reject that incompatible history; validation never consumes the recorded result or substitutes an unjournaled error. If collision handling terminates the document before another durable effect is reached, the durable -root detects the unconsumed eval `Yield` before writing `Close(err)`, raises +root detects the unaligned eval `Yield` before writing `Close(err)`, raises `TerminalDivergenceError` with the collision as its cause, and leaves the -retained journal unchanged. A compatible definition can still replay it. +retained journal unchanged. Root and child termination apply this check to the +whole retained coroutine subtree, including completed children the current +definition never claimed. A compatible definition can still replay it. `ephemeral eval` validates its exports against durable names before execution; it may atomically replace an existing live binding. A failed block publishes none of its exports. @@ -4463,6 +4465,14 @@ journal actually stopped describing the run. Every one of these is discovered through the same cycle-safe cause traversal, so what the caller receives is the failure itself rather than the wrapper it travelled in. +**Backing-journal persistence failures are rethrown on the same terms.** A +failed `Yield` append raises `DurablePersistenceError` before the effect +consumer can resume successfully. A failed `Close` append raises the same +error. Its cause is the storage adapter's error, and neither case writes a +compensating `Close(err)` that would misstate the persistence failure as a +document outcome. Pre-persistence policy rejection remains the policy's +ordinary failure and may be recorded by a separately admitted `Close(err)`. + **Durability discovery traverses the whole cause graph.** No wrapper keeps a durability failure from being found, including a content failure a component recovered from (§5.1.2). Recovery settles which failure the *document* reports, @@ -6423,13 +6433,13 @@ visible warning blocks, gather into a separate error report). | FA6 | Both at once | A fatal error is still found when the wrapper holding it is itself cyclic | | FA7 | Documentation failures | A `DocumentationError` is discovered the same way | | FA8 | Ordinary errors are unaffected | A cyclic ordinary error is printed and the next block still runs | -| FA9 | Every durability failure | `StaleInputError`, `DivergenceError`, `TerminalDivergenceError`, `EarlyReturnDivergenceError`, and `ContinuePastCloseDivergenceError` are each discovered as fatal, bare and wrapped | -| FA10 | Precedence, either order | Each of the five outranks a `DocumentationError` in an `AggregateError`, whichever comes first | +| FA9 | Every durability failure | `StaleInputError`, `DivergenceError`, `TerminalDivergenceError`, `EarlyReturnDivergenceError`, `ContinuePastCloseDivergenceError`, and `DurablePersistenceError` are each discovered as fatal, bare and wrapped | +| FA10 | Precedence, either order | Each durability failure outranks a `DocumentationError` in an `AggregateError`, whichever comes first | | FA11 | Precedence through a teardown | The same holds for an `InvocationTeardownError`'s stage failures | | FA12 | Precedence at any depth | Nesting either one deeper than the other does not change the answer | | FA13 | No durability failure | A `DocumentationError` is reported when the graph holds none, and `durabilityFailure` finds nothing | | FA14 | Precedence with a cycle | A mixed graph that is also cyclic still reports the durability failure | -| FA15 | A content failure hides nothing fatal | A durability failure beneath a `ContentError` — set by a subclass and by assignment — is found by both `durabilityFailure` and `fatalCause`, for each of the five kinds | +| FA15 | A content failure hides nothing fatal | A durability failure beneath a `ContentError` — set by a subclass and by assignment — is found by both `durabilityFailure` and `fatalCause`, for every kind | | FA16 | Wherever the content failure sits | The same holds beneath an ordinary cause, inside an `AggregateError`, inside an `InvocationTeardownError`, and through all three at once | | FA17 | No resurrection | A `DocumentationError` a component recovered from is not reported as the outward failure, while the same one reached without crossing a content failure still is | | FA18 | Precedence behind a content failure | A durability failure beneath a recovered content failure outranks a documentation failure, in either wrapper order | From ef7ae6d4a011d29dd9fa83f73f5660f6c245ad7b Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:25:39 -0400 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=90=9B=20fence=20durable=20runs=20aft?= =?UTF-8?q?er=20protocol=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture.md | 11 +- packages/durable-streams/context.ts | 13 +- packages/durable-streams/durability.ts | 94 ++++- packages/durable-streams/effect.ts | 22 +- .../specs/effection-integration.md | 31 +- .../specs/protocol-specification.md | 16 +- .../durable-streams/tests/fail-stop.test.ts | 375 ++++++++++++++++++ 7 files changed, 537 insertions(+), 25 deletions(-) create mode 100644 packages/durable-streams/tests/fail-stop.test.ts diff --git a/architecture.md b/architecture.md index b0bef8aa..5aee4698 100644 --- a/architecture.md +++ b/architecture.md @@ -629,9 +629,14 @@ A durability failure (§6.11) says the journal no longer describes the document execution. No middleware sees it; it is never the document's own outcome. A backing-journal append failure is the same kind of boundary failure: it preserves the adapter error as its cause, resumes no consumer with an -unpersisted success, and never triggers a compensating `Close`. A -pre-persistence policy rejection remains the policy's ordinary document -failure; the guarded stream marks that boundary before the backing append. +unpersisted success, and never triggers a compensating `Close`. The first +durability failure is shared by the root and every durable child. From that +point, durable entry fails with that exact error before replay or execution, +and the ordered append boundary rechecks it immediately before storage. Work +already executing may finish, but its pending append and every later append are +fenced. A pre-persistence policy rejection remains the policy's ordinary +document failure; the guarded stream marks that boundary before the backing +append and does not activate the fail-stop state. ## Attempts diff --git a/packages/durable-streams/context.ts b/packages/durable-streams/context.ts index 6c210fe4..d14c5ac6 100644 --- a/packages/durable-streams/context.ts +++ b/packages/durable-streams/context.ts @@ -6,11 +6,20 @@ * coroutineId and childCounter. */ -import { type Context, createContext } from "effection"; +import { type Context, createContext, type Operation } from "effection"; import type { ReplayIndex } from "./replay-index.ts"; import type { DurableStream } from "./stream.ts"; import type { CoroutineId } from "./types.ts"; +export interface DurableAppendFence { + hold(): Operation; +} + +export interface DurabilityState { + failure?: Error; + appendFence?: DurableAppendFence; +} + export interface DurableContext { /** Shared replay index (built from stream on startup). */ replayIndex: ReplayIndex; @@ -21,7 +30,7 @@ export interface DurableContext { /** Counter for assigning child IDs. */ childCounter: number; /** Protocol failure shared by the root and every durable child. */ - durability?: { failure?: Error }; + durability?: DurabilityState; } /** diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index 1332b796..3630ba02 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -1,5 +1,6 @@ -import type { Operation } from "effection"; -import type { DurableContext } from "./context.ts"; +import { ensure, resource, withResolvers } from "effection"; +import type { Operation, WithResolvers } from "effection"; +import type { DurableAppendFence, DurableContext, DurabilityState } from "./context.ts"; import { ContinuePastCloseDivergenceError, DivergenceError, @@ -10,6 +11,65 @@ import { import { isDurableEventRejection, unwrapDurableEventRejection } from "./guard.ts"; import type { DurableEvent } from "./types.ts"; +interface AppendTurn { + readonly gate: WithResolvers; + granted: boolean; +} + +function createAppendFence(): DurableAppendFence { + let held = false; + const waiting: AppendTurn[] = []; + + function release(): void { + const next = waiting.shift(); + if (next === undefined) { + held = false; + return; + } + next.granted = true; + next.gate.resolve(); + } + + return { + hold: () => + resource(function* (provide) { + const turn: AppendTurn = { gate: withResolvers(), granted: false }; + + yield* ensure(() => { + if (turn.granted) { + release(); + return; + } + const index = waiting.indexOf(turn); + if (index >= 0) { + waiting.splice(index, 1); + } + }); + + if (held) { + waiting.push(turn); + yield* turn.gate.operation; + } else { + held = true; + turn.granted = true; + } + + yield* provide(); + }), + }; +} + +function durabilityState(ctx: DurableContext): DurabilityState { + ctx.durability ??= {}; + return ctx.durability; +} + +function appendFence(ctx: DurableContext): DurableAppendFence { + const state = durabilityState(ctx); + state.appendFence ??= createAppendFence(); + return state.appendFence; +} + export function findDurabilityFailure(error: unknown): Error | undefined { const visited = new Set(); const pending: unknown[] = [error]; @@ -42,9 +102,9 @@ export function findDurabilityFailure(error: unknown): Error | undefined { } export function rememberDurabilityFailure(ctx: DurableContext, error: Error): Error { - ctx.durability ??= {}; - ctx.durability.failure ??= error; - return ctx.durability.failure; + const state = durabilityState(ctx); + state.failure ??= error; + return state.failure; } export function activeDurabilityFailure(ctx: DurableContext, error?: unknown): Error | undefined { @@ -59,19 +119,33 @@ export function activeDurabilityFailure(ctx: DurableContext, error?: unknown): E } export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): Operation { + const existing = activeDurabilityFailure(ctx); + if (existing) { + throw existing; + } + + yield* appendFence(ctx).hold(); + + const admitted = activeDurabilityFailure(ctx); + if (admitted) { + throw admitted; + } + try { yield* ctx.stream.append(event); } catch (error) { if (isDurableEventRejection(error)) { const rejection = unwrapDurableEventRejection(error); - const failure = findDurabilityFailure(rejection); + const failure = activeDurabilityFailure(ctx, rejection); if (failure) { - rememberDurabilityFailure(ctx, failure); + throw failure; } throw rejection; } - const failure = new DurablePersistenceError(event.type, error); - rememberDurabilityFailure(ctx, failure); - throw failure; + const active = activeDurabilityFailure(ctx, error); + if (active) { + throw active; + } + throw rememberDurabilityFailure(ctx, new DurablePersistenceError(event.type, error)); } } diff --git a/packages/durable-streams/effect.ts b/packages/durable-streams/effect.ts index f120ccf0..5cffa6f6 100644 --- a/packages/durable-streams/effect.ts +++ b/packages/durable-streams/effect.ts @@ -23,7 +23,11 @@ import type { Operation } from "effection"; import { type DurableContext, DurableCtx } from "./context.ts"; import { Divergence } from "./divergence.ts"; -import { appendDurableEvent, rememberDurabilityFailure } from "./durability.ts"; +import { + activeDurabilityFailure, + appendDurableEvent, + rememberDurabilityFailure, +} from "./durability.ts"; import { StaleInputError } from "./errors.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { protocolToEffection, serializeError } from "./serialize.ts"; @@ -202,6 +206,11 @@ export function createDurableEffect( routine, ): (resolve: Resolve>) => void { const ctx = routine.scope.expect(DurableCtx); + const durabilityFailure = activeDurabilityFailure(ctx); + if (durabilityFailure) { + resolve({ ok: false, error: durabilityFailure }); + return (exit) => exit(VOID_OK); + } const replay = checkReplay(desc, resolve, routine, ctx); if (replay.path === "replayed") { return replay.teardown; @@ -311,6 +320,11 @@ export function createDurableOperation( routine, ): (resolve: Resolve>) => void { const ctx = routine.scope.expect(DurableCtx); + const durabilityFailure = activeDurabilityFailure(ctx); + if (durabilityFailure) { + resolve({ ok: false, error: durabilityFailure }); + return (exit) => exit(VOID_OK); + } const replay = checkReplay(desc, resolve, routine, ctx); if (replay.path === "replayed") { return replay.teardown; @@ -320,6 +334,12 @@ export function createDurableOperation( // Run the entire execute → capture → persist → resolve sequence // as a structured operation in the routine's scope. routine.scope.run(function* () { + const active = activeDurabilityFailure(ctx); + if (active) { + resolve({ ok: false, error: active }); + return; + } + let result: Result; try { const value = yield* execute(); diff --git a/packages/durable-streams/specs/effection-integration.md b/packages/durable-streams/specs/effection-integration.md index 298fb84a..0022cee8 100644 --- a/packages/durable-streams/specs/effection-integration.md +++ b/packages/durable-streams/specs/effection-integration.md @@ -461,7 +461,12 @@ the spec's "Strategy B: buffered write with deferred resume." If the append rejects, `DurablePersistenceError` is delivered through Effection's error channel and recorded in the shared durable context. Root and child terminal handling therefore cannot reinterpret it as a workflow failure or -append a compensating `Close(err)`. +append a compensating `Close(err)`. Durable entry checks that shared state +before replay matching or executor startup. Appends pass through a shared FIFO +boundary and check it again immediately before calling the stream adapter, so +an append queued behind the failure never reaches storage. An executor that +started concurrently may complete, but it cannot persist its result after the +failure becomes active. **Transparency (§4.3).** During replay, `resolve()` is called synchronously with the stored result (converted via `protocolToEffection()`). The reducer @@ -710,7 +715,10 @@ circuits would emit a second Close event. **Shared persistence state.** Root and child contexts share the first active durability failure. If a child append fails while sibling teardown runs, no sibling can serialize the persistence failure as its own cancellation or -ordinary error outcome. +ordinary error outcome. The state is fail-stop across catches: subsequent root, +child, or sibling durable entry raises the exact first error without consuming +replay or starting an executor. The append boundary preserves the same identity +and fences work already queued for persistence. ### 8.2 durableSpawn @@ -966,9 +974,11 @@ Key details: - **Backing persistence is outside workflow outcomes.** Every backing append failure records a shared `DurablePersistenceError` before it reaches root or child terminal handling. A failed yield never resumes successfully, and a - failed successful close is not retried as `Close(err)`. A - `guardDurableStream` gate rejection is marked separately and remains the - gate's ordinary workflow failure. + failed successful close is not retried as `Close(err)`. Once active, that + exact failure rejects all later durable entry and ordered append across the + root and children; no later replay entry is consumed and no later executor + begins. A `guardDurableStream` gate rejection is marked separately, remains + the gate's ordinary workflow failure, and does not activate fail-stop state. --- @@ -1284,7 +1294,11 @@ All cases from §6.3 are implemented and tested (DEC-008): The same checks run at child termination. The terminal errors carry consumed and total counts; exceptional termination also carries the execution failure as its cause. An already-active durability failure bypasses ordinary terminal -serialization even when no unconsumed yield remains. +serialization even when no unconsumed yield remains. This terminal check is the +last defense, not the first: durable entry and the ordered append boundary also +reject with the exact active failure, so workflow code cannot catch a +durability error and advance replay, execute another effect, or persist another +event. ### 12.6 Durable `each()` — design and implementation plan @@ -1659,8 +1673,9 @@ crashes. 5. ~~Implement `durableRun`~~ — Entry point with in-memory stream (`run.ts`). 6. ~~Run Tier 1 tests~~ — Golden run, full replay, crash-at-N, - persist-before-resume, actor handoff — all passing - (`durable-run.test.ts`). + persist-before-resume, actor handoff, and caught fail-stop behavior across + root and child coroutines — all passing (`durable-run.test.ts`, + `fail-stop.test.ts`). 7. ~~Run Tier 2 tests~~ — All divergence detection cases passing (`divergence.test.ts`). 8. ~~Implement `durableSpawn`, `durableAll`, `durableRace`~~ — Workflow diff --git a/packages/durable-streams/specs/protocol-specification.md b/packages/durable-streams/specs/protocol-specification.md index 09ce96d2..0501e9c6 100644 --- a/packages/durable-streams/specs/protocol-specification.md +++ b/packages/durable-streams/specs/protocol-specification.md @@ -465,6 +465,16 @@ failed terminal `Close` append follows the same rule. A pre-persistence gate rejection is distinct: the rejected event does not reach the backend, and the gate's ordinary failure may be recorded by a separately admitted `Close(err)`. +The first durability failure makes the root and all of its child coroutines +fail-stop. Every later durable-effect entry MUST raise that exact error before +replay matching or live execution. Every ordered append MUST recheck the shared +failure immediately before invoking storage; an append waiting behind the +operation that failed MUST NOT reach the stream adapter. Concurrent executors +that started before the failure may finish, but their pending appends are +fenced. The first error and, for `DurablePersistenceError`, its adapter cause +retain their identity even when workflow code catches the error. An ordinary +pre-persistence policy rejection does not activate fail-stop state. + ### 5.1 Why this is a hard invariant If the generator advances past a yield point whose resolution is not in @@ -585,7 +595,9 @@ Terminal alignment covers the whole terminating coroutine subtree. All terminal divergence cases raise a durability error and leave the retained journal unchanged. A durability failure discovered after the last replay entry was consumed remains a durability failure and is never serialized as -`Close(err)`. +`Close(err)`. Once any durability failure is active, no later replay entry is +consumed, durable executor begins, or ordered `Yield` or `Close` append reaches +storage. ### 6.4 What is NOT checked @@ -1037,6 +1049,7 @@ For reference, the complete set of invariants defined in this specification: | 11 | Append-Only | §11.3 | No mutation or deletion | | 12 | Prefix-Closed | §11.3 | No gaps in the stream | | 13 | Monotonic Indexing | §11.3 | Sequential offsets | +| 14 | Durability Fail-Stop | §5 | First durability failure fences later work | --- @@ -1055,6 +1068,7 @@ These tests MUST pass for the protocol to be considered implemented. | 5 | **Crash after last effect** | Provide all `Yield` events but no `Close` events. | All effects replayed; close events written; same result. | | 6 | **Persist-before-resume verification** | Inject crash between effect resolution and `iterator.next()`. | On resume, the resolved effect is in the stream; no replay gap; no duplicate execution. | | 6b | **Fail-once persistence** | Fail the first `Yield` append and, separately, the successful root `Close` append. | The adapter failure remains the cause; no compensating `Close` is appended and an unpersisted effect never resumes successfully. | +| 6c | **Caught durability failure** | Catch persistence and replay-divergence failures, then attempt later effects and concurrent child appends. | The exact first failure escapes; no later replay is consumed or executor starts, and no append queued behind the failure reaches storage. An ordinary gate rejection remains catchable and permits later effects. | | 7 | **Actor handoff** | Process A writes first N events, terminates. Process B reads stream, resumes. | B replays N events (none re-executed), continues live; correct result. | ### Tier 2 — Divergence detection diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts new file mode 100644 index 00000000..f1a61749 --- /dev/null +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -0,0 +1,375 @@ +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { sleep, withResolvers, type Operation } from "effection"; +import { + DivergenceError, + type DurableEvent, + DurablePersistenceError, + type DurableStream, + InMemoryStream, + ReplayGuard, + type Workflow, + durableAction, + durableAll, + durableCall, + durableRun, + ephemeral, + guardDurableStream, + serializeDurableEvent, +} from "../mod.ts"; + +function eventBytes(events: DurableEvent[]): string { + return events.map(serializeDurableEvent).join(""); +} + +class FailOnceStream implements DurableStream { + readonly inner: InMemoryStream; + appendAttempts = 0; + private failed = false; + + constructor( + readonly failure: Error, + events: DurableEvent[] = [], + ) { + this.inner = new InMemoryStream(events); + } + + *readAll(): Operation { + return yield* this.inner.readAll(); + } + + *append(event: DurableEvent): Operation { + this.appendAttempts++; + if (!this.failed) { + this.failed = true; + throw this.failure; + } + yield* this.inner.append(event); + } + + snapshot(): DurableEvent[] { + return this.inner.snapshot(); + } +} + +class BlockingFailureStream implements DurableStream { + readonly firstAppendStarted = withResolvers(); + readonly releaseFirstAppend = withResolvers(); + readonly inner = new InMemoryStream(); + readonly attempts: DurableEvent[] = []; + + constructor(readonly failure: Error) {} + + *readAll(): Operation { + return yield* this.inner.readAll(); + } + + *append(event: DurableEvent): Operation { + this.attempts.push(structuredClone(event)); + if (this.attempts.length === 1) { + this.firstAppendStarted.resolve(); + yield* this.releaseFirstAppend.operation; + throw this.failure; + } + yield* this.inner.append(event); + } + + snapshot(): DurableEvent[] { + return this.inner.snapshot(); + } +} + +describe("durable fail-stop boundary", () => { + it("fences a later callback executor after a caught persistence failure", function* () { + const adapterFailure = new Error("first append failed"); + const stream = new FailOnceStream(adapterFailure); + let firstExecutions = 0; + let laterExecutions = 0; + let caught: unknown; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("poison", () => { + firstExecutions++; + return Promise.resolve("completed"); + }); + } catch (error) { + caught = error; + } + return yield* durableAction("later", (resolve) => { + laterExecutions++; + resolve("must-not-run"); + return () => {}; + }); + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(firstExecutions).toBe(1); + expect(laterExecutions).toBe(0); + expect(stream.appendAttempts).toBe(1); + expect(stream.snapshot()).toEqual([]); + expect(caught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(adapterFailure); + }); + + it("fences replay consumption and live execution after a caught divergence", function* () { + const retained: DurableEvent[] = [ + { + type: "yield", + coroutineId: "root", + description: { type: "call", name: "first" }, + result: { status: "ok", value: "one" }, + }, + { + type: "yield", + coroutineId: "root", + description: { type: "call", name: "second" }, + result: { status: "ok", value: "two" }, + }, + ]; + const stream = new InMemoryStream(retained); + const before = eventBytes(stream.snapshot()); + let replayDecisions = 0; + let laterExecutions = 0; + let caught: unknown; + let failure: unknown; + + yield* ReplayGuard.around({ + *check([event], next) { + return yield* next(event); + }, + decide([event], next) { + replayDecisions++; + return next(event); + }, + }); + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("changed", () => { + laterExecutions++; + return Promise.resolve("must-not-run"); + }); + } catch (error) { + caught = error; + } + yield* durableCall("first", () => { + laterExecutions++; + return Promise.resolve("must-not-run"); + }); + yield* durableCall("second", () => { + laterExecutions++; + return Promise.resolve("must-not-run"); + }); + return yield* durableCall("live", () => { + laterExecutions++; + return Promise.resolve("must-not-run"); + }); + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(caught).toBeInstanceOf(DivergenceError); + expect(failure).toBe(caught); + expect(replayDecisions).toBe(0); + expect(laterExecutions).toBe(0); + expect(stream.appendCount).toBe(0); + expect(eventBytes(stream.snapshot())).toBe(before); + }); + + it("shares the first durability failure across child and sibling entry", function* () { + const adapterFailure = new Error("child append failed"); + const stream = new FailOnceStream(adapterFailure); + const failureActive = withResolvers(); + const siblingObserved = withResolvers(); + let poisonExecutions = 0; + let siblingExecutions = 0; + let childCaught: unknown; + let siblingCaught: unknown; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + yield* durableAll([ + function* () { + try { + yield* durableCall("child-poison", () => { + poisonExecutions++; + return Promise.resolve("completed"); + }); + } catch (error) { + childCaught = error; + failureActive.resolve(); + } + yield* ephemeral(siblingObserved.operation); + return "caught"; + }, + function* () { + yield* ephemeral(failureActive.operation); + try { + yield* durableCall("sibling-later", () => { + siblingExecutions++; + return Promise.resolve("must-not-run"); + }); + } catch (error) { + siblingCaught = error; + } + siblingObserved.resolve(); + return "caught"; + }, + ]); + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(poisonExecutions).toBe(1); + expect(siblingExecutions).toBe(0); + expect(stream.appendAttempts).toBe(1); + expect(stream.snapshot()).toEqual([]); + expect(childCaught).toBe(failure); + expect(siblingCaught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(adapterFailure); + }); + + it("fences an append queued behind the append that activates failure", function* () { + const adapterFailure = new Error("blocked append failed"); + const stream = new BlockingFailureStream(adapterFailure); + const secondExecuted = withResolvers(); + const secondObserved = withResolvers(); + let firstExecutions = 0; + let secondExecutions = 0; + let firstCaught: unknown; + let secondCaught: unknown; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + yield* durableAll([ + function* () { + try { + yield* durableCall("first-append", () => { + firstExecutions++; + return Promise.resolve("first"); + }); + } catch (error) { + firstCaught = error; + } + yield* ephemeral(secondObserved.operation); + return "caught"; + }, + function* () { + yield* ephemeral(stream.firstAppendStarted.operation); + try { + yield* durableCall("queued-append", () => { + secondExecutions++; + secondExecuted.resolve(); + return Promise.resolve("second"); + }); + } catch (error) { + secondCaught = error; + } + secondObserved.resolve(); + return "caught"; + }, + function* () { + yield* ephemeral(secondExecuted.operation); + yield* ephemeral(sleep(0)); + stream.releaseFirstAppend.resolve(); + return "released"; + }, + ]); + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(firstExecutions).toBe(1); + expect(secondExecutions).toBe(1); + expect(stream.attempts).toHaveLength(1); + expect(stream.snapshot()).toEqual([]); + expect(firstCaught).toBe(failure); + expect(secondCaught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(adapterFailure); + }); + + it("keeps a caught guard-policy rejection non-poisoning", function* () { + const backend = new InMemoryStream(); + const rejection = new Error("policy rejected first yield"); + let blockedExecutions = 0; + let laterExecutions = 0; + let caught: unknown; + const stream = guardDurableStream( + backend, + // deno-lint-ignore require-yield + function* (event) { + if (event.type === "yield" && event.description.name === "blocked") { + throw rejection; + } + }, + ); + + const result = yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("blocked", () => { + blockedExecutions++; + return Promise.resolve("blocked"); + }); + } catch (error) { + caught = error; + } + return yield* durableCall("later", () => { + laterExecutions++; + return Promise.resolve("completed"); + }); + }, + { stream }, + ); + + expect(result).toBe("completed"); + expect(caught).toBe(rejection); + expect(blockedExecutions).toBe(1); + expect(laterExecutions).toBe(1); + expect(backend.appendCount).toBe(2); + expect( + backend + .snapshot() + .map((event) => + event.type === "yield" ? `yield:${event.description.name}` : `close:${event.coroutineId}`, + ), + ).toEqual(["yield:later", "close:root"]); + }); +}); From 9165cf939c48cef6f14348158b5acd0f2aca7ed0 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:51:02 -0400 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=90=9B=20classify=20backing=20failure?= =?UTF-8?q?s=20by=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture.md | 6 +- packages/durable-streams/durability.ts | 2 +- .../specs/effection-integration.md | 8 ++- .../specs/protocol-specification.md | 9 ++- .../durable-streams/tests/fail-stop.test.ts | 63 +++++++++++++++++++ 5 files changed, 79 insertions(+), 9 deletions(-) diff --git a/architecture.md b/architecture.md index 5aee4698..cbea5bb3 100644 --- a/architecture.md +++ b/architecture.md @@ -633,8 +633,10 @@ unpersisted success, and never triggers a compensating `Close`. The first durability failure is shared by the root and every durable child. From that point, durable entry fails with that exact error before replay or execution, and the ordered append boundary rechecks it immediately before storage. Work -already executing may finish, but its pending append and every later append are -fenced. A pre-persistence policy rejection remains the policy's ordinary +already executing may finish. Concurrent work that began earlier cannot be +undone, but its not-yet-started append is fenced. An unmarked backing-stream +rejection is a persistence failure regardless of the error class the adapter +throws. A pre-persistence policy rejection remains the policy's ordinary document failure; the guarded stream marks that boundary before the backing append and does not activate the fail-stop state. diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index 3630ba02..a4745656 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -142,7 +142,7 @@ export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): O } throw rejection; } - const active = activeDurabilityFailure(ctx, error); + const active = activeDurabilityFailure(ctx); if (active) { throw active; } diff --git a/packages/durable-streams/specs/effection-integration.md b/packages/durable-streams/specs/effection-integration.md index 0022cee8..3ed7f1e0 100644 --- a/packages/durable-streams/specs/effection-integration.md +++ b/packages/durable-streams/specs/effection-integration.md @@ -464,9 +464,11 @@ terminal handling therefore cannot reinterpret it as a workflow failure or append a compensating `Close(err)`. Durable entry checks that shared state before replay matching or executor startup. Appends pass through a shared FIFO boundary and check it again immediately before calling the stream adapter, so -an append queued behind the failure never reaches storage. An executor that -started concurrently may complete, but it cannot persist its result after the -failure becomes active. +an append queued behind the failure never reaches storage. Concurrent work that +began earlier cannot be undone, but its not-yet-started append is fenced. +Every unmarked stream-adapter rejection is wrapped as persistence failure +regardless of its error class. Only a rejection marked by the pre-persistence +gate is unwrapped as a policy outcome. **Transparency (§4.3).** During replay, `resolve()` is called synchronously with the stored result (converted via `protocolToEffection()`). The reducer diff --git a/packages/durable-streams/specs/protocol-specification.md b/packages/durable-streams/specs/protocol-specification.md index 0501e9c6..8892010a 100644 --- a/packages/durable-streams/specs/protocol-specification.md +++ b/packages/durable-streams/specs/protocol-specification.md @@ -464,14 +464,17 @@ consumer with an unjournaled success or append a compensating `Close(err)`. A failed terminal `Close` append follows the same rule. A pre-persistence gate rejection is distinct: the rejected event does not reach the backend, and the gate's ordinary failure may be recorded by a separately admitted `Close(err)`. +This distinction is source-based: every unmarked backing-stream rejection is a +persistence failure regardless of its error class, while the marked gate +rejection is unwrapped as the policy outcome. The first durability failure makes the root and all of its child coroutines fail-stop. Every later durable-effect entry MUST raise that exact error before replay matching or live execution. Every ordered append MUST recheck the shared failure immediately before invoking storage; an append waiting behind the -operation that failed MUST NOT reach the stream adapter. Concurrent executors -that started before the failure may finish, but their pending appends are -fenced. The first error and, for `DurablePersistenceError`, its adapter cause +operation that failed MUST NOT reach the stream adapter. Concurrent work that +began earlier cannot be undone, but its not-yet-started append is fenced. The +first error and, for `DurablePersistenceError`, its adapter cause retain their identity even when workflow code catches the error. An ordinary pre-persistence policy rejection does not activate fail-stop state. diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts index f1a61749..cf8c0fb1 100644 --- a/packages/durable-streams/tests/fail-stop.test.ts +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -2,12 +2,15 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { sleep, withResolvers, type Operation } from "effection"; import { + ContinuePastCloseDivergenceError, DivergenceError, type DurableEvent, DurablePersistenceError, type DurableStream, InMemoryStream, ReplayGuard, + StaleInputError, + TerminalDivergenceError, type Workflow, durableAction, durableAll, @@ -80,6 +83,66 @@ class BlockingFailureStream implements DurableStream { } describe("durable fail-stop boundary", () => { + it("classifies unmarked adapter failures by source", function* () { + const description = { type: "call", name: "adapter-error" }; + const adapterFailures = [ + new StaleInputError("adapter reported stale input"), + new DivergenceError("root", 0, description, description), + new TerminalDivergenceError("root", 0, 1), + new ContinuePastCloseDivergenceError("root", 0), + new DurablePersistenceError("yield", new Error("nested persistence failure")), + ]; + + for (const adapterFailure of adapterFailures) { + const stream = new FailOnceStream(adapterFailure); + let firstExecutions = 0; + let laterExecutions = 0; + let firstCaught: unknown; + let laterCaught: unknown; + let failure: unknown; + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("adapter-error", () => { + firstExecutions++; + return Promise.resolve("completed"); + }); + } catch (error) { + firstCaught = error; + } + try { + yield* durableCall("later", () => { + laterExecutions++; + return Promise.resolve("must-not-run"); + }); + } catch (error) { + laterCaught = error; + } + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(firstExecutions).toBe(1); + expect(laterExecutions).toBe(0); + expect(stream.appendAttempts).toBe(1); + expect(stream.snapshot()).toEqual([]); + expect(firstCaught).toBe(failure); + expect(laterCaught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure).not.toBe(adapterFailure); + expect(failure.cause).toBe(adapterFailure); + } + }); + it("fences a later callback executor after a caught persistence failure", function* () { const adapterFailure = new Error("first append failed"); const stream = new FailOnceStream(adapterFailure); From 85ef143ea9e26864642265636751b2adf3ed2d62 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:21:36 -0400 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=90=9B=20preserve=20marked=20policy?= =?UTF-8?q?=20failures=20by=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/durable-streams/durability.ts | 2 +- .../durable-streams/tests/fail-stop.test.ts | 73 ++++++++++++++++--- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index a4745656..e59649be 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -136,7 +136,7 @@ export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): O } catch (error) { if (isDurableEventRejection(error)) { const rejection = unwrapDurableEventRejection(error); - const failure = activeDurabilityFailure(ctx, rejection); + const failure = activeDurabilityFailure(ctx); if (failure) { throw failure; } diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts index cf8c0fb1..afc7c2f0 100644 --- a/packages/durable-streams/tests/fail-stop.test.ts +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -82,18 +82,20 @@ class BlockingFailureStream implements DurableStream { } } +function durabilityClassErrors(): Error[] { + const description = { type: "call", name: "classified-error" }; + return [ + new StaleInputError("classified stale input"), + new DivergenceError("root", 0, description, description), + new TerminalDivergenceError("root", 0, 1), + new ContinuePastCloseDivergenceError("root", 0), + new DurablePersistenceError("yield", new Error("classified persistence failure")), + ]; +} + describe("durable fail-stop boundary", () => { it("classifies unmarked adapter failures by source", function* () { - const description = { type: "call", name: "adapter-error" }; - const adapterFailures = [ - new StaleInputError("adapter reported stale input"), - new DivergenceError("root", 0, description, description), - new TerminalDivergenceError("root", 0, 1), - new ContinuePastCloseDivergenceError("root", 0), - new DurablePersistenceError("yield", new Error("nested persistence failure")), - ]; - - for (const adapterFailure of adapterFailures) { + for (const adapterFailure of durabilityClassErrors()) { const stream = new FailOnceStream(adapterFailure); let firstExecutions = 0; let laterExecutions = 0; @@ -143,6 +145,57 @@ describe("durable fail-stop boundary", () => { } }); + it("keeps marked policy failures non-poisoning regardless of class", function* () { + for (const policyFailure of durabilityClassErrors()) { + const backend = new InMemoryStream(); + let blockedExecutions = 0; + let laterExecutions = 0; + let caught: unknown; + const stream = guardDurableStream( + backend, + // deno-lint-ignore require-yield + function* (event) { + if (event.type === "yield" && event.description.name === "blocked") { + throw policyFailure; + } + }, + ); + + const result = yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("blocked", () => { + blockedExecutions++; + return Promise.resolve("blocked"); + }); + } catch (error) { + caught = error; + } + return yield* durableCall("later", () => { + laterExecutions++; + return Promise.resolve("completed"); + }); + }, + { stream }, + ); + + expect(result).toBe("completed"); + expect(caught).toBe(policyFailure); + expect(blockedExecutions).toBe(1); + expect(laterExecutions).toBe(1); + expect(backend.appendCount).toBe(2); + expect( + backend + .snapshot() + .map((event) => + event.type === "yield" + ? `yield:${event.description.name}` + : `close:${event.coroutineId}`, + ), + ).toEqual(["yield:later", "close:root"]); + } + }); + it("fences a later callback executor after a caught persistence failure", function* () { const adapterFailure = new Error("first append failed"); const stream = new FailOnceStream(adapterFailure); From 0c0fd8a30339012590eedb5d0c4c7631f73d77d9 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:40:46 -0400 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=90=9B=20scope=20policy=20rejection?= =?UTF-8?q?=20to=20append=20occurrence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/durable-streams/durability.ts | 17 ++++-- packages/durable-streams/guard.ts | 50 +++++----------- .../durable-streams/tests/fail-stop.test.ts | 58 +++++++++++++++++++ 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index e59649be..47d81f96 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -8,7 +8,7 @@ import { StaleInputError, TerminalDivergenceError, } from "./errors.ts"; -import { isDurableEventRejection, unwrapDurableEventRejection } from "./guard.ts"; +import { withDurableEventRejectionObserver } from "./guard.ts"; import type { DurableEvent } from "./types.ts"; interface AppendTurn { @@ -131,16 +131,23 @@ export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): O throw admitted; } + let policyRejection: unknown; + let policyRejected = false; try { - yield* ctx.stream.append(event); + yield* withDurableEventRejectionObserver( + (error) => { + policyRejection = error; + policyRejected = true; + }, + () => ctx.stream.append(event), + ); } catch (error) { - if (isDurableEventRejection(error)) { - const rejection = unwrapDurableEventRejection(error); + if (policyRejected && Object.is(error, policyRejection)) { const failure = activeDurabilityFailure(ctx); if (failure) { throw failure; } - throw rejection; + throw error; } const active = activeDurabilityFailure(ctx); if (active) { diff --git a/packages/durable-streams/guard.ts b/packages/durable-streams/guard.ts index 8160f442..af563055 100644 --- a/packages/durable-streams/guard.ts +++ b/packages/durable-streams/guard.ts @@ -14,47 +14,21 @@ * produces at most one journal event. */ -import type { Operation } from "effection"; +import { createContext, type Operation } from "effection"; import type { DurableStream } from "./stream.ts"; import type { DurableEvent } from "./types.ts"; -const EVENT_REJECTION = Symbol.for("@effectionx/durable-streams/event-rejection"); +type DurableEventRejectionObserver = (error: unknown) => void; -class WrappedDurableEventRejection extends Error { - constructor(readonly rejection: unknown) { - super(rejection instanceof Error ? rejection.message : String(rejection), { - cause: rejection, - }); - } -} - -function markEventRejection(error: unknown): Error { - const rejection = error instanceof Error ? error : new Error(String(error)); - if ( - Reflect.defineProperty(rejection, EVENT_REJECTION, { - value: true, - configurable: false, - enumerable: false, - writable: false, - }) - ) { - return rejection; - } - return new WrappedDurableEventRejection(error); -} - -export function isDurableEventRejection(error: unknown): boolean { - if (error instanceof WrappedDurableEventRejection) { - return true; - } - if ((typeof error !== "object" || error === null) && typeof error !== "function") { - return false; - } - return Reflect.get(error, EVENT_REJECTION) === true; -} +const EventRejectionObserver = createContext( + "@effectionx/durable-streams/event-rejection-observer", +); -export function unwrapDurableEventRejection(error: unknown): unknown { - return error instanceof WrappedDurableEventRejection ? error.rejection : error; +export function withDurableEventRejectionObserver( + observer: DurableEventRejectionObserver, + operation: () => Operation, +): Operation { + return EventRejectionObserver.with(observer, operation); } /** @@ -92,7 +66,9 @@ export function guardDurableStream(stream: DurableStream, gate: DurableEventGate try { yield* gate(structuredClone(event)); } catch (error) { - throw markEventRejection(error); + const observer = yield* EventRejectionObserver.get(); + observer?.(error); + throw error; } yield* stream.append(event); }, diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts index afc7c2f0..af3bf92e 100644 --- a/packages/durable-streams/tests/fail-stop.test.ts +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -196,6 +196,64 @@ describe("durable fail-stop boundary", () => { } }); + it("classifies a reused policy error by each append occurrence", function* () { + const sharedFailure = new Error("reused policy and adapter failure"); + const backend = new FailOnceStream(sharedFailure); + let blockedExecutions = 0; + let laterExecutions = 0; + let policyCaught: unknown; + let persistenceCaught: unknown; + let failure: unknown; + const stream = guardDurableStream( + backend, + // deno-lint-ignore require-yield + function* (event) { + if (event.type === "yield" && event.description.name === "blocked") { + throw sharedFailure; + } + }, + ); + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("blocked", () => { + blockedExecutions++; + return Promise.resolve("blocked"); + }); + } catch (error) { + policyCaught = error; + } + try { + yield* durableCall("later", () => { + laterExecutions++; + return Promise.resolve("completed"); + }); + } catch (error) { + persistenceCaught = error; + } + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(blockedExecutions).toBe(1); + expect(laterExecutions).toBe(1); + expect(policyCaught).toBe(sharedFailure); + expect(persistenceCaught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(sharedFailure); + expect(backend.appendAttempts).toBe(1); + expect(backend.snapshot()).toEqual([]); + }); + it("fences a later callback executor after a caught persistence failure", function* () { const adapterFailure = new Error("first append failed"); const stream = new FailOnceStream(adapterFailure); From b09ba719b3c24f62ad83e8a9d1779111592e19c0 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:50:55 -0400 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=90=9B=20share=20append=20rejection?= =?UTF-8?q?=20occurrences=20structurally?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/durable-streams/durability.ts | 17 ++- packages/durable-streams/guard.ts | 23 ++-- .../durable-streams/tests/fail-stop.test.ts | 121 ++++++++++-------- 3 files changed, 90 insertions(+), 71 deletions(-) diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index 47d81f96..ad8f7a3e 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -8,7 +8,10 @@ import { StaleInputError, TerminalDivergenceError, } from "./errors.ts"; -import { withDurableEventRejectionObserver } from "./guard.ts"; +import { + type DurableEventRejectionOccurrence, + withDurableEventRejectionOccurrence, +} from "./guard.ts"; import type { DurableEvent } from "./types.ts"; interface AppendTurn { @@ -131,18 +134,14 @@ export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): O throw admitted; } - let policyRejection: unknown; - let policyRejected = false; + const occurrence: DurableEventRejectionOccurrence = { rejected: false }; try { - yield* withDurableEventRejectionObserver( - (error) => { - policyRejection = error; - policyRejected = true; - }, + yield* withDurableEventRejectionOccurrence( + occurrence, () => ctx.stream.append(event), ); } catch (error) { - if (policyRejected && Object.is(error, policyRejection)) { + if (occurrence.rejected && Object.is(error, occurrence.error)) { const failure = activeDurabilityFailure(ctx); if (failure) { throw failure; diff --git a/packages/durable-streams/guard.ts b/packages/durable-streams/guard.ts index af563055..e5bef107 100644 --- a/packages/durable-streams/guard.ts +++ b/packages/durable-streams/guard.ts @@ -18,17 +18,21 @@ import { createContext, type Operation } from "effection"; import type { DurableStream } from "./stream.ts"; import type { DurableEvent } from "./types.ts"; -type DurableEventRejectionObserver = (error: unknown) => void; +export interface DurableEventRejectionOccurrence { + rejected: boolean; + error?: unknown; +} -const EventRejectionObserver = createContext( - "@effectionx/durable-streams/event-rejection-observer", +const EventRejectionOccurrence = createContext( + "effectionx.durable-streams.event-rejection-occurrence", + undefined, ); -export function withDurableEventRejectionObserver( - observer: DurableEventRejectionObserver, +export function withDurableEventRejectionOccurrence( + occurrence: DurableEventRejectionOccurrence, operation: () => Operation, ): Operation { - return EventRejectionObserver.with(observer, operation); + return EventRejectionOccurrence.with(occurrence, operation); } /** @@ -66,8 +70,11 @@ export function guardDurableStream(stream: DurableStream, gate: DurableEventGate try { yield* gate(structuredClone(event)); } catch (error) { - const observer = yield* EventRejectionObserver.get(); - observer?.(error); + const occurrence = yield* EventRejectionOccurrence.get(); + if (occurrence !== undefined) { + occurrence.rejected = true; + occurrence.error = error; + } throw error; } yield* stream.append(event); diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts index af3bf92e..5b5f8f9c 100644 --- a/packages/durable-streams/tests/fail-stop.test.ts +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { sleep, withResolvers, type Operation } from "effection"; +import { sleep, until, withResolvers, type Operation } from "effection"; import { ContinuePastCloseDivergenceError, DivergenceError, @@ -93,6 +93,66 @@ function durabilityClassErrors(): Error[] { ]; } +function* assertReusedPolicyErrorClassification( + guard: typeof guardDurableStream, +): Operation { + const sharedFailure = new Error("reused policy and adapter failure"); + const backend = new FailOnceStream(sharedFailure); + let blockedExecutions = 0; + let laterExecutions = 0; + let policyCaught: unknown; + let persistenceCaught: unknown; + let failure: unknown; + const stream = guard( + backend, + // deno-lint-ignore require-yield + function* (event) { + if (event.type === "yield" && event.description.name === "blocked") { + throw sharedFailure; + } + }, + ); + + try { + yield* durableRun( + function* (): Workflow { + try { + yield* durableCall("blocked", () => { + blockedExecutions++; + return Promise.resolve("blocked"); + }); + } catch (error) { + policyCaught = error; + } + try { + yield* durableCall("later", () => { + laterExecutions++; + return Promise.resolve("completed"); + }); + } catch (error) { + persistenceCaught = error; + } + return "must-not-close"; + }, + { stream }, + ); + } catch (error) { + failure = error; + } + + expect(blockedExecutions).toBe(1); + expect(laterExecutions).toBe(1); + expect(policyCaught).toBe(sharedFailure); + expect(persistenceCaught).toBe(failure); + expect(failure).toBeInstanceOf(DurablePersistenceError); + if (!(failure instanceof DurablePersistenceError)) { + throw new Error("expected durable persistence failure"); + } + expect(failure.cause).toBe(sharedFailure); + expect(backend.appendAttempts).toBe(1); + expect(backend.snapshot()).toEqual([]); +} + describe("durable fail-stop boundary", () => { it("classifies unmarked adapter failures by source", function* () { for (const adapterFailure of durabilityClassErrors()) { @@ -197,61 +257,14 @@ describe("durable fail-stop boundary", () => { }); it("classifies a reused policy error by each append occurrence", function* () { - const sharedFailure = new Error("reused policy and adapter failure"); - const backend = new FailOnceStream(sharedFailure); - let blockedExecutions = 0; - let laterExecutions = 0; - let policyCaught: unknown; - let persistenceCaught: unknown; - let failure: unknown; - const stream = guardDurableStream( - backend, - // deno-lint-ignore require-yield - function* (event) { - if (event.type === "yield" && event.description.name === "blocked") { - throw sharedFailure; - } - }, - ); + yield* assertReusedPolicyErrorClassification(guardDurableStream); + }); - try { - yield* durableRun( - function* (): Workflow { - try { - yield* durableCall("blocked", () => { - blockedExecutions++; - return Promise.resolve("blocked"); - }); - } catch (error) { - policyCaught = error; - } - try { - yield* durableCall("later", () => { - laterExecutions++; - return Promise.resolve("completed"); - }); - } catch (error) { - persistenceCaught = error; - } - return "must-not-close"; - }, - { stream }, - ); - } catch (error) { - failure = error; - } + it("shares append occurrence classification across loaded copies", function* () { + const loadedCopy = yield* until(import("../guard.ts?loaded-copy=fail-stop")); - expect(blockedExecutions).toBe(1); - expect(laterExecutions).toBe(1); - expect(policyCaught).toBe(sharedFailure); - expect(persistenceCaught).toBe(failure); - expect(failure).toBeInstanceOf(DurablePersistenceError); - if (!(failure instanceof DurablePersistenceError)) { - throw new Error("expected durable persistence failure"); - } - expect(failure.cause).toBe(sharedFailure); - expect(backend.appendAttempts).toBe(1); - expect(backend.snapshot()).toEqual([]); + expect(loadedCopy.guardDurableStream).not.toBe(guardDurableStream); + yield* assertReusedPolicyErrorClassification(loadedCopy.guardDurableStream); }); it("fences a later callback executor after a caught persistence failure", function* () { From e92be05b153570ec8e00abd3bf335cf18afbd940 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:54:30 -0400 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=90=9B=20keep=20cross-copy=20regressi?= =?UTF-8?q?on=20portable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/durable-streams/durability.ts | 5 +---- packages/durable-streams/tests/fail-stop.test.ts | 9 +++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/durable-streams/durability.ts b/packages/durable-streams/durability.ts index ad8f7a3e..a6467ce4 100644 --- a/packages/durable-streams/durability.ts +++ b/packages/durable-streams/durability.ts @@ -136,10 +136,7 @@ export function* appendDurableEvent(ctx: DurableContext, event: DurableEvent): O const occurrence: DurableEventRejectionOccurrence = { rejected: false }; try { - yield* withDurableEventRejectionOccurrence( - occurrence, - () => ctx.stream.append(event), - ); + yield* withDurableEventRejectionOccurrence(occurrence, () => ctx.stream.append(event)); } catch (error) { if (occurrence.rejected && Object.is(error, occurrence.error)) { const failure = activeDurabilityFailure(ctx); diff --git a/packages/durable-streams/tests/fail-stop.test.ts b/packages/durable-streams/tests/fail-stop.test.ts index 5b5f8f9c..64af4329 100644 --- a/packages/durable-streams/tests/fail-stop.test.ts +++ b/packages/durable-streams/tests/fail-stop.test.ts @@ -93,9 +93,7 @@ function durabilityClassErrors(): Error[] { ]; } -function* assertReusedPolicyErrorClassification( - guard: typeof guardDurableStream, -): Operation { +function* assertReusedPolicyErrorClassification(guard: typeof guardDurableStream): Operation { const sharedFailure = new Error("reused policy and adapter failure"); const backend = new FailOnceStream(sharedFailure); let blockedExecutions = 0; @@ -261,7 +259,10 @@ describe("durable fail-stop boundary", () => { }); it("shares append occurrence classification across loaded copies", function* () { - const loadedCopy = yield* until(import("../guard.ts?loaded-copy=fail-stop")); + const loadedCopySpecifier = "../guard.ts" + "?loaded-copy=fail-stop"; + const loadGuardCopy: () => Promise = () => + import(loadedCopySpecifier); + const loadedCopy = yield* until(loadGuardCopy()); expect(loadedCopy.guardDurableStream).not.toBe(guardDurableStream); yield* assertReusedPolicyErrorClassification(loadedCopy.guardDurableStream);