(CODUX-216) Make publications and execution durable

docs-web/architecture/node-flow-durable-execution.md

@@ -0,0 +1,7 @@
+# Node Flow Durable Execution
+
+Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run.
+
+Runs are durably queued and leased with bounded global and project concurrency. Node attempts retain attempt number, executor and invocation identity, artifact digest, redacted payloads, credential ids, failure class, and retry decision. Retryable failures use bounded exponential backoff and jitter, while cancellation and timeout signals propagate to provider and HTTP work.
+
+On restart, expired pre-invocation work is safely requeued. Work with an external invocation and an unknown outcome moves to `attention_required` and is never silently replayed. Credential values are resolved only for the active node and are not retained in run history or diagnostics.

docs-web/architecture/node-flows.md

@@ -14,3 +14,5 @@ Node flows are project-owned, versioned Graph v2 workflows.
 | `output` | Selects the result. |
 
 These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2.
+
+Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](./node-flow-durable-execution.md).

docs-web/content/docs/architecture-node-flow-durable-execution.mdx

@@ -0,0 +1,7 @@
+# Node Flow Durable Execution
+
+Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run.
+
+Runs are durably queued and leased with bounded global and project concurrency. Node attempts retain attempt number, executor and invocation identity, artifact digest, redacted payloads, credential ids, failure class, and retry decision. Retryable failures use bounded exponential backoff and jitter, while cancellation and timeout signals propagate to provider and HTTP work.
+
+On restart, expired pre-invocation work is safely requeued. Work with an external invocation and an unknown outcome moves to `attention_required` and is never silently replayed. Credential values are resolved only for the active node and are not retained in run history or diagnostics.

docs-web/content/docs/architecture-node-flows.mdx

@@ -14,3 +14,5 @@ Node flows are project-owned, versioned Graph v2 workflows.
 | `output` | Selects the result. |
 
 These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2.
+
+Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](/docs/architecture-node-flow-durable-execution).

docs-web/content/docs/registry.ts

@@ -108,6 +108,7 @@ export type DocsSlug =
   | 'user-dashboard-custom-dashboards'
   | 'architecture-custom-dashboard-foundation'
   | 'architecture-managed-container-runtime'
+  | 'architecture-node-flow-durable-execution'
   | 'architecture-node-flow-foundation'
   | 'architecture-node-flows'
   | 'architecture-speech-input'
@@ -858,6 +859,13 @@ export const docsRegistry: Record<DocsSlug, DocsRegistryEntry> = {
     title: "Managed Container Runtime",
     description: "The managed container runtime removes first-invocation Docker builds while keeping provider binaries local to each user's Docker host.",
   },
+  'architecture-node-flow-durable-execution': {
+    id: 'architecture-node-flow-durable-execution',
+    path: '/docs/architecture-node-flow-durable-execution',
+    section: 'Architecture',
+    title: "Node Flow Durable Execution",
+    description: "Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run.",
+  },
   'architecture-node-flow-foundation': {
     id: 'architecture-node-flow-foundation',
     path: '/docs/architecture-node-flow-foundation',
@@ -1001,6 +1009,7 @@ export const orderedDocs: DocsRegistryEntry[] = [
   docsRegistry['user-dashboard-custom-dashboards'],
   docsRegistry['architecture-custom-dashboard-foundation'],
   docsRegistry['architecture-managed-container-runtime'],
+  docsRegistry['architecture-node-flow-durable-execution'],
   docsRegistry['architecture-node-flow-foundation'],
   docsRegistry['architecture-node-flows'],
   docsRegistry['architecture-speech-input'],

docs-web/content/docs/user-dashboard-scheduler.mdx

@@ -22,11 +22,13 @@ Each scheduler entry has a **target** — the thing that runs when it fires:
 | **Message** | Posts a project message (for example, a recurring planning or status prompt). |
 | **Memory remediation** | Runs the long-term memory cleanup workflow on a schedule. |
 
-Node-flow entries store `nodeFlowTarget = { flowId, input?, flowVersion? }` inside the existing
+Node-flow entries store `nodeFlowTarget = { flowId, input?, versionSelection }` inside the existing
 target JSON payload, validate that the flow belongs to the selected project, and run through the
 node-flow runtime with scheduler trigger metadata when due. Blank dashboard input is omitted, and
 supplied input must be a JSON object.
 
+Choose a pinned published version when every occurrence must execute the same immutable snapshot, or latest published when each occurrence should pick up the newest publication. A legacy `flowVersion` is treated as a pinned version and affects execution, not just audit metadata.
+
 The backend scheduler contract also supports agent-created wakeups.
 Agent wakeups are stored in the target JSON payload with `origin` and `source` set to
 `agent_scheduler`, plus `createdByAgentId` when the creating agent provides it. Agent wakeups post

docs-web/routes/docs.architecture-node-flow-durable-execution.lazy.tsx

@@ -0,0 +1,11 @@
+import { createLazyFileRoute } from '@tanstack/react-router'
+import ArchitectureNodeFlowDurableExecutionContent from '../content/docs/architecture-node-flow-durable-execution.mdx'
+import { DocsPage } from '../components/docs/DocsPage'
+
+export const Route = createLazyFileRoute('/docs/architecture-node-flow-durable-execution')({
+  component: () => (
+    <DocsPage id="architecture-node-flow-durable-execution">
+      <ArchitectureNodeFlowDurableExecutionContent />
+    </DocsPage>
+  )
+})

docs-web/user/dashboard/scheduler.md

@@ -22,11 +22,13 @@ Each scheduler entry has a **target** — the thing that runs when it fires:
 | **Message** | Posts a project message (for example, a recurring planning or status prompt). |
 | **Memory remediation** | Runs the long-term memory cleanup workflow on a schedule. |
 
-Node-flow entries store `nodeFlowTarget = { flowId, input?, flowVersion? }` inside the existing
+Node-flow entries store `nodeFlowTarget = { flowId, input?, versionSelection }` inside the existing
 target JSON payload, validate that the flow belongs to the selected project, and run through the
 node-flow runtime with scheduler trigger metadata when due. Blank dashboard input is omitted, and
 supplied input must be a JSON object.
 
+Choose a pinned published version when every occurrence must execute the same immutable snapshot, or latest published when each occurrence should pick up the newest publication. A legacy `flowVersion` is treated as a pinned version and affects execution, not just audit metadata.
+
 The backend scheduler contract also supports agent-created wakeups.
 Agent wakeups are stored in the target JSON payload with `origin` and `source` set to
 `agent_scheduler`, plus `createdByAgentId` when the creating agent provides it. Agent wakeups post

docs/SUMMARY.md

@@ -39,6 +39,7 @@
 - [Agent Knowledge Base](./architecture/agent-knowledge-base.md)
 - [Node Flow Foundation](./architecture/node-flow-foundation.md)
 - [Node Flows](./architecture/node-flows.md)
+- [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md)
 - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md)
 - [Memory Claims and Evidence](./architecture/memory-claims.md)
 - [Speech Input Architecture](./architecture/speech-input.md)

docs/architecture/node-flow-durable-execution.md

@@ -0,0 +1,15 @@
+# Node Flow Durable Execution
+
+Node-flow execution is publication based. Saving a flow appends an immutable version and publication containing the normalized graph and an immutable execution-policy snapshot. Manual, MCP, and scheduled callers select either `{ mode: "pinned", version: N }` or `{ mode: "latest_published" }`; the runtime never executes the mutable `node_flows` graph.
+
+## Durable lifecycle
+
+Runs move through `queued`, `running`, `approval_waiting`, `retry_waiting`, `attention_required`, and terminal `succeeded`, `failed`, or `cancelled` states. A queue claim assigns an executor, lease expiry, and heartbeat. Global and per-project limits bound active claims. Cancellation and node timeouts propagate through `AbortSignal`.
+
+Every node execution creates a numbered attempt with executor identity, optional execution invocation id, SHA-256 output digest, redacted input/output, credential ids, failure classification, and retry decision. Retryable timeout, quota, and transient failures use the publication policy's bounded exponential backoff and jitter. Credential values are resolved only at the node boundary and are never written to run, attempt, invocation, or diagnostic records.
+
+## Recovery contract
+
+Startup recovery scans queued and waiting work plus running work with expired leases. A pre-invocation attempt can be requeued safely without inserting a duplicate attempt. An expired attempt with an invocation id has an unknown externally observable outcome and moves to `attention_required`; Code UX does not silently replay it. Approval- and retry-waiting runs retain their durable state until their prerequisite becomes actionable.
+
+The relevant tables are `node_flow_publications`, `node_flow_runs`, `node_flow_node_runs`, and `node_flow_node_attempts`. Attempt history is available at `GET /api/node-flow-runs/:runId/attempts` and contains only redacted payloads and credential identifiers.

docs/architecture/node-flows.md

@@ -11,10 +11,12 @@ Node-flow persistence is owned by `NodeFlowRepository` and stored in SQLite:
 | Table | Purpose |
 | --- | --- |
 | `node_flows` | Current project-scoped flow record: id, project id, title, description, normalized `graph_json`, current version, and timestamps. |
-| `node_flow_versions` | Immutable snapshots written on create and every update. Versions keep the graph saved at that point, even though current runtime execution uses the latest flow record. |
+| `node_flow_versions` | Immutable edit snapshots written on create and every update. |
+| `node_flow_publications` | Immutable executable graph and execution-policy snapshots selected by pinned or latest-published runs. |
 | `node_flow_agent_skills` | Agent attachment table keyed by flow and agent preset. It stores the skill display name and description used when exposing the flow as a repeatable agent capability. |
 | `node_flow_runs` | Flow run records with status, version, trigger type, redacted trigger payload, redacted input/output, error message, timestamps, and optional `execution_invocation_id`. |
 | `node_flow_node_runs` | Per-node run records with status, node id, redacted input/output, error message, timestamps, and optional `execution_invocation_id`. |
+| `node_flow_node_attempts` | Numbered attempts with executor/invocation identity, artifact digest, credential ids, redacted payloads, failure class, and retry decision. |
 
 All graphs, widget schemas, run inputs, outputs, and trigger payloads are stored as JSON text and hydrated into typed contracts at the repository boundary. Flow, version, run, and attachment records belong to a project. Agent attachment operations verify that the target agent preset belongs to the same project as the flow.
 
@@ -37,7 +39,7 @@ Dashboard-only editable canvas state lives in `dashboard/src/v2/lib/nodes-canvas
 
 ## Runtime
 
-`NodeFlowRuntimeService.runFlow(projectId, flowId, input, options)` revalidates the saved graph, checks project ownership, and then executes nodes in the validator's topological order.
+`NodeFlowRuntimeService.runFlow(projectId, flowId, input, options)` resolves an explicit pinned or latest-published snapshot, revalidates that immutable graph, claims a durable lease, and executes nodes in topological order. See [Node Flow Durable Execution](./node-flow-durable-execution.md) for queue, retry, lease, recovery, quota, and redaction guarantees.
 
 Runtime-supported node types are:
 
@@ -75,7 +77,7 @@ Cancellation records cancelled node rows for the current and remaining nodes. At
 
 ## Scheduling
 
-Scheduler entries with `targetType: "node_flow"` persist `nodeFlowTarget = { flowId, input?, flowVersion? }` inside `scheduler_entries.target_json`. Ownership is validated when entries are created or updated and again before due-run execution.
+Scheduler entries with `targetType: "node_flow"` persist an explicit `versionSelection`: pinned schedules continue to execute version N after N+1 is published, while latest-published schedules resolve the newest publication at dispatch time. Legacy `flowVersion` values normalize to pinned selection and are executable semantics, not audit-only metadata. Ownership is validated when entries are created or updated and again before due-run execution.
 
 Due runs call `NodeFlowRuntimeService.runFlow` with `triggerType = "scheduler"` and trigger payload metadata for the scheduler entry id, scheduled occurrence time, target type, and persisted flow version when present. Node-flow schedules advance only when `runFlow` returns a run status of `succeeded`. Returned `failed` or `cancelled` runs mark the scheduler entry `failed` with the run error and still count the attempted occurrence in `lastRunAt` and `runCount`; runtime startup rejections mark failure without creating a false successful schedule run.
 

docs/dashboard/scheduler.md

@@ -78,7 +78,7 @@ The target payload keys are:
 - `chatTarget`: `{ bodyMarkdown, threadId?, title?, connectionId? }`
 - `memoryRemediationTarget`: `{ mode, source? }`
 - `taskTarget`: `{ taskId, provider?, origin: "agent_scheduler", source: "agent_scheduler", createdByAgentId? }`
-- `nodeFlowTarget`: `{ flowId, input?, flowVersion? }`
+- `nodeFlowTarget`: `{ flowId, input?, versionSelection }`; legacy `flowVersion` normalizes to pinned selection
 - `agentWakeupTarget`: `{ bodyMarkdown, threadId?, title?, connectionId?, origin: "agent_scheduler", source: "agent_scheduler", createdByAgentId? }`
 
 `node_flow` entries keep their flow id and optional input in `target_json`; ownership is checked when entries are created or updated and again before due-run execution. The persisted `flowVersion` is target metadata and is passed in scheduler trigger payloads for auditability; the current runtime executes through the latest node-flow runtime API. Due-run handling treats the returned node-flow run status as authoritative: only `succeeded` advances the schedule as successful, while `failed` and `cancelled` mark the scheduler entry `failed`, persist the run error, and record the attempted occurrence in `lastRunAt` and `runCount`. `agent_wakeup` and `task` entries always normalize `origin` and `source` to `agent_scheduler` in `target_json`. When the creator supplies `createdByAgentId`, it is preserved with the target payload for later authorization, audit, and notification work. Existing sprint, quicksprint, chat, memory remediation, recurrence, pause/resume, and `after_sprint_end` anchor rows continue to hydrate from the same JSON payload without a schema migration.
@@ -156,7 +156,7 @@ For sprint targets, failures from either automatic planning or direct orchestrat
 
 ### Node-Flow Schedules
 
-Node-flow schedules use `targetType: "node_flow"` and `nodeFlowTarget = { flowId, input?, flowVersion? }`.
+Node-flow schedules use `targetType: "node_flow"` and `nodeFlowTarget = { flowId, input?, versionSelection }`. A pinned selection always executes that published graph and policy snapshot after newer versions are published; `latest_published` resolves the newest publication per occurrence.
 
 Behavior:
 

docs/index.md

@@ -66,6 +66,7 @@ Use this page as the main entrypoint.
 28. [Agent Knowledge Base](./architecture/agent-knowledge-base.md)
 29. [Node Flow Foundation](./architecture/node-flow-foundation.md)
 30. [Node Flows](./architecture/node-flows.md)
+31. [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md)
 31. [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md)
 32. [Memory Claims and Evidence](./architecture/memory-claims.md)
 33. [Speech Input Architecture](./architecture/speech-input.md)
@@ -158,6 +159,7 @@ Use this page as the main entrypoint.
 - [Agent Knowledge Base](./architecture/agent-knowledge-base.md)
 - [Node Flow Foundation](./architecture/node-flow-foundation.md)
 - [Node Flows](./architecture/node-flows.md)
+- [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md)
 - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md)
 - [Memory Claims and Evidence](./architecture/memory-claims.md)
 - [Speech Input Architecture](./architecture/speech-input.md)

src/app/dependency-factory/dashboard-factory.ts

@@ -31,6 +31,7 @@ import { SpeechSynthesisService } from "../../services/speech-synthesis-service.
 import { SpeechModelManager } from "../../services/speech-model-manager.js";
 import { NodeFlowRuntimeService } from "../../services/node-flow-runtime-service.js";
 import { NodeFlowService } from "../../services/node-flow-service.js";
+import { NodeFlowRecoveryService } from "../../services/node-flows/node-flow-recovery-service.js";
 import { resolveEffectiveDashboardSettings } from "../../services/settings-resolution-service.js";
 
 export interface DashboardDependencies {
@@ -230,6 +231,9 @@ export function createDashboardDependencies(
     credentialBroker: coreDeps.credentialBroker,
     getDashboardSettings: (projectId) => resolveDashboardSettings({ projectId }),
   });
+  if (coreDeps.nodeFlowRepository) {
+    new NodeFlowRecoveryService(coreDeps.nodeFlowRepository).recover();
+  }
   const nodeFlowService = new NodeFlowService(coreDeps.nodeFlowRepository, nodeFlowRuntimeService);
 
   const activityCacheService = new ActivityCacheService(

src/contracts/node-flow-execution-policy-types.ts

@@ -0,0 +1,45 @@
+export type NodeFlowVersionSelection =
+  | { mode: "latest_published" }
+  | { mode: "pinned"; version: number };
+
+export type NodeFlowFailureClassification =
+  | "cancelled"
+  | "timeout"
+  | "quota"
+  | "validation"
+  | "credential"
+  | "transient"
+  | "permanent"
+  | "unknown_side_effect";
+
+export interface NodeFlowRetryPolicySnapshot {
+  maxAttempts: number;
+  backoffMs: number;
+  maxBackoffMs: number;
+  jitterRatio: number;
+  retryableClasses: NodeFlowFailureClassification[];
+}
+
+export interface NodeFlowExecutionPolicySnapshot {
+  maxConcurrentRuns: number;
+  maxConcurrentRunsPerProject: number;
+  leaseDurationMs: number;
+  heartbeatIntervalMs: number;
+  defaultTimeoutMs: number;
+  retry: NodeFlowRetryPolicySnapshot;
+}
+
+export const DEFAULT_NODE_FLOW_EXECUTION_POLICY: Readonly<NodeFlowExecutionPolicySnapshot> = Object.freeze({
+  maxConcurrentRuns: 4,
+  maxConcurrentRunsPerProject: 2,
+  leaseDurationMs: 30_000,
+  heartbeatIntervalMs: 10_000,
+  defaultTimeoutMs: 60_000,
+  retry: Object.freeze({
+    maxAttempts: 1,
+    backoffMs: 500,
+    maxBackoffMs: 30_000,
+    jitterRatio: 0.2,
+    retryableClasses: Object.freeze(["timeout", "quota", "transient"]) as NodeFlowFailureClassification[],
+  }),
+});