From 3e86d54c26e6a40bb8cc644d6494e058afddf170 Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 04:07:12 +0000 Subject: [PATCH 01/25] feat(task T01): implement via codex --- dashboard/src/v2/lib/nodes-agent-surface.ts | 5 +- dashboard/src/v2/lib/nodes-canvas-state.ts | 121 +++++++++++-- docs-web/architecture/node-flow-foundation.md | 7 + docs-web/architecture/node-flows.md | 16 ++ .../architecture-node-flow-foundation.mdx | 7 + .../content/docs/architecture-node-flows.mdx | 16 ++ docs-web/content/docs/registry.ts | 18 ++ .../docs/user-dashboard-node-flows.mdx | 4 + .../docs/user-dashboard-nodes-canvas.mdx | 4 + ...architecture-node-flow-foundation.lazy.tsx | 11 ++ .../docs.architecture-node-flows.lazy.tsx | 11 ++ docs-web/user/dashboard/node-flows.md | 4 + docs-web/user/dashboard/nodes-canvas.md | 4 + docs/architecture/node-flow-foundation.md | 8 + docs/architecture/node-flows.md | 8 + docs/dashboard/node-flows.md | 4 + docs/dashboard/nodes-canvas.md | 4 + src/contracts/node-definition-types.ts | 46 +++++ src/contracts/node-flow-types.ts | 70 ++++++++ .../node-flows/node-definition-registry.ts | 91 ++++++++++ src/domain/node-flows/node-flow-migrators.ts | 110 ++++++++++++ src/domain/node-flows/node-flow-validation.ts | 162 +++++++++++++++++- src/repositories/db/app-db-migrations.ts | 46 +++++ src/repositories/node-flow-repository.ts | 2 + src/services/node-flow-runtime-service.ts | 11 +- .../node-definition-registry.test.ts | 17 ++ .../node-flows/node-flow-migrators.test.ts | 31 ++++ .../node-flows/node-flow-validation.test.ts | 37 +++- .../repositories/node-flow-repository.test.ts | 18 ++ .../services/node-flow-service.test.ts | 2 +- .../dashboard/lib/nodes-canvas-state.test.ts | 15 ++ 31 files changed, 880 insertions(+), 30 deletions(-) create mode 100644 docs-web/architecture/node-flow-foundation.md create mode 100644 docs-web/architecture/node-flows.md create mode 100644 docs-web/content/docs/architecture-node-flow-foundation.mdx create mode 100644 docs-web/content/docs/architecture-node-flows.mdx create mode 100644 docs-web/routes/docs.architecture-node-flow-foundation.lazy.tsx create mode 100644 docs-web/routes/docs.architecture-node-flows.lazy.tsx create mode 100644 src/contracts/node-definition-types.ts create mode 100644 src/domain/node-flows/node-definition-registry.ts create mode 100644 src/domain/node-flows/node-flow-migrators.ts create mode 100644 tests/backend/domain/node-flows/node-definition-registry.test.ts create mode 100644 tests/backend/domain/node-flows/node-flow-migrators.test.ts diff --git a/dashboard/src/v2/lib/nodes-agent-surface.ts b/dashboard/src/v2/lib/nodes-agent-surface.ts index dcf3b0808f..8326f5004c 100644 --- a/dashboard/src/v2/lib/nodes-agent-surface.ts +++ b/dashboard/src/v2/lib/nodes-agent-surface.ts @@ -12,6 +12,7 @@ import type { NodeCanvasValidationIssue, } from "./nodes-canvas-state.js"; import { + deserializeNodeCanvasGraphWithMigration, nodesCanvasReducer, normalizeNodeCanvasGraph, validateNodeCanvasGraph, @@ -278,8 +279,8 @@ const applyParsedCommand = ( return nextGraph; } case "replace_graph": { - const parsed = JSON.parse(command.serializedGraph) as unknown; - return nodesCanvasReducer(graph, { type: "replace_graph", graph: normalizeNodeCanvasGraph(parsed) }); + const migration = deserializeNodeCanvasGraphWithMigration(command.serializedGraph); + return nodesCanvasReducer(graph, { type: "replace_graph", graph: migration.graph }); } } }; diff --git a/dashboard/src/v2/lib/nodes-canvas-state.ts b/dashboard/src/v2/lib/nodes-canvas-state.ts index 2a35928413..0b363bd42f 100644 --- a/dashboard/src/v2/lib/nodes-canvas-state.ts +++ b/dashboard/src/v2/lib/nodes-canvas-state.ts @@ -1,3 +1,5 @@ +import type { NodeFlowGraph, NodeFlowJsonObject, NodeFlowJsonValue, NodeFlowPort } from "../../../../src/contracts/node-flow-types.js"; + export type NodeCanvasNodeKind = "trigger" | "agent" | "task" | "condition" | "output"; export type NodeCanvasPortDirection = "input" | "output"; export type NodeCanvasPortType = "control" | "agent" | "task" | "condition" | "result" | "data"; @@ -66,11 +68,18 @@ export interface NodeCanvasSelectionState { } export interface NodeCanvasGraph { + schemaVersion?: 2; nodes: NodeCanvasNode[]; edges: NodeCanvasEdge[]; selection: NodeCanvasSelectionState; } +export interface NodeCanvasGraphMigrationResult { + graph: NodeCanvasGraph; + migrated: boolean; + legacySnapshot: unknown | null; +} + export type NodeCanvasValidationIssueCode = | "duplicate_node_id" | "missing_edge_source_node" @@ -201,6 +210,7 @@ export const createInitialNodeCanvasGraph = (): NodeCanvasGraph => { ]; return normalizeNodeCanvasGraph({ + schemaVersion: 2, nodes, edges: [ createNodeCanvasEdge("edge-trigger-1-event-agent-1-in", "trigger-1", "event", "agent-1", "in"), @@ -487,18 +497,73 @@ export const validateNodeCanvasGraph = (graph: NodeCanvasGraph): NodeCanvasValid }; export const serializeNodeCanvasGraph = (graph: NodeCanvasGraph): string => ( - JSON.stringify(toStableJson(normalizeNodeCanvasGraph(graph)), null, 2) + JSON.stringify(toStableJson(toCanonicalNodeFlowGraph(graph)), null, 2) ); +export const toCanonicalNodeFlowGraph = (graph: NodeCanvasGraph): NodeFlowGraph => { + const normalized = normalizeNodeCanvasGraph(graph); + return { + schemaVersion: 2, + nodes: normalized.nodes.map((node) => ({ + id: node.id, + type: node.kind, + title: node.label, + description: node.description, + position: node.position, + definition: { type: node.kind, version: 1 }, + ports: [...node.inputPorts, ...node.outputPorts].map(toCanonicalPort), + credentialBindings: [], + policy: {}, + capabilities: [], + sideEffect: "none", + disabled: false, + data: { + canvas: { + config: node.config as unknown as NodeFlowJsonValue, + metadata: node.metadata as unknown as NodeFlowJsonValue, + }, + }, + })), + edges: normalized.edges.map((edge) => ({ + id: edge.id, + fromNodeId: edge.source.nodeId, + toNodeId: edge.target.nodeId, + fromHandle: edge.source.portId, + toHandle: edge.target.portId, + })), + metadata: { canvasSelection: normalized.selection as unknown as NodeFlowJsonObject }, + }; +}; + +const toCanonicalPort = (port: NodeCanvasPort): NodeFlowPort => ({ + id: port.id, + direction: port.direction, + schema: { type: "object", description: port.type }, + required: port.required, +}); + export const deserializeNodeCanvasGraph = (serialized: string): NodeCanvasGraph => { + return deserializeNodeCanvasGraphWithMigration(serialized).graph; +}; + +export const deserializeNodeCanvasGraphWithMigration = (serialized: string): NodeCanvasGraphMigrationResult => { try { const parsed: unknown = JSON.parse(serialized); - return normalizeNodeCanvasGraph(parsed); + return migrateNodeCanvasGraph(parsed); } catch { - return createInitialNodeCanvasGraph(); + return { graph: createInitialNodeCanvasGraph(), migrated: false, legacySnapshot: null }; } }; +export const migrateNodeCanvasGraph = (input: unknown): NodeCanvasGraphMigrationResult => { + const migrated = !isRecord(input) || input.schemaVersion !== 2; + return { + graph: normalizeNodeCanvasGraph(input), + migrated, + legacySnapshot: migrated ? cloneStableValue(input) : null, + }; +}; + export const normalizeNodeCanvasGraph = (input: unknown): NodeCanvasGraph => { if (!isRecord(input)) { return createInitialNodeCanvasGraph(); @@ -517,15 +582,19 @@ export const normalizeNodeCanvasGraph = (input: unknown): NodeCanvasGraph => { : []; const validNodeIds = new Set(parsedNodes.map((node) => node.id)); - const parsedSelection = parseSelection(input.selection, validNodeIds, new Set(parsedEdges.map((edge) => edge.id))); + const canvasSelection = isRecord(input.metadata) ? input.metadata.canvasSelection : undefined; + const parsedSelection = parseSelection(input.selection ?? canvasSelection, validNodeIds, new Set(parsedEdges.map((edge) => edge.id))); return { + schemaVersion: 2, nodes: parsedNodes.sort(compareById), edges: parsedEdges.sort(compareById), selection: parsedSelection, }; }; +const cloneStableValue = (value: unknown): unknown => structuredClone(value); + const createNodeCanvasEdge = ( id: string, sourceNodeId: string, @@ -619,10 +688,11 @@ const normalizeMetadata = (metadata: NodeCanvasNodeMetadata): NodeCanvasNodeMeta }); const parseNode = (value: unknown): NodeCanvasNode | null => { - if (!isRecord(value) || !isString(value.id) || !isNodeKind(value.kind)) { + const kindValue = isRecord(value) ? value.kind ?? value.type : undefined; + if (!isRecord(value) || !isString(value.id) || !isNodeKind(kindValue)) { return null; } - const template = createNodeCanvasNode(value.kind, { id: value.id }); + const template = createNodeCanvasNode(kindValue, { id: value.id }); const position = isRecord(value.position) ? { x: finiteOrDefault(typeof value.position.x === "number" ? value.position.x : undefined, template.position.x), @@ -630,18 +700,25 @@ const parseNode = (value: unknown): NodeCanvasNode | null => { } : template.position; + const canvasData = readCanonicalCanvasData(value.data); + const canonicalPorts = Array.isArray(value.ports) ? value.ports : undefined; return { ...template, - label: isString(value.label) ? value.label : template.label, + label: isString(value.label) ? value.label : isString(value.title) ? value.title : template.label, description: isString(value.description) ? value.description : template.description, position, - inputPorts: parsePorts(value.inputPorts, "input", template.inputPorts), - outputPorts: parsePorts(value.outputPorts, "output", template.outputPorts), - config: parseConfig(value.config, template.config), - metadata: parseMetadata(value.metadata), + inputPorts: parsePorts(value.inputPorts ?? canonicalPorts?.filter((port) => isRecord(port) && port.direction === "input"), "input", template.inputPorts), + outputPorts: parsePorts(value.outputPorts ?? canonicalPorts?.filter((port) => isRecord(port) && port.direction === "output"), "output", template.outputPorts), + config: parseConfig(value.config ?? canvasData?.config, template.config), + metadata: parseMetadata(value.metadata ?? canvasData?.metadata), }; }; +const readCanonicalCanvasData = (value: unknown): Record | null => { + if (!isRecord(value) || !isRecord(value.canvas)) return null; + return value.canvas; +}; + const parsePorts = ( value: unknown, expectedDirection: TDirection, @@ -660,17 +737,19 @@ const parsePort = ( value: unknown, expectedDirection: TDirection, ): (NodeCanvasPort & { direction: TDirection }) | null => { - if (!isRecord(value) || !isString(value.id) || !isString(value.label) || !isString(value.type)) { + if (!isRecord(value) || !isString(value.id)) { return null; } - if (!["control", "agent", "task", "condition", "result", "data"].includes(value.type)) { + const canonicalDescription = isRecord(value.schema) && isString(value.schema.description) ? value.schema.description : undefined; + const portType = isString(value.type) ? value.type : canonicalDescription ?? "data"; + if (!["control", "agent", "task", "condition", "result", "data"].includes(portType)) { return null; } return { id: value.id, - label: value.label, + label: isString(value.label) ? value.label : value.id, direction: expectedDirection, - type: value.type as NodeCanvasPortType, + type: portType as NodeCanvasPortType, ...(typeof value.required === "boolean" ? { required: value.required } : {}), }; }; @@ -724,16 +803,20 @@ const parseMetadata = (value: unknown): NodeCanvasNodeMetadata => { }; const parseEdge = (value: unknown): NodeCanvasEdge | null => { - if (!isRecord(value) || !isString(value.id) || !isRecord(value.source) || !isRecord(value.target)) { + if (!isRecord(value) || !isString(value.id)) { return null; } - if (!isString(value.source.nodeId) || !isString(value.source.portId) || !isString(value.target.nodeId) || !isString(value.target.portId)) { + const sourceNodeId = isRecord(value.source) ? value.source.nodeId : value.fromNodeId; + const sourcePortId = isRecord(value.source) ? value.source.portId : value.fromHandle; + const targetNodeId = isRecord(value.target) ? value.target.nodeId : value.toNodeId; + const targetPortId = isRecord(value.target) ? value.target.portId : value.toHandle; + if (!isString(sourceNodeId) || !isString(sourcePortId) || !isString(targetNodeId) || !isString(targetPortId)) { return null; } return { id: value.id, - source: { nodeId: value.source.nodeId, portId: value.source.portId }, - target: { nodeId: value.target.nodeId, portId: value.target.portId }, + source: { nodeId: sourceNodeId, portId: sourcePortId }, + target: { nodeId: targetNodeId, portId: targetPortId }, ...(isString(value.label) ? { label: value.label } : {}), }; }; diff --git a/docs-web/architecture/node-flow-foundation.md b/docs-web/architecture/node-flow-foundation.md new file mode 100644 index 0000000000..acd3237449 --- /dev/null +++ b/docs-web/architecture/node-flow-foundation.md @@ -0,0 +1,7 @@ +# Node Flow Foundation + +Code UX uses one canonical Graph v2 contract across backend, MCP, runtime, and dashboard. Graphs carry `schemaVersion: 2`, versioned definitions, typed ports, credential-id bindings, bounded policies, capabilities, side effects, disabled state, flow schemas, and optional immutable publication metadata. + +Validation resolves definitions, checks configuration, ports, policies, graph limits, and cycles, and rejects plaintext secrets and custom source. Only `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output` are executable. + +Persisted Graph v1 records retain their original immutable version and append deterministic Graph v2. Browser migration retains its original snapshot outside executable graph JSON. diff --git a/docs-web/architecture/node-flows.md b/docs-web/architecture/node-flows.md new file mode 100644 index 0000000000..493c84b123 --- /dev/null +++ b/docs-web/architecture/node-flows.md @@ -0,0 +1,16 @@ +# Node Flows + +Node flows are project-owned, versioned Graph v2 workflows. + +## Implemented runtime nodes + +| Type | Execution | +| --- | --- | +| `input` | Emits run input. | +| `set_fields` | Transforms object fields. | +| `template` | Renders text templates. | +| `provider_prompt` | Invokes a configured CLI provider. | +| `http_request` | Performs a bounded HTTP/HTTPS request. | +| `output` | Selects the result. | + +These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. diff --git a/docs-web/content/docs/architecture-node-flow-foundation.mdx b/docs-web/content/docs/architecture-node-flow-foundation.mdx new file mode 100644 index 0000000000..acd3237449 --- /dev/null +++ b/docs-web/content/docs/architecture-node-flow-foundation.mdx @@ -0,0 +1,7 @@ +# Node Flow Foundation + +Code UX uses one canonical Graph v2 contract across backend, MCP, runtime, and dashboard. Graphs carry `schemaVersion: 2`, versioned definitions, typed ports, credential-id bindings, bounded policies, capabilities, side effects, disabled state, flow schemas, and optional immutable publication metadata. + +Validation resolves definitions, checks configuration, ports, policies, graph limits, and cycles, and rejects plaintext secrets and custom source. Only `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output` are executable. + +Persisted Graph v1 records retain their original immutable version and append deterministic Graph v2. Browser migration retains its original snapshot outside executable graph JSON. diff --git a/docs-web/content/docs/architecture-node-flows.mdx b/docs-web/content/docs/architecture-node-flows.mdx new file mode 100644 index 0000000000..493c84b123 --- /dev/null +++ b/docs-web/content/docs/architecture-node-flows.mdx @@ -0,0 +1,16 @@ +# Node Flows + +Node flows are project-owned, versioned Graph v2 workflows. + +## Implemented runtime nodes + +| Type | Execution | +| --- | --- | +| `input` | Emits run input. | +| `set_fields` | Transforms object fields. | +| `template` | Renders text templates. | +| `provider_prompt` | Invokes a configured CLI provider. | +| `http_request` | Performs a bounded HTTP/HTTPS request. | +| `output` | Selects the result. | + +These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. diff --git a/docs-web/content/docs/registry.ts b/docs-web/content/docs/registry.ts index 6e1bf494dc..0b0792112c 100644 --- a/docs-web/content/docs/registry.ts +++ b/docs-web/content/docs/registry.ts @@ -107,6 +107,8 @@ export type DocsSlug = | 'user-dashboard-custom-dashboards' | 'architecture-custom-dashboard-foundation' | 'architecture-managed-container-runtime' + | 'architecture-node-flow-foundation' + | 'architecture-node-flows' | 'architecture-speech-input' | 'architecture-speech-output' | 'architecture-worker-clarification-contract' @@ -848,6 +850,20 @@ export const docsRegistry: Record = { title: "Managed Container Runtime", description: "The managed container runtime removes first-invocation Docker builds while keeping provider binaries local to each user's Docker host.", }, + 'architecture-node-flow-foundation': { + id: 'architecture-node-flow-foundation', + path: '/docs/architecture-node-flow-foundation', + section: 'Architecture', + title: "Node Flow Foundation", + description: "Code UX uses one canonical Graph v2 contract across backend, MCP, runtime, and dashboard. Graphs carry schemaVersion: 2, versioned definitions, typed ports, credential-id bindings, bounded policies, capabilities, side...", + }, + 'architecture-node-flows': { + id: 'architecture-node-flows', + path: '/docs/architecture-node-flows', + section: 'Architecture', + title: "Node Flows", + description: "Node flows are project-owned, versioned Graph v2 workflows.", + }, 'architecture-speech-input': { id: 'architecture-speech-input', path: '/docs/architecture-speech-input', @@ -976,6 +992,8 @@ export const orderedDocs: DocsRegistryEntry[] = [ docsRegistry['user-dashboard-custom-dashboards'], docsRegistry['architecture-custom-dashboard-foundation'], docsRegistry['architecture-managed-container-runtime'], + docsRegistry['architecture-node-flow-foundation'], + docsRegistry['architecture-node-flows'], docsRegistry['architecture-speech-input'], docsRegistry['architecture-speech-output'], docsRegistry['architecture-worker-clarification-contract'], diff --git a/docs-web/content/docs/user-dashboard-node-flows.mdx b/docs-web/content/docs/user-dashboard-node-flows.mdx index 2edb85fe58..949de5d5ed 100644 --- a/docs-web/content/docs/user-dashboard-node-flows.mdx +++ b/docs-web/content/docs/user-dashboard-node-flows.mdx @@ -39,3 +39,7 @@ A flow can be attached to a project agent preset as a repeatable skill with a na ## Scheduling Use the [Scheduler](/docs/user-dashboard-scheduler) page to run a saved node flow once or on a recurrence. Scheduled node-flow entries select a project-owned flow and may include optional JSON object input. Pause, resume, failure handling, and due-run behavior match the normal scheduler model. + +## Graph v2 boundary + +The dashboard edits the shared Graph v2 contract. The executable registry is limited to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`; planned palette concepts are not runtime handlers. diff --git a/docs-web/content/docs/user-dashboard-nodes-canvas.mdx b/docs-web/content/docs/user-dashboard-nodes-canvas.mdx index f668ef9733..2c7a5bc4b7 100644 --- a/docs-web/content/docs/user-dashboard-nodes-canvas.mdx +++ b/docs-web/content/docs/user-dashboard-nodes-canvas.mdx @@ -41,3 +41,7 @@ The page displays a deterministic graph summary for command workflows, including ## Empty and reset states `Clear` empties the canvas while keeping the palette available. `Reset` restores the starter trigger -> agent -> task -> condition -> output graph. The layout collapses to a single column at smaller widths so controls remain reachable without overlapping. + +## Graph v2 migration + +Serialization writes `schemaVersion: 2`. Legacy browser v1 values migrate deterministically with their untouched snapshot retained separately. Trigger, agent, task, condition, and output are planning concepts; executable definitions are limited to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`. diff --git a/docs-web/routes/docs.architecture-node-flow-foundation.lazy.tsx b/docs-web/routes/docs.architecture-node-flow-foundation.lazy.tsx new file mode 100644 index 0000000000..d4eebd0943 --- /dev/null +++ b/docs-web/routes/docs.architecture-node-flow-foundation.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import ArchitectureNodeFlowFoundationContent from '../content/docs/architecture-node-flow-foundation.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/architecture-node-flow-foundation')({ + component: () => ( + + + + ) +}) diff --git a/docs-web/routes/docs.architecture-node-flows.lazy.tsx b/docs-web/routes/docs.architecture-node-flows.lazy.tsx new file mode 100644 index 0000000000..8f7e769eab --- /dev/null +++ b/docs-web/routes/docs.architecture-node-flows.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import ArchitectureNodeFlowsContent from '../content/docs/architecture-node-flows.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/architecture-node-flows')({ + component: () => ( + + + + ) +}) diff --git a/docs-web/user/dashboard/node-flows.md b/docs-web/user/dashboard/node-flows.md index 32db788798..d532ef1050 100644 --- a/docs-web/user/dashboard/node-flows.md +++ b/docs-web/user/dashboard/node-flows.md @@ -39,3 +39,7 @@ A flow can be attached to a project agent preset as a repeatable skill with a na ## Scheduling Use the [Scheduler](./scheduler.md) page to run a saved node flow once or on a recurrence. Scheduled node-flow entries select a project-owned flow and may include optional JSON object input. Pause, resume, failure handling, and due-run behavior match the normal scheduler model. + +## Graph v2 boundary + +The dashboard edits the shared Graph v2 contract. The executable registry is limited to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`; planned palette concepts are not runtime handlers. diff --git a/docs-web/user/dashboard/nodes-canvas.md b/docs-web/user/dashboard/nodes-canvas.md index f668ef9733..2c7a5bc4b7 100644 --- a/docs-web/user/dashboard/nodes-canvas.md +++ b/docs-web/user/dashboard/nodes-canvas.md @@ -41,3 +41,7 @@ The page displays a deterministic graph summary for command workflows, including ## Empty and reset states `Clear` empties the canvas while keeping the palette available. `Reset` restores the starter trigger -> agent -> task -> condition -> output graph. The layout collapses to a single column at smaller widths so controls remain reachable without overlapping. + +## Graph v2 migration + +Serialization writes `schemaVersion: 2`. Legacy browser v1 values migrate deterministically with their untouched snapshot retained separately. Trigger, agent, task, condition, and output are planning concepts; executable definitions are limited to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`. diff --git a/docs/architecture/node-flow-foundation.md b/docs/architecture/node-flow-foundation.md index 6bb2573ceb..7a854f6f65 100644 --- a/docs/architecture/node-flow-foundation.md +++ b/docs/architecture/node-flow-foundation.md @@ -77,3 +77,11 @@ Dashboard routes are registered through `registerNodeFlowRoutes`: - `GET /api/node-flow-runs/:runId/node-runs` Handlers stay thin and delegate behavior to `NodeFlowService`. + +## Canonical Graph v2 + +Normalized graphs carry `schemaVersion: 2`. Nodes reference a stable definition type and version and carry typed ports, credential-id bindings, bounded retry/timeout policies, capabilities, side-effect classification, and disabled state. Graphs may declare typed input/output schemas and immutable publication metadata. + +The typed registry includes configuration and UI schemas, ports, credential slots, capabilities, side effects, default policies, documentation, deprecation, and execution kind. Only `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output` are executable. Other palette ideas are planned concepts without runtime handlers. + +Validation resolves definitions and checks configuration, handles, policies, graph limits, and cycles with field-level issues. Graph JSON rejects secret-shaped fields and generated/custom source fields. Persisted Graph v1 rows keep their original immutable snapshot and append deterministic Graph v2 as a new current version. diff --git a/docs/architecture/node-flows.md b/docs/architecture/node-flows.md index 42914ddc4f..6268a53066 100644 --- a/docs/architecture/node-flows.md +++ b/docs/architecture/node-flows.md @@ -98,3 +98,11 @@ Use these rules: - Validate every node field before saving: required prompt/template/url fields, finite numeric limits, supported HTTP method, JSON object input, and select defaults that match options. - Keep flows deterministic and rerunnable. Avoid hidden dependence on local time, ambient chat state, or one-off sprint context unless it is explicitly passed as JSON input. - Preserve inspection value. Name nodes for the operation they perform, keep edges acyclic, and make the output node return the artifact another operator or agent will actually consume. + +## Graph v2 contract and migration + +Graph v2 is the single workflow model used by backend, MCP, runtime, and dashboard. It adds `schemaVersion: 2`, stable definition references, typed ports and flow schemas, credential-id bindings, retry and timeout policies, capability and side-effect metadata, disabled state, and optional immutable publication metadata. Plaintext credentials, secret-shaped fields, generated source, and custom code are not valid graph data. + +The executable registry is exactly `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`. Planned trigger, agent-router, task, condition, notification, and integration entries are not executable until handlers exist. + +Backend Graph v1 migration retains the exact prior version and appends deterministic v2. Browser canvas v1 migration returns the untouched legacy snapshot separately from the normalized graph. diff --git a/docs/dashboard/node-flows.md b/docs/dashboard/node-flows.md index 87ff413060..3c78d9d122 100644 --- a/docs/dashboard/node-flows.md +++ b/docs/dashboard/node-flows.md @@ -91,3 +91,7 @@ Nodes must stay usable on desktop and mobile: - validation and run errors must be text, not color-only - focus order should follow library -> canvas -> inspector -> run/attachment panels - JSON editors and textareas should preserve visible labels and error messages on small screens + +## Graph v2 boundary + +The dashboard edits the shared Graph v2 contract rather than owning a second execution model. A graph is executable only when every node resolves to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, or `output`. Planning palette concepts are not runtime handlers. diff --git a/docs/dashboard/nodes-canvas.md b/docs/dashboard/nodes-canvas.md index 2bd32be0a2..7d44194533 100644 --- a/docs/dashboard/nodes-canvas.md +++ b/docs/dashboard/nodes-canvas.md @@ -73,3 +73,7 @@ The page displays the deterministic `buildNodeCanvasAgentSummary` output so agen `Clear` replaces the graph with an empty canvas. The canvas and inspector show empty/no-selection states, while the palette remains available for recovery. `Reset` restores the starter trigger -> agent -> task -> condition -> output graph and refreshes the exchange JSON. The page uses responsive grid columns that collapse into a single column at smaller widths so the palette, canvas, inspector, validation panel, and JSON exchange controls remain reachable without overlapping. + +## Graph v2 migration + +Serialization writes `schemaVersion: 2`. Importing a legacy browser v1 value deterministically normalizes it and retains the untouched snapshot separately. Trigger, agent, task, condition, and output palette entries are planning concepts; executable definitions are limited to `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`. diff --git a/src/contracts/node-definition-types.ts b/src/contracts/node-definition-types.ts new file mode 100644 index 0000000000..6e49a1fca6 --- /dev/null +++ b/src/contracts/node-definition-types.ts @@ -0,0 +1,46 @@ +import type { + NodeFlowExecutionPolicy, + NodeFlowPort, + NodeFlowSideEffect, + NodeFlowValueSchema, + NodeWidgetSchema, +} from "./node-flow-types.js"; + +export type NodeDefinitionExecutionKind = "local" | "provider" | "http" | "unavailable"; + +export interface NodeDefinitionCredentialRequirement { + slot: string; + label: string; + required: boolean; + allowedKinds: string[]; +} + +export interface NodeDefinitionUiManifest { + label: string; + description: string; + category: string; + icon?: string; + widgetSchema: NodeWidgetSchema; +} + +export interface NodeDefinitionDeprecation { + deprecated: boolean; + message?: string; + replacementType?: string; +} + +export interface NodeDefinitionManifest { + type: string; + version: number; + executable: boolean; + executionKind: NodeDefinitionExecutionKind; + configurationSchema: NodeFlowValueSchema; + ui: NodeDefinitionUiManifest; + ports: NodeFlowPort[]; + credentials: NodeDefinitionCredentialRequirement[]; + capabilities: string[]; + sideEffect: NodeFlowSideEffect; + defaultPolicy: NodeFlowExecutionPolicy; + documentation: string; + deprecation: NodeDefinitionDeprecation; +} diff --git a/src/contracts/node-flow-types.ts b/src/contracts/node-flow-types.ts index 1921310dd9..8e19d7ab7a 100644 --- a/src/contracts/node-flow-types.ts +++ b/src/contracts/node-flow-types.ts @@ -15,6 +15,66 @@ export type NodeFlowJsonValue = | { [key: string]: NodeFlowJsonValue }; export type NodeFlowJsonObject = { [key: string]: NodeFlowJsonValue }; +export const NODE_FLOW_SCHEMA_VERSION = 2 as const; +export type NodeFlowSchemaVersion = typeof NODE_FLOW_SCHEMA_VERSION; + +export type NodeFlowPortDirection = "input" | "output"; +export type NodeFlowPortCardinality = "one" | "many"; +export type NodeFlowSideEffect = "none" | "read" | "write" | "external"; + +export interface NodeFlowValueSchema { + type: "any" | "object" | "array" | "string" | "number" | "boolean" | "null"; + description?: string; + required?: string[]; + properties?: Record; + items?: NodeFlowValueSchema; +} + +export interface NodeFlowDefinitionReference { + type: string; + version: number; +} + +export interface NodeFlowPort { + id: string; + direction: NodeFlowPortDirection; + schema: NodeFlowValueSchema; + required?: boolean; + cardinality?: NodeFlowPortCardinality; +} + +export interface NodeFlowCredentialBinding { + slot: string; + credentialId: string; +} + +export interface NodeFlowRetryPolicy { + maxAttempts: number; + backoffMs: number; + maxBackoffMs?: number; +} + +export interface NodeFlowTimeoutPolicy { + timeoutMs: number; +} + +export interface NodeFlowExecutionPolicy { + retry?: NodeFlowRetryPolicy; + timeout?: NodeFlowTimeoutPolicy; +} + +export interface NodeFlowPublicationMetadata { + publicationId: string; + publishedAt: string; + publishedBy: string; + sourceVersion: number; +} + +export interface NodeFlowSchemas { + input?: NodeFlowValueSchema; + output?: NodeFlowValueSchema; +} + export interface NodeWidgetSelectOption { label: string; value: string | number | boolean; @@ -51,6 +111,13 @@ export interface NodeFlowNode { widgetSchema?: NodeWidgetSchema; position?: NodeFlowNodePosition; data?: NodeFlowJsonObject; + definition?: NodeFlowDefinitionReference; + ports?: NodeFlowPort[]; + credentialBindings?: NodeFlowCredentialBinding[]; + policy?: NodeFlowExecutionPolicy; + capabilities?: string[]; + sideEffect?: NodeFlowSideEffect; + disabled?: boolean; } export interface NodeFlowEdge { @@ -62,10 +129,13 @@ export interface NodeFlowEdge { } export interface NodeFlowGraph { + schemaVersion?: NodeFlowSchemaVersion; nodes: NodeFlowNode[]; edges: NodeFlowEdge[]; inputSchema?: NodeWidgetSchema; + schemas?: NodeFlowSchemas; metadata?: NodeFlowJsonObject; + publication?: Readonly; } export interface NodeFlowRecord { diff --git a/src/domain/node-flows/node-definition-registry.ts b/src/domain/node-flows/node-definition-registry.ts new file mode 100644 index 0000000000..498c0fc1d4 --- /dev/null +++ b/src/domain/node-flows/node-definition-registry.ts @@ -0,0 +1,91 @@ +import type { NodeDefinitionManifest } from "../../contracts/node-definition-types.js"; +import type { NodeFlowPort, NodeFlowValueSchema, NodeWidgetField } from "../../contracts/node-flow-types.js"; + +const objectSchema = (required: string[] = [], properties: Record = {}) => ({ + type: "object" as const, + ...(required.length > 0 ? { required } : {}), + ...(Object.keys(properties).length > 0 ? { properties } : {}), +}); + +const dataPort = (id: string, direction: "input" | "output", required = false): NodeFlowPort => ({ + id, + direction, + schema: { type: "object" }, + required, + cardinality: direction === "input" ? "many" : "one", +}); + +const field = ( + id: string, + label: string, + type: NodeWidgetField["type"], + required = false, +): NodeWidgetField => ({ id, label, type, required }); + +const manifests: NodeDefinitionManifest[] = [ + { + type: "input", version: 1, executable: true, executionKind: "local", + configurationSchema: objectSchema(), + ui: { label: "Input", description: "Emits the flow input.", category: "control", widgetSchema: { fields: [] } }, + ports: [dataPort("output", "output")], credentials: [], capabilities: [], sideEffect: "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, + { + type: "set_fields", version: 1, executable: true, executionKind: "local", + configurationSchema: objectSchema([], { fields: { type: "object" }, values: { type: "object" }, replace: { type: "boolean" } }), + ui: { label: "Set fields", description: "Adds or replaces object fields.", category: "transform", widgetSchema: { fields: [field("fields", "Fields", "json")] } }, + ports: [dataPort("input", "input"), dataPort("output", "output")], credentials: [], capabilities: [], sideEffect: "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, + { + type: "template", version: 1, executable: true, executionKind: "local", + configurationSchema: objectSchema(["template"], { template: { type: "string" }, prompt: { type: "string" }, outputKey: { type: "string" } }), + ui: { label: "Template", description: "Renders a deterministic text template.", category: "transform", widgetSchema: { fields: [field("template", "Template", "textarea", true), field("outputKey", "Output key", "text")] } }, + ports: [dataPort("input", "input"), dataPort("output", "output")], credentials: [], capabilities: [], sideEffect: "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, + { + type: "provider_prompt", version: 1, executable: true, executionKind: "provider", + configurationSchema: objectSchema(["prompt"], { prompt: { type: "string" }, template: { type: "string" }, provider: { type: "string" } }), + ui: { label: "Provider prompt", description: "Runs a prompt through a configured CLI provider.", category: "ai", widgetSchema: { fields: [field("prompt", "Prompt", "textarea", true), field("provider", "Provider", "text")] } }, + ports: [dataPort("input", "input"), dataPort("output", "output")], + credentials: [{ slot: "provider", label: "Provider connection", required: false, allowedKinds: ["provider"] }], + capabilities: ["provider.execute"], sideEffect: "external", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, + { + type: "http_request", version: 1, executable: true, executionKind: "http", + configurationSchema: objectSchema(["url"], { url: { type: "string" }, method: { type: "string" }, timeoutMs: { type: "number" }, headers: { type: "object" } }), + ui: { label: "HTTP request", description: "Calls a bounded HTTP or HTTPS endpoint.", category: "integration", widgetSchema: { fields: [field("url", "URL", "text", true), field("method", "Method", "text")] } }, + ports: [dataPort("input", "input"), dataPort("output", "output")], + credentials: [{ slot: "auth", label: "HTTP credential", required: false, allowedKinds: ["http"] }], + capabilities: ["network.http"], sideEffect: "external", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 }, timeout: { timeoutMs: 30_000 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, + { + type: "output", version: 1, executable: true, executionKind: "local", + configurationSchema: objectSchema(), + ui: { label: "Output", description: "Selects the flow result.", category: "control", widgetSchema: { fields: [] } }, + ports: [dataPort("input", "input")], credentials: [], capabilities: [], sideEffect: "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, documentation: "docs/architecture/node-flows.md#runtime", + deprecation: { deprecated: false }, + }, +]; + +const keyFor = (type: string, version: number): string => `${type}@${version}`; +const registry = new Map(manifests.map((manifest) => [keyFor(manifest.type, manifest.version), manifest])); + +export const listNodeDefinitions = (): readonly NodeDefinitionManifest[] => manifests; + +export const resolveNodeDefinition = (type: string, version: number): NodeDefinitionManifest | null => ( + registry.get(keyFor(type, version)) ?? null +); + +export const resolveLatestNodeDefinition = (type: string): NodeDefinitionManifest | null => ( + manifests.filter((manifest) => manifest.type === type).sort((left, right) => right.version - left.version)[0] ?? null +); diff --git a/src/domain/node-flows/node-flow-migrators.ts b/src/domain/node-flows/node-flow-migrators.ts new file mode 100644 index 0000000000..527ae746c3 --- /dev/null +++ b/src/domain/node-flows/node-flow-migrators.ts @@ -0,0 +1,110 @@ +import { + NODE_FLOW_SCHEMA_VERSION, + type NodeFlowGraph, + type NodeFlowJsonObject, + type NodeFlowJsonValue, + type NodeFlowNode, +} from "../../contracts/node-flow-types.js"; +import { resolveLatestNodeDefinition } from "./node-definition-registry.js"; + +export interface NodeFlowMigrationResult { + graph: NodeFlowGraph; + migrated: boolean; + legacySnapshot: TLegacy | null; +} + +const cloneJson = (value: T): T => JSON.parse(JSON.stringify(value)) as T; + +export function migrateNodeFlowGraph(graph: unknown): NodeFlowMigrationResult { + if (isRecord(graph) && graph.schemaVersion === NODE_FLOW_SCHEMA_VERSION) { + return { graph: cloneJson(graph as unknown as NodeFlowGraph), migrated: false, legacySnapshot: null }; + } + + const legacy = isRecord(graph) ? cloneJson(graph as unknown as NodeFlowGraph) : { nodes: [], edges: [] }; + const nodes = Array.isArray(legacy.nodes) ? legacy.nodes.map(migrateNode) : []; + const definitionByNode = new Map(nodes.map((node) => [node.id, resolveLatestNodeDefinition(node.type)])); + const edges = Array.isArray(legacy.edges) ? legacy.edges.map((edge) => ({ + ...edge, + ...(edge.fromHandle ? {} : { fromHandle: definitionByNode.get(edge.fromNodeId)?.ports.find((port) => port.direction === "output")?.id }), + ...(edge.toHandle ? {} : { toHandle: definitionByNode.get(edge.toNodeId)?.ports.find((port) => port.direction === "input")?.id }), + })) : []; + + return { + migrated: true, + legacySnapshot: legacy, + graph: { + schemaVersion: NODE_FLOW_SCHEMA_VERSION, + nodes, + edges, + ...(legacy.inputSchema ? { inputSchema: legacy.inputSchema } : {}), + ...(legacy.schemas ? { schemas: legacy.schemas } : {}), + ...(legacy.metadata ? { metadata: legacy.metadata } : {}), + ...(legacy.publication ? { publication: legacy.publication } : {}), + }, + }; +} + +function migrateNode(node: NodeFlowNode): NodeFlowNode { + const definition = node.definition + ? resolveLatestNodeDefinition(node.definition.type) + : resolveLatestNodeDefinition(node.type); + return { + ...node, + definition: node.definition ?? { type: node.type, version: definition?.version ?? 1 }, + ports: node.ports ?? definition?.ports.map((port) => cloneJson(port)) ?? [], + credentialBindings: node.credentialBindings ?? [], + policy: node.policy ?? (definition ? cloneJson(definition.defaultPolicy) : {}), + capabilities: node.capabilities ?? [...(definition?.capabilities ?? [])], + sideEffect: node.sideEffect ?? definition?.sideEffect ?? "none", + disabled: node.disabled ?? false, + }; +} + +export function migrateNodeCanvasGraphV1(graph: unknown): NodeFlowMigrationResult { + if (isRecord(graph) && graph.schemaVersion === NODE_FLOW_SCHEMA_VERSION && Array.isArray(graph.nodes) && graph.nodes.every(isCanonicalNode)) { + return migrateNodeFlowGraph(graph); + } + const legacySnapshot = cloneJson(graph); + const legacy = isRecord(graph) ? graph : {}; + const nodes = Array.isArray(legacy.nodes) ? legacy.nodes.filter(isRecord).map((node): NodeFlowNode => ({ + id: stringValue(node.id), + type: stringValue(node.kind) || stringValue(node.type), + title: stringValue(node.label) || stringValue(node.title) || stringValue(node.id), + description: stringValue(node.description) || undefined, + position: isPosition(node.position) ? { x: node.position.x, y: node.position.y } : undefined, + definition: { type: stringValue(node.kind) || stringValue(node.type), version: 1 }, + ports: [...readPorts(node.inputPorts, "input"), ...readPorts(node.outputPorts, "output")], + data: canvasNodeData(node), + credentialBindings: [], policy: {}, capabilities: [], sideEffect: "none", disabled: false, + })).filter((node) => node.id && node.type) : []; + const edges = Array.isArray(legacy.edges) ? legacy.edges.filter(isRecord).map((edge) => ({ + id: stringValue(edge.id) || undefined, + fromNodeId: endpointValue(edge.source, "nodeId"), + toNodeId: endpointValue(edge.target, "nodeId"), + fromHandle: endpointValue(edge.source, "portId") || undefined, + toHandle: endpointValue(edge.target, "portId") || undefined, + })).filter((edge) => edge.fromNodeId && edge.toNodeId) : []; + return { + migrated: true, + legacySnapshot, + graph: { schemaVersion: NODE_FLOW_SCHEMA_VERSION, nodes, edges, metadata: { canvasSelection: jsonValue(legacy.selection) } }, + }; +} + +function readPorts(value: unknown, direction: "input" | "output") { + return Array.isArray(value) ? value.filter(isRecord).map((port) => ({ + id: stringValue(port.id), direction, schema: { type: "any" as const }, required: port.required === true, + })).filter((port) => port.id) : []; +} + +function canvasNodeData(node: Record): NodeFlowJsonObject { + const config = Array.isArray(node.config) ? Object.fromEntries(node.config.filter(isRecord).map((entry) => [stringValue(entry.id), jsonValue(entry.value)]).filter(([id]) => id)) : {}; + return { config, canvasMetadata: jsonValue(node.metadata) }; +} + +const jsonValue = (value: unknown): NodeFlowJsonValue => JSON.parse(JSON.stringify(value ?? null)) as NodeFlowJsonValue; +const endpointValue = (value: unknown, key: string): string => isRecord(value) ? stringValue(value[key]) : ""; +const stringValue = (value: unknown): string => typeof value === "string" ? value : ""; +const isRecord = (value: unknown): value is Record => Boolean(value) && typeof value === "object" && !Array.isArray(value); +const isPosition = (value: unknown): value is { x: number; y: number } => isRecord(value) && Number.isFinite(value.x) && Number.isFinite(value.y); +const isCanonicalNode = (value: unknown): boolean => isRecord(value) && isRecord(value.definition); diff --git a/src/domain/node-flows/node-flow-validation.ts b/src/domain/node-flows/node-flow-validation.ts index 0b5d2cbf77..ad98c35254 100644 --- a/src/domain/node-flows/node-flow-validation.ts +++ b/src/domain/node-flows/node-flow-validation.ts @@ -2,14 +2,23 @@ import { ValidationError } from "../../repositories/repository-utils.js"; import type { NodeFlowEdge, NodeFlowGraph, + NodeFlowJsonObject, NodeFlowJsonValue, NodeFlowNode, NodeFlowValidationIssue, NodeFlowValidationResponse, + NodeFlowValueSchema, NodeWidgetField, NodeWidgetFieldType, NodeWidgetSchema, } from "../../contracts/node-flow-types.js"; +import { NODE_FLOW_SCHEMA_VERSION } from "../../contracts/node-flow-types.js"; +import { migrateNodeFlowGraph } from "./node-flow-migrators.js"; +import { resolveNodeDefinition } from "./node-definition-registry.js"; + +const MAX_GRAPH_NODES = 250; +const MAX_GRAPH_EDGES = 1_000; +const FORBIDDEN_GRAPH_KEY = /^(?:sourceCode|generatedSource|code|script|apiKey|authorization|cookie|password|secret|token)$/i; const WIDGET_FIELD_TYPES = new Set([ "text", @@ -243,6 +252,78 @@ function normalizeWidgetSchema( return { fields: normalizedFields }; } +function containsForbiddenGraphValue(value: NodeFlowJsonValue): boolean { + if (Array.isArray(value)) return value.some(containsForbiddenGraphValue); + if (!value || typeof value !== "object") return false; + return Object.entries(value).some(([key, entry]) => FORBIDDEN_GRAPH_KEY.test(key) || containsForbiddenGraphValue(entry)); +} + +function validatePolicy( + policy: NodeFlowNode["policy"], + path: string, + issues: NodeFlowValidationIssue[], +): void { + if (!policy) return; + const retry = policy.retry; + if (retry && (!Number.isInteger(retry.maxAttempts) || retry.maxAttempts < 1 || retry.maxAttempts > 10)) { + issues.push(issue(`${path}.retry.maxAttempts`, "invalid_retry_policy", "Retry maxAttempts must be an integer from 1 to 10.")); + } + if (retry && (!Number.isFinite(retry.backoffMs) || retry.backoffMs < 0 || retry.backoffMs > 300_000)) { + issues.push(issue(`${path}.retry.backoffMs`, "invalid_retry_policy", "Retry backoffMs must be between 0 and 300000.")); + } + if (retry?.maxBackoffMs !== undefined && (!Number.isFinite(retry.maxBackoffMs) || retry.maxBackoffMs < retry.backoffMs)) { + issues.push(issue(`${path}.retry.maxBackoffMs`, "invalid_retry_policy", "Retry maxBackoffMs must be at least backoffMs.")); + } + if (policy.timeout && (!Number.isInteger(policy.timeout.timeoutMs) || policy.timeout.timeoutMs < 1 || policy.timeout.timeoutMs > 300_000)) { + issues.push(issue(`${path}.timeout.timeoutMs`, "invalid_timeout_policy", "Timeout must be an integer from 1 to 300000 milliseconds.")); + } +} + +function validateCredentialBindings( + node: NodeFlowNode, + nodePath: string, + allowedSlots: string[], + issues: NodeFlowValidationIssue[], +): void { + const slots = new Set(); + (node.credentialBindings ?? []).forEach((binding, index) => { + const path = `${nodePath}.credentialBindings[${index}]`; + if (!trimmedString(binding.slot)) issues.push(issue(`${path}.slot`, "required", "Credential slot is required.")); + if (!trimmedString(binding.credentialId)) issues.push(issue(`${path}.credentialId`, "required", "Credential binding must reference a credential id.")); + if (slots.has(binding.slot)) issues.push(issue(`${path}.slot`, "duplicate_credential_slot", `Duplicate credential slot: ${binding.slot}`)); + if (!allowedSlots.includes(binding.slot)) issues.push(issue(`${path}.slot`, "unknown_credential_slot", `Definition does not declare credential slot: ${binding.slot}`)); + slots.add(binding.slot); + }); +} + +function validateConfiguration( + data: NodeFlowJsonObject, + schema: NodeFlowValueSchema, + nodePath: string, + issues: NodeFlowValidationIssue[], +): void { + const values = isPlainJsonObject(data.values) ? data.values : {}; + for (const key of schema.required ?? []) { + const aliases = key === "template" || key === "prompt" ? ["template", "prompt"] : [key]; + const present = aliases.some((alias) => data[alias] !== undefined || values[alias] !== undefined); + if (!present) issues.push(issue(`${nodePath}.data.${key}`, "required_configuration", `Node configuration requires ${key}.`)); + } + for (const [key, propertySchema] of Object.entries(schema.properties ?? {})) { + const value = data[key] ?? values[key]; + if (value !== undefined && !matchesValueSchema(value, propertySchema.type)) { + issues.push(issue(`${nodePath}.data.${key}`, "invalid_configuration_type", `Node configuration ${key} must be ${propertySchema.type}.`)); + } + } +} + +function matchesValueSchema(value: NodeFlowJsonValue, type: NodeFlowValueSchema["type"]): boolean { + if (type === "any") return true; + if (type === "null") return value === null; + if (type === "array") return Array.isArray(value); + if (type === "object") return Boolean(value) && typeof value === "object" && !Array.isArray(value); + return typeof value === type; +} + function normalizeNode(rawNode: NodeFlowNode, index: number, issues: NodeFlowValidationIssue[]): NodeFlowNode | null { const nodePath = `nodes[${index}]`; if (!rawNode || typeof rawNode !== "object" || Array.isArray(rawNode)) { @@ -280,6 +361,36 @@ function normalizeNode(rawNode: NodeFlowNode, index: number, issues: NodeFlowVal issues.push(issue(`${nodePath}.data`, "invalid_data", `Node ${id} data must be a JSON object.`)); } + const definitionRef = rawNode.definition ?? { type, version: 1 }; + const definition = resolveNodeDefinition(definitionRef.type, definitionRef.version); + if (definitionRef.type !== type) { + issues.push(issue(`${nodePath}.definition.type`, "definition_type_mismatch", "Node type must match its definition reference.")); + } + if (!definition) { + issues.push(issue(`${nodePath}.definition`, "unknown_node_definition", `Unknown node definition: ${definitionRef.type}@${definitionRef.version}`)); + } + if (definition && rawNode.sideEffect !== undefined && rawNode.sideEffect !== definition.sideEffect) { + issues.push(issue(`${nodePath}.sideEffect`, "definition_metadata_mismatch", "Node side effect must match its definition.")); + } + if (definition && rawNode.capabilities !== undefined && [...rawNode.capabilities].sort().join("\0") !== [...definition.capabilities].sort().join("\0")) { + issues.push(issue(`${nodePath}.capabilities`, "definition_metadata_mismatch", "Node capabilities must match its definition.")); + } + const ports = rawNode.ports ?? definition?.ports ?? []; + const portIds = new Set(); + ports.forEach((port, portIndex) => { + const portPath = `${nodePath}.ports[${portIndex}]`; + if (!trimmedString(port.id)) issues.push(issue(`${portPath}.id`, "required", "Port id is required.")); + if (portIds.has(port.id)) issues.push(issue(`${portPath}.id`, "duplicate_port_id", `Duplicate port id: ${port.id}`)); + portIds.add(port.id); + if (port.direction !== "input" && port.direction !== "output") issues.push(issue(`${portPath}.direction`, "invalid_port_direction", "Port direction must be input or output.")); + }); + validatePolicy(rawNode.policy, `${nodePath}.policy`, issues); + validateCredentialBindings(rawNode, nodePath, definition?.credentials.map((credential) => credential.slot) ?? [], issues); + if (rawNode.data && containsForbiddenGraphValue(rawNode.data)) { + issues.push(issue(`${nodePath}.data`, "unsafe_graph_data", "Graph data cannot contain raw secrets or custom source code.")); + } + if (definition) validateConfiguration(isPlainJsonObject(rawNode.data) ? rawNode.data : {}, definition.configurationSchema, nodePath, issues); + return { id, type, @@ -288,6 +399,13 @@ function normalizeNode(rawNode: NodeFlowNode, index: number, issues: NodeFlowVal ...(widgetSchema ? { widgetSchema } : {}), ...(position ? { position } : {}), ...(rawNode.data !== undefined && isPlainJsonObject(rawNode.data) ? { data: rawNode.data } : {}), + definition: definitionRef, + ports, + credentialBindings: rawNode.credentialBindings ?? [], + policy: rawNode.policy ?? definition?.defaultPolicy ?? {}, + capabilities: definition?.capabilities ?? rawNode.capabilities ?? [], + sideEffect: definition?.sideEffect ?? rawNode.sideEffect ?? "none", + disabled: rawNode.disabled ?? false, }; } @@ -329,16 +447,17 @@ function computeExecutionOrder( inDegree.set(edge.toNodeId, (inDegree.get(edge.toNodeId) ?? 0) + 1); } - const ready = nodes.filter((node) => inDegree.get(node.id) === 0).map((node) => node.id); + const ready = nodes.filter((node) => inDegree.get(node.id) === 0).map((node) => node.id).sort(); const order: string[] = []; while (ready.length > 0) { const nodeId = ready.shift()!; order.push(nodeId); - for (const nextId of outgoing.get(nodeId) ?? []) { + for (const nextId of [...(outgoing.get(nodeId) ?? [])].sort()) { const nextInDegree = (inDegree.get(nextId) ?? 0) - 1; inDegree.set(nextId, nextInDegree); if (nextInDegree === 0) { ready.push(nextId); + ready.sort(); } } } @@ -363,13 +482,20 @@ export function validateNodeFlowGraph(graph: unknown): NodeFlowValidationRespons }; } - const rawGraph = graph as NodeFlowGraph; + const migration = migrateNodeFlowGraph(graph); + const rawGraph = migration.graph; if (!Array.isArray(rawGraph.nodes)) { issues.push(issue("nodes", "required", "Node flow graph requires a nodes array.")); } if (!Array.isArray(rawGraph.edges)) { issues.push(issue("edges", "required", "Node flow graph requires an edges array.")); } + if (Array.isArray(rawGraph.nodes) && rawGraph.nodes.length > MAX_GRAPH_NODES) { + issues.push(issue("nodes", "graph_limit_exceeded", `Node flow graph supports at most ${MAX_GRAPH_NODES} nodes.`)); + } + if (Array.isArray(rawGraph.edges) && rawGraph.edges.length > MAX_GRAPH_EDGES) { + issues.push(issue("edges", "graph_limit_exceeded", `Node flow graph supports at most ${MAX_GRAPH_EDGES} edges.`)); + } const nodes = Array.isArray(rawGraph.nodes) ? rawGraph.nodes.map((node, index) => normalizeNode(node, index, issues)).filter((node): node is NodeFlowNode => Boolean(node)) @@ -397,18 +523,33 @@ export function validateNodeFlowGraph(graph: unknown): NodeFlowValidationRespons if (!nodeIds.has(edge.toNodeId)) { issues.push(issue(`edges[${index}].toNodeId`, "invalid_edge_endpoint", `Edge target node does not exist: ${edge.toNodeId}`)); } + const source = nodes.find((node) => node.id === edge.fromNodeId); + const target = nodes.find((node) => node.id === edge.toNodeId); + if (source && edge.fromHandle && !source.ports?.some((port) => port.id === edge.fromHandle && port.direction === "output")) { + issues.push(issue(`edges[${index}].fromHandle`, "invalid_source_port", `Source port does not exist or is not an output: ${edge.fromHandle}`)); + } + if (target && edge.toHandle && !target.ports?.some((port) => port.id === edge.toHandle && port.direction === "input")) { + issues.push(issue(`edges[${index}].toHandle`, "invalid_target_port", `Target port does not exist or is not an input: ${edge.toHandle}`)); + } }); const inputSchema = normalizeWidgetSchema(rawGraph.inputSchema, "inputSchema", issues); if (rawGraph.metadata !== undefined && !isPlainJsonObject(rawGraph.metadata)) { issues.push(issue("metadata", "invalid_metadata", "Node flow graph metadata must be a JSON object.")); } + if (rawGraph.metadata && containsForbiddenGraphValue(rawGraph.metadata)) { + issues.push(issue("metadata", "unsafe_graph_metadata", "Graph metadata cannot contain raw secrets or custom source code.")); + } + validatePublication(rawGraph.publication, issues); const normalizedGraph: NodeFlowGraph = { + schemaVersion: NODE_FLOW_SCHEMA_VERSION, nodes, edges, ...(inputSchema ? { inputSchema } : {}), + ...(rawGraph.schemas ? { schemas: rawGraph.schemas } : {}), ...(rawGraph.metadata !== undefined && isPlainJsonObject(rawGraph.metadata) ? { metadata: rawGraph.metadata } : {}), + ...(rawGraph.publication ? { publication: rawGraph.publication } : {}), }; const executionOrder = computeExecutionOrder(nodes, edges, issues); const valid = issues.length === 0; @@ -419,6 +560,21 @@ export function validateNodeFlowGraph(graph: unknown): NodeFlowValidationRespons }; } +function validatePublication( + publication: NodeFlowGraph["publication"], + issues: NodeFlowValidationIssue[], +): void { + if (!publication) return; + if (!trimmedString(publication.publicationId)) issues.push(issue("publication.publicationId", "required", "Publication id is required.")); + if (!trimmedString(publication.publishedBy)) issues.push(issue("publication.publishedBy", "required", "Publication author is required.")); + if (!trimmedString(publication.publishedAt) || !Number.isFinite(Date.parse(publication.publishedAt))) { + issues.push(issue("publication.publishedAt", "invalid_publication", "Publication timestamp must be ISO-compatible.")); + } + if (!Number.isInteger(publication.sourceVersion) || publication.sourceVersion < 1) { + issues.push(issue("publication.sourceVersion", "invalid_publication", "Publication sourceVersion must be a positive integer.")); + } +} + export function normalizeNodeFlowGraph(graph: unknown): NormalizedNodeFlowValidation { const result = validateNodeFlowGraph(graph); if (!result.valid || !result.graph || !result.executionOrder) { diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index 5cde8f7975..b874fd2de6 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -1,4 +1,5 @@ import { DatabaseAdapter } from "./database-adapter.js"; +import { migrateNodeFlowGraph } from "../../domain/node-flows/node-flow-migrators.js"; export function ensureColumn(db: DatabaseAdapter, tableName: string, columnName: string, columnDefinition: string): void { // Using direct sqlite PRAGMA for now, until we abstract schema reflections @@ -267,6 +268,50 @@ export function ensureNodeFlowTables(db: DatabaseAdapter): void { ensureIndex(db, "idx_node_flow_node_runs_run_created", "node_flow_node_runs", "run_id, created_at ASC"); } +interface LegacyNodeFlowRow { + id: string; + project_id: string; + title: string; + description: string | null; + graph_json: string; + version: number | string; + updated_at: string; +} + +export function migratePersistedNodeFlowGraphs(db: DatabaseAdapter): void { + const rows = db.prepare("SELECT id, project_id, title, description, graph_json, version, updated_at FROM node_flows ORDER BY id ASC") + .all() as unknown as LegacyNodeFlowRow[]; + for (const row of rows) { + let parsed: unknown; + try { + parsed = JSON.parse(row.graph_json); + } catch { + continue; + } + const migration = migrateNodeFlowGraph(parsed); + if (!migration.migrated) continue; + const currentVersion = Number(row.version); + const nextVersion = currentVersion + 1; + const migratedJson = JSON.stringify(migration.graph); + db.transaction(() => { + const originalExists = db.prepare("SELECT id FROM node_flow_versions WHERE flow_id = ? AND version = ?") + .get(row.id, currentVersion); + if (!originalExists) { + db.prepare(` + INSERT INTO node_flow_versions (id, flow_id, project_id, version, title, description, graph_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run(`${row.id}:v${currentVersion}:legacy`, row.id, row.project_id, currentVersion, row.title, row.description ?? "", row.graph_json, row.updated_at); + } + db.prepare(` + INSERT INTO node_flow_versions (id, flow_id, project_id, version, title, description, graph_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run(`${row.id}:v${nextVersion}:schema-v2`, row.id, row.project_id, nextVersion, row.title, row.description ?? "", migratedJson, row.updated_at); + db.prepare("UPDATE node_flows SET graph_json = ?, version = ? WHERE id = ?") + .run(migratedJson, nextVersion, row.id); + }); + } +} + export function ensureCustomDashboardTables(db: DatabaseAdapter): void { db.exec(` CREATE TABLE IF NOT EXISTS custom_dashboards ( @@ -580,6 +625,7 @@ export function runMigrations(db: DatabaseAdapter): void { ensureTaskSelfReflectionRatingTables(db); ensureConversationDraftTables(db); ensureNodeFlowTables(db); + migratePersistedNodeFlowGraphs(db); ensureCustomDashboardTables(db); ensureColumn(db, "projects", "initialization_mode", "TEXT NOT NULL DEFAULT 'existing'"); diff --git a/src/repositories/node-flow-repository.ts b/src/repositories/node-flow-repository.ts index f5df530558..faf2e5ae48 100644 --- a/src/repositories/node-flow-repository.ts +++ b/src/repositories/node-flow-repository.ts @@ -20,6 +20,7 @@ import type { UpdateNodeFlowInput, UpdateNodeFlowRunInput, } from "../contracts/node-flow-types.js"; +import { migratePersistedNodeFlowGraphs } from "./db/app-db-migrations.js"; interface NodeFlowRow { id: string; @@ -96,6 +97,7 @@ export class NodeFlowRepository { private readonly realtimeNotifier?: DashboardRealtimeMutationNotifier, ) { this.db = storage.getDatabase(); + migratePersistedNodeFlowGraphs(this.db); } listFlows(projectId: string): NodeFlowRecord[] { diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index 3ff01dbd70..d99082ef0b 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -1,5 +1,6 @@ import { randomUUID } from "crypto"; import { normalizeNodeFlowGraph } from "../domain/node-flows/node-flow-validation.js"; +import { resolveNodeDefinition } from "../domain/node-flows/node-definition-registry.js"; import { ValidationError, EntityNotFoundError } from "../repositories/repository-utils.js"; import { DEFAULT_DASHBOARD_SETTINGS } from "../repositories/settings-defaults.js"; import type { NodeFlowRepository } from "../repositories/node-flow-repository.js"; @@ -26,7 +27,6 @@ import type { RunNodeFlowOptions, } from "../contracts/node-flow-types.js"; -const SUPPORTED_NODE_TYPES = new Set(["input", "set_fields", "template", "provider_prompt", "http_request", "output"]); const EXTERNALLY_OBSERVABLE_NODE_TYPES = new Set(["provider_prompt", "http_request"]); const CLI_PROVIDER_IDS = new Set(["gemini", "codex", "claude-code", "qwen-code", "opencode", "antigravity", "mockup-cli"]); const SECRET_KEY_PATTERN = /(api[_-]?key|authorization|cookie|password|secret|token)/i; @@ -148,6 +148,10 @@ export class NodeFlowRuntimeService { await this.persistSkippedNode(context, node, "skipped", "Skipped because an upstream node failed."); continue; } + if (node.disabled) { + await this.persistSkippedNode(context, node, "skipped", "Skipped because the node is disabled."); + continue; + } const nodeRun = this.deps.nodeFlowRepository.createNodeRun({ runId: run.id, @@ -234,7 +238,10 @@ export class NodeFlowRuntimeService { } private requireSupportedNodes(graph: NodeFlowGraph): void { - const unsupported = graph.nodes.filter((node) => !SUPPORTED_NODE_TYPES.has(node.type)); + const unsupported = graph.nodes.filter((node) => { + const reference = node.definition ?? { type: node.type, version: 1 }; + return resolveNodeDefinition(reference.type, reference.version)?.executable !== true; + }); if (unsupported.length > 0) { throw new ValidationError(`Unsupported node flow node type: ${unsupported[0]!.type}.`); } diff --git a/tests/backend/domain/node-flows/node-definition-registry.test.ts b/tests/backend/domain/node-flows/node-definition-registry.test.ts new file mode 100644 index 0000000000..fd4fdefe02 --- /dev/null +++ b/tests/backend/domain/node-flows/node-definition-registry.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { listNodeDefinitions, resolveNodeDefinition } from "../../../../src/domain/node-flows/node-definition-registry.js"; + +describe("node definition registry", () => { + it("registers only the six implemented executable node types", () => { + expect(listNodeDefinitions().filter((definition) => definition.executable).map((definition) => definition.type)).toEqual([ + "input", "set_fields", "template", "provider_prompt", "http_request", "output", + ]); + expect(resolveNodeDefinition("condition", 1)).toBeNull(); + expect(resolveNodeDefinition("http_request", 1)).toMatchObject({ + executionKind: "http", + sideEffect: "external", + capabilities: ["network.http"], + defaultPolicy: { timeout: { timeoutMs: 30_000 } }, + }); + }); +}); diff --git a/tests/backend/domain/node-flows/node-flow-migrators.test.ts b/tests/backend/domain/node-flows/node-flow-migrators.test.ts new file mode 100644 index 0000000000..5ee0b0727a --- /dev/null +++ b/tests/backend/domain/node-flows/node-flow-migrators.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { migrateNodeCanvasGraphV1, migrateNodeFlowGraph } from "../../../../src/domain/node-flows/node-flow-migrators.js"; + +describe("node flow migrators", () => { + it("deterministically migrates backend v1 while retaining an untouched snapshot", () => { + const legacy = { nodes: [{ id: "start", type: "input", title: "Start" }], edges: [] }; + const first = migrateNodeFlowGraph(legacy); + const second = migrateNodeFlowGraph(legacy); + + expect(first).toEqual(second); + expect(first.legacySnapshot).toEqual(legacy); + expect(first.graph).toMatchObject({ + schemaVersion: 2, + nodes: [{ definition: { type: "input", version: 1 }, disabled: false, sideEffect: "none" }], + }); + expect(legacy).toEqual({ nodes: [{ id: "start", type: "input", title: "Start" }], edges: [] }); + }); + + it("migrates a browser v1 canvas into the canonical graph without embedding its snapshot", () => { + const legacy = { + nodes: [{ id: "trigger-1", kind: "trigger", label: "Trigger", description: "Draft", position: { x: 1, y: 2 }, inputPorts: [], outputPorts: [{ id: "event" }], config: [], metadata: {} }], + edges: [], selection: { nodeIds: ["trigger-1"], edgeIds: [] }, + }; + const result = migrateNodeCanvasGraphV1(legacy); + + expect(result.legacySnapshot).toEqual(legacy); + expect(result.graph.schemaVersion).toBe(2); + expect(result.graph.metadata).toEqual({ canvasSelection: legacy.selection }); + expect(JSON.stringify(result.graph)).not.toContain("legacySnapshot"); + }); +}); diff --git a/tests/backend/domain/node-flows/node-flow-validation.test.ts b/tests/backend/domain/node-flows/node-flow-validation.test.ts index c69d912574..2e56cc6bbb 100644 --- a/tests/backend/domain/node-flows/node-flow-validation.test.ts +++ b/tests/backend/domain/node-flows/node-flow-validation.test.ts @@ -15,8 +15,8 @@ const validGraph = (): NodeFlowGraph => ({ ], }, nodes: [ - { id: "start", type: "prompt", title: "Start" }, - { id: "finish", type: "agent", title: "Finish" }, + { id: "start", type: "input", title: "Start" }, + { id: "finish", type: "output", title: "Finish" }, ], edges: [ { fromNodeId: "start", toNodeId: "finish" }, @@ -43,7 +43,7 @@ describe("node flow validation", () => { it("rejects duplicate node ids and invalid edge endpoints", () => { const graph = validGraph(); - graph.nodes.push({ id: "start", type: "agent", title: "Duplicate" }); + graph.nodes.push({ id: "start", type: "output", title: "Duplicate" }); graph.edges.push({ fromNodeId: "missing", toNodeId: "finish" }); const result = validateNodeFlowGraph(graph); @@ -81,4 +81,35 @@ describe("node flow validation", () => { "invalid_select_default", ])); }); + + it("rejects unknown definitions, invalid policies, and unsafe graph values with field paths", () => { + const graph = validGraph(); + graph.nodes[0] = { + ...graph.nodes[0]!, + definition: { type: "planned_only", version: 1 }, + policy: { retry: { maxAttempts: 0, backoffMs: -1 }, timeout: { timeoutMs: 0 } }, + data: { generatedSource: "not allowed" }, + }; + + const result = validateNodeFlowGraph(graph); + + expect(result.errors).toEqual(expect.arrayContaining([ + expect.objectContaining({ field: "nodes[0].definition", code: "unknown_node_definition" }), + expect.objectContaining({ field: "nodes[0].policy.retry.maxAttempts", code: "invalid_retry_policy" }), + expect.objectContaining({ field: "nodes[0].policy.timeout.timeoutMs", code: "invalid_timeout_policy" }), + expect.objectContaining({ field: "nodes[0].data", code: "unsafe_graph_data" }), + ])); + }); + + it("orders independent nodes deterministically by stable id", () => { + const graph: NodeFlowGraph = { + nodes: [ + { id: "z", type: "input", title: "Z" }, + { id: "a", type: "input", title: "A" }, + ], + edges: [], + }; + + expect(validateNodeFlowGraph(graph).executionOrder).toEqual(["a", "z"]); + }); }); diff --git a/tests/backend/repositories/node-flow-repository.test.ts b/tests/backend/repositories/node-flow-repository.test.ts index a12202b9b0..af2b020b71 100644 --- a/tests/backend/repositories/node-flow-repository.test.ts +++ b/tests/backend/repositories/node-flow-repository.test.ts @@ -42,6 +42,24 @@ afterEach(async () => { }); describe("NodeFlowRepository", () => { + it("persists a deterministic v2 migration as a new version and preserves v1 history", async () => { + const { dir, storage, projectRepository, nodeFlowRepository } = await createRepositories(); + const project = projectRepository.createProject({ name: "Migration fixture", sourceType: "local", sourceRef: dir }); + const flow = nodeFlowRepository.createFlow(project.id, { title: "Legacy flow", graph: graph() }); + const original = JSON.stringify(graph()); + storage.getDatabase().prepare("UPDATE node_flows SET graph_json = ?, version = 1 WHERE id = ?").run(original, flow.id); + storage.getDatabase().prepare("UPDATE node_flow_versions SET graph_json = ? WHERE flow_id = ? AND version = 1").run(original, flow.id); + + const migratedRepository = new NodeFlowRepository(storage); + const migrated = migratedRepository.getFlow(flow.id); + const versions = migratedRepository.listVersions(flow.id); + + expect(migrated).toMatchObject({ version: 2, graph: { schemaVersion: 2 } }); + expect(versions.map((version) => version.version)).toEqual([2, 1]); + expect(versions[1]?.graph).toEqual(graph()); + expect(new NodeFlowRepository(storage).getFlow(flow.id)).toEqual(migrated); + }); + it("creates, updates, versions, lists, and deletes node flows", async () => { const { dir, projectRepository, nodeFlowRepository } = await createRepositories(); const project = projectRepository.createProject({ diff --git a/tests/backend/services/node-flow-service.test.ts b/tests/backend/services/node-flow-service.test.ts index 1baad7dc66..1560a2d47b 100644 --- a/tests/backend/services/node-flow-service.test.ts +++ b/tests/backend/services/node-flow-service.test.ts @@ -28,7 +28,7 @@ async function createService(): Promise<{ const validGraph = (): NodeFlowGraph => ({ nodes: [ { id: "input", type: "input", title: "Input" }, - { id: "agent", type: "agent", title: "Agent" }, + { id: "agent", type: "output", title: "Output" }, ], edges: [{ fromNodeId: "input", toNodeId: "agent" }], }); diff --git a/tests/dashboard/lib/nodes-canvas-state.test.ts b/tests/dashboard/lib/nodes-canvas-state.test.ts index 01ae403b9e..be3bb89561 100644 --- a/tests/dashboard/lib/nodes-canvas-state.test.ts +++ b/tests/dashboard/lib/nodes-canvas-state.test.ts @@ -4,6 +4,7 @@ import { createInitialNodeCanvasGraph, deserializeNodeCanvasGraph, layoutNodeCanvasGraph, + migrateNodeCanvasGraph, nodesCanvasReducer, serializeNodeCanvasGraph, validateNodeCanvasGraph, @@ -196,6 +197,20 @@ describe("nodes canvas state", () => { expect(recovered.selection).toEqual({ nodeIds: ["agent-9"], edgeIds: ["edge-1"] }); }); + it("migrates legacy canvas snapshots deterministically without embedding the original", () => { + const current = createInitialNodeCanvasGraph(); + const { schemaVersion: _schemaVersion, ...legacy } = current; + const first = migrateNodeCanvasGraph(legacy); + const second = migrateNodeCanvasGraph(legacy); + + expect(first).toEqual(second); + expect(first.migrated).toBe(true); + expect(first.graph.schemaVersion).toBe(2); + expect(first.legacySnapshot).toEqual(legacy); + expect(JSON.stringify(first.graph)).not.toContain("legacySnapshot"); + expect(migrateNodeCanvasGraph(first.graph)).toMatchObject({ migrated: false, legacySnapshot: null }); + }); + it("lays out graphs deterministically from ids and edges", () => { const graph = { ...createInitialNodeCanvasGraph(), From 83fa6f61cc14036e4702223d33bee90ea26815d1 Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 04:27:21 +0000 Subject: [PATCH 02/25] feat(task T02): implement via codex --- .../settings/AutomationCredentialManager.tsx | 21 +++++ .../AutomationCredentialManager.test.tsx | 15 ++++ .../panels/SettingsIntegrationsPanel.tsx | 2 + .../automation-credential-api.test.ts | 5 ++ .../src/v2/lib/automation-credential-api.ts | 15 ++++ .../docs/operations-credential-security.mdx | 26 ++++++ docs-web/content/docs/registry.ts | 9 ++ docs-web/operations/credential-security.md | 26 ++++++ ...cs.operations-credential-security.lazy.tsx | 11 +++ docs/SUMMARY.md | 1 + docs/index.md | 2 + docs/operations/credential-security.md | 30 +++++++ src/app/dependency-factory/core-factory.ts | 17 ++++ .../dependency-factory/dashboard-factory.ts | 3 + src/contracts/automation-credential-types.ts | 86 ++++++++++++++++++ src/electron/credential-key-persistence.ts | 23 +++++ src/electron/main.ts | 9 +- .../electron-safe-storage-key-provider.ts | 43 +++++++++ .../security/encrypted-sqlite-secret-store.ts | 21 +++++ .../external-key-provider-adapters.ts | 22 +++++ .../security/mounted-key-file-provider.ts | 38 ++++++++ .../automation-credential-repository.ts | 88 +++++++++++++++++++ src/repositories/db/app-db-migrations.ts | 84 ++++++++++++++++++ src/server/automation-credential-routes.ts | 21 +++++ src/server/dashboard-route-registration.ts | 2 + src/server/dashboard-server.ts | 2 + src/services/credentials/credential-broker.ts | 88 +++++++++++++++++++ src/services/credentials/encryption-utils.ts | 63 +++++++++++++ .../credentials/key-provider-registry.ts | 13 +++ src/services/credentials/key-provider.ts | 21 +++++ src/services/credentials/secret-store.ts | 23 +++++ src/services/node-flow-runtime-service.ts | 15 +++- .../automation-credential-repository.test.ts | 20 +++++ .../automation-credential-routes.test.ts | 8 ++ .../services/credential-encryption.test.ts | 12 +++ .../node-flow-runtime-service.test.ts | 11 ++- 36 files changed, 891 insertions(+), 5 deletions(-) create mode 100644 dashboard/src/v2/components/settings/AutomationCredentialManager.tsx create mode 100644 dashboard/src/v2/components/settings/__tests__/AutomationCredentialManager.test.tsx create mode 100644 dashboard/src/v2/lib/__tests__/automation-credential-api.test.ts create mode 100644 dashboard/src/v2/lib/automation-credential-api.ts create mode 100644 docs-web/content/docs/operations-credential-security.mdx create mode 100644 docs-web/operations/credential-security.md create mode 100644 docs-web/routes/docs.operations-credential-security.lazy.tsx create mode 100644 docs/operations/credential-security.md create mode 100644 src/contracts/automation-credential-types.ts create mode 100644 src/electron/credential-key-persistence.ts create mode 100644 src/infrastructure/security/electron-safe-storage-key-provider.ts create mode 100644 src/infrastructure/security/encrypted-sqlite-secret-store.ts create mode 100644 src/infrastructure/security/external-key-provider-adapters.ts create mode 100644 src/infrastructure/security/mounted-key-file-provider.ts create mode 100644 src/repositories/automation-credential-repository.ts create mode 100644 src/server/automation-credential-routes.ts create mode 100644 src/services/credentials/credential-broker.ts create mode 100644 src/services/credentials/encryption-utils.ts create mode 100644 src/services/credentials/key-provider-registry.ts create mode 100644 src/services/credentials/key-provider.ts create mode 100644 src/services/credentials/secret-store.ts create mode 100644 tests/backend/repositories/automation-credential-repository.test.ts create mode 100644 tests/backend/server/automation-credential-routes.test.ts create mode 100644 tests/backend/services/credential-encryption.test.ts diff --git a/dashboard/src/v2/components/settings/AutomationCredentialManager.tsx b/dashboard/src/v2/components/settings/AutomationCredentialManager.tsx new file mode 100644 index 0000000000..607c7c34f9 --- /dev/null +++ b/dashboard/src/v2/components/settings/AutomationCredentialManager.tsx @@ -0,0 +1,21 @@ +import type { FunctionComponent } from "preact"; +import { useEffect, useState } from "preact/hooks"; +import { KeyRound, Plus, RefreshCw, ShieldAlert } from "lucide-preact"; +import type { AutomationCredentialMetadata, CredentialBackendHealth } from "../../../../../src/contracts/automation-credential-types.js"; +import { createAutomationCredential, fetchAutomationCredentials, fetchCredentialHealth, revokeAutomationCredential, testAutomationCredential } from "../../lib/automation-credential-api.js"; + +export const AutomationCredentialManager:FunctionComponent<{projectId:string}> = ({projectId}) => { + const [credentials,setCredentials]=useState([]); const [health,setHealth]=useState(null); + const [name,setName]=useState(""); const [kind,setKind]=useState(""); const [value,setValue]=useState(""); const [busy,setBusy]=useState(false); const [error,setError]=useState(null); + const load=async()=>{setError(null);try{const [nextCredentials,nextHealth]=await Promise.all([fetchAutomationCredentials(projectId),fetchCredentialHealth()]);setCredentials(nextCredentials);setHealth(nextHealth);}catch(caught){setError(caught instanceof Error?caught.message:String(caught));}}; + useEffect(()=>{void load();},[projectId]); + const create=async()=>{setBusy(true);setError(null);try{await createAutomationCredential(projectId,{name,kind,value,scope:"project",capabilities:["read"]});setName("");setKind("");setValue("");await load();}catch(caught){setError(caught instanceof Error?caught.message:String(caught));}finally{setBusy(false);}}; + return
+

Automation credentials

Values are write-only and encrypted before local persistence.

+ {health&&!health.available?
{health.reason??"Secure key storage is unavailable. Credential writes are disabled."}
:null} + {error?
{error}
:null} +
+ +
    {credentials.map((credential)=>
  • {credential.name}
    {credential.kind} · {credential.scope} · {credential.status} · v{credential.version}
  • )}
+
; +}; diff --git a/dashboard/src/v2/components/settings/__tests__/AutomationCredentialManager.test.tsx b/dashboard/src/v2/components/settings/__tests__/AutomationCredentialManager.test.tsx new file mode 100644 index 0000000000..873c537554 --- /dev/null +++ b/dashboard/src/v2/components/settings/__tests__/AutomationCredentialManager.test.tsx @@ -0,0 +1,15 @@ +// @vitest-environment jsdom +import { render,screen,waitFor } from "@testing-library/preact"; +import { beforeEach,describe,expect,it,vi } from "vitest"; +import { AutomationCredentialManager } from "../AutomationCredentialManager.js"; +import { fetchAutomationCredentials,fetchCredentialHealth } from "../../../lib/automation-credential-api.js"; + +vi.mock("../../../lib/automation-credential-api.js",()=>({ + fetchAutomationCredentials:vi.fn(), fetchCredentialHealth:vi.fn(), createAutomationCredential:vi.fn(), + testAutomationCredential:vi.fn(), revokeAutomationCredential:vi.fn(), +})); + +describe("AutomationCredentialManager",()=>{ + beforeEach(()=>{vi.clearAllMocks();vi.mocked(fetchAutomationCredentials).mockResolvedValue([{id:"credential-1",name:"Deployment token",kind:"api-token",scope:"project",projectId:"project-1",allowedProjectIds:[],capabilities:["read"],status:"active",configured:true,keyId:"root",keyVersion:1,version:1,lastValidatedAt:null,validationStatus:"untested",createdAt:"now",updatedAt:"now"}]);vi.mocked(fetchCredentialHealth).mockResolvedValue({available:false,secure:false,provider:"electron-safe-storage",keyId:null,keyVersion:null,reason:"OS secure storage is unavailable."});}); + it("renders metadata and disables secret writes when secure storage is unavailable",async()=>{render();expect(await screen.findByText("Deployment token")).toBeTruthy();expect(screen.getByRole("alert").textContent).toContain("OS secure storage is unavailable.");await waitFor(()=>expect((screen.getByRole("button",{name:"Store credential"}) as HTMLButtonElement).disabled).toBe(true));expect(document.body.textContent).not.toContain("plain-secret");}); +}); diff --git a/dashboard/src/v2/components/settings/panels/SettingsIntegrationsPanel.tsx b/dashboard/src/v2/components/settings/panels/SettingsIntegrationsPanel.tsx index fe0ee70660..34d745b166 100644 --- a/dashboard/src/v2/components/settings/panels/SettingsIntegrationsPanel.tsx +++ b/dashboard/src/v2/components/settings/panels/SettingsIntegrationsPanel.tsx @@ -49,6 +49,7 @@ import type { } from "../../../lib/chat-provider-api.js"; import { isDeprecatedProvider, providerLifecycle } from "../../../lib/provider-lifecycle.js"; import { LocalFilePickerField } from "../LocalFilePickerField.js"; +import { AutomationCredentialManager } from "../AutomationCredentialManager.js"; type PublicProviderId = Exclude; @@ -1823,6 +1824,7 @@ export const SettingsIntegrationsPanel: FunctionComponent<{ state: SettingsPageS return (
+ {state.selectedProject?.id ? : null} ({fetchJson:vi.fn()})); +describe("automation credential api",()=>{beforeEach(()=>vi.clearAllMocks());it("uses write-only create and rotate endpoints",async()=>{vi.mocked(fetchJson).mockResolvedValue({});await createAutomationCredential("project/one",{name:"Token",kind:"api-token",value:"secret"});expect(fetchJson).toHaveBeenCalledWith("/api/projects/project%2Fone/credentials",expect.objectContaining({method:"POST",body:JSON.stringify({name:"Token",kind:"api-token",value:"secret"})}));await rotateAutomationCredential("project/one","credential/one","next");expect(fetchJson).toHaveBeenLastCalledWith("/api/projects/project%2Fone/credentials/credential%2Fone/rotate",expect.objectContaining({method:"POST",body:JSON.stringify({value:"next"})}));});}); diff --git a/dashboard/src/v2/lib/automation-credential-api.ts b/dashboard/src/v2/lib/automation-credential-api.ts new file mode 100644 index 0000000000..2c5b4d1694 --- /dev/null +++ b/dashboard/src/v2/lib/automation-credential-api.ts @@ -0,0 +1,15 @@ +import type { AutomationCredentialBinding, AutomationCredentialMetadata, CreateAutomationCredentialInput, CredentialBackendHealth } from "../../../../src/contracts/automation-credential-types.js"; +import { fetchJson } from "../../lib/api/fetch-json.js"; + +const json = (method: string, body?: unknown): RequestInit => ({ method, headers: { "Content-Type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); +const base = (projectId: string) => `/api/projects/${encodeURIComponent(projectId)}/credentials`; +export const fetchCredentialHealth = (): Promise => fetchJson("/api/credentials/health"); +export const fetchAutomationCredentials = (projectId:string):Promise => fetchJson(base(projectId)); +export const createAutomationCredential = (projectId:string,input:CreateAutomationCredentialInput):Promise => fetchJson(base(projectId),json("POST",input)); +export const bindAutomationCredential = (projectId:string,id:string,input:{bindingKey:string;capabilities:string[]}):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/bind`,json("POST",input)); +export const testAutomationCredential = (projectId:string,id:string):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/test`,json("POST")); +export const rotateAutomationCredential = (projectId:string,id:string,value:string):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/rotate`,json("POST",{value})); +export const replaceAutomationCredential = (projectId:string,id:string,value:string):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/replace`,json("POST",{value})); +export const revokeAutomationCredential = (projectId:string,id:string):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/revoke`,json("POST")); +export const promoteAutomationCredential = (projectId:string,id:string,allowedProjectIds:string[]):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/promote`,json("POST",{allowedProjectIds})); +export const restrictAutomationCredential = (projectId:string,id:string,input:{allowedProjectIds:string[];capabilities:string[]}):Promise => fetchJson(`${base(projectId)}/${encodeURIComponent(id)}/restrict`,json("POST",input)); diff --git a/docs-web/content/docs/operations-credential-security.mdx b/docs-web/content/docs/operations-credential-security.mdx new file mode 100644 index 0000000000..e76d076cfc --- /dev/null +++ b/docs-web/content/docs/operations-credential-security.mdx @@ -0,0 +1,26 @@ +# Automation Credential Security + +Code UX resolves canonical node credential IDs and named project binding keys through the credential broker. Stored values are not exposed to nodes, dashboard reads, MCP payloads, agent context, run inspection records, or access audits. + +## Scope and policy + +- Project credentials are owned by one project. +- Global credentials require an explicit project allowlist. +- Both the binding and credential must approve the requested capability. +- Revoked, unavailable, missing, cross-project, or insufficiently capable credentials fail closed. + +Create, rotate, and replace requests are write-only. API responses contain configuration and status metadata but never stored values. + +## Encryption and key custody + +The SQLite secret store uses AES-256-GCM envelope encryption with a unique data key, payload nonce, and key-wrapping nonce for every write. Credential ownership and workspace context are authenticated. SQLite stores ciphertext, authentication tags, wrapped keys, nonces, and key identifiers/versions—not root keys. + +Headless mode requires `CODE_UX_CREDENTIAL_KEY_FILE` to point to a mounted file containing a base64- or hexadecimal-encoded 32-byte key. Electron uses the OS `safeStorage` boundary. Vault and KMS adapters report explicit health states. If secure key material is unavailable, credential operations fail closed; there is no plaintext fallback. + +## Recovery and rotation + +Back up root keys separately from `app.db`; the database alone cannot recover credentials. Rotation creates a fresh encrypted envelope, increments the credential version, and records metadata about the transition. Revocation prevents subsequent resolution while retaining audit metadata. + +## Dashboard API + +Credential management uses project-scoped dashboard routes. List, health, and mutation responses return metadata only. Secret values are accepted only by create, rotate, and replace operations. diff --git a/docs-web/content/docs/registry.ts b/docs-web/content/docs/registry.ts index 0b0792112c..522ed80858 100644 --- a/docs-web/content/docs/registry.ts +++ b/docs-web/content/docs/registry.ts @@ -103,6 +103,7 @@ export type DocsSlug = | 'architecture-external-chat-providers' | 'architecture-configuration-resolution' | 'architecture-security' + | 'operations-credential-security' | 'settings-google-drive-mount' | 'user-dashboard-custom-dashboards' | 'architecture-custom-dashboard-foundation' @@ -822,6 +823,13 @@ export const docsRegistry: Record = { title: "Security model", description: "Code UX is designed to run as a single-user trusted process on a developer's workstation or a dedicated server. This page documents what is and is not protected, the threat model, and the recommended deployment posture.", }, + 'operations-credential-security': { + id: 'operations-credential-security', + path: '/docs/operations-credential-security', + section: 'User Guide', + title: "Automation Credential Security", + description: "Code UX resolves canonical node credential IDs and named project binding keys through the credential broker. Stored values are not exposed to nodes, dashboard reads, MCP payloads, agent context, run inspection records...", + }, 'settings-google-drive-mount': { id: 'settings-google-drive-mount', path: '/docs/settings-google-drive-mount', @@ -988,6 +996,7 @@ export const orderedDocs: DocsRegistryEntry[] = [ docsRegistry['architecture-external-chat-providers'], docsRegistry['architecture-configuration-resolution'], docsRegistry['architecture-security'], + docsRegistry['operations-credential-security'], docsRegistry['settings-google-drive-mount'], docsRegistry['user-dashboard-custom-dashboards'], docsRegistry['architecture-custom-dashboard-foundation'], diff --git a/docs-web/operations/credential-security.md b/docs-web/operations/credential-security.md new file mode 100644 index 0000000000..e76d076cfc --- /dev/null +++ b/docs-web/operations/credential-security.md @@ -0,0 +1,26 @@ +# Automation Credential Security + +Code UX resolves canonical node credential IDs and named project binding keys through the credential broker. Stored values are not exposed to nodes, dashboard reads, MCP payloads, agent context, run inspection records, or access audits. + +## Scope and policy + +- Project credentials are owned by one project. +- Global credentials require an explicit project allowlist. +- Both the binding and credential must approve the requested capability. +- Revoked, unavailable, missing, cross-project, or insufficiently capable credentials fail closed. + +Create, rotate, and replace requests are write-only. API responses contain configuration and status metadata but never stored values. + +## Encryption and key custody + +The SQLite secret store uses AES-256-GCM envelope encryption with a unique data key, payload nonce, and key-wrapping nonce for every write. Credential ownership and workspace context are authenticated. SQLite stores ciphertext, authentication tags, wrapped keys, nonces, and key identifiers/versions—not root keys. + +Headless mode requires `CODE_UX_CREDENTIAL_KEY_FILE` to point to a mounted file containing a base64- or hexadecimal-encoded 32-byte key. Electron uses the OS `safeStorage` boundary. Vault and KMS adapters report explicit health states. If secure key material is unavailable, credential operations fail closed; there is no plaintext fallback. + +## Recovery and rotation + +Back up root keys separately from `app.db`; the database alone cannot recover credentials. Rotation creates a fresh encrypted envelope, increments the credential version, and records metadata about the transition. Revocation prevents subsequent resolution while retaining audit metadata. + +## Dashboard API + +Credential management uses project-scoped dashboard routes. List, health, and mutation responses return metadata only. Secret values are accepted only by create, rotate, and replace operations. diff --git a/docs-web/routes/docs.operations-credential-security.lazy.tsx b/docs-web/routes/docs.operations-credential-security.lazy.tsx new file mode 100644 index 0000000000..1d0ade949b --- /dev/null +++ b/docs-web/routes/docs.operations-credential-security.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import OperationsCredentialSecurityContent from '../content/docs/operations-credential-security.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/operations-credential-security')({ + component: () => ( + + + + ) +}) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 87f1e6ec11..5e0a632d23 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -105,6 +105,7 @@ - [Operations Runbook](./operations/runbook.md) - [Secure Headless Server Mode](./operations/server-mode.md) - [Security Hardening](./operations/security-hardening.md) +- [Automation Credential Security](./operations/credential-security.md) - [Logging and Correlation IDs](./operations/logging-and-correlation.md) - [CLI Commands Reference](./reference/cli-commands.md) - `codeux` management syntax, aliases, prompting, JSON payloads, and approvals - [CLI Management Surface](./operations/management-cli.md) diff --git a/docs/index.md b/docs/index.md index 2a2646bfb4..201c31ab40 100644 --- a/docs/index.md +++ b/docs/index.md @@ -104,6 +104,7 @@ Use this page as the main entrypoint. 6. [Secure Headless Server Mode](./operations/server-mode.md) 7. [Operations Runbook](./operations/runbook.md) 8. [Security Hardening](./operations/security-hardening.md) +9. [Automation Credential Security](./operations/credential-security.md) 9. [Logging and Correlation IDs](./operations/logging-and-correlation.md) 10. [CLI Commands Reference](./reference/cli-commands.md) - `codeux` management syntax, aliases, prompting, JSON payloads, and approvals 11. [CLI Management Surface](./operations/management-cli.md) @@ -211,6 +212,7 @@ Use this page as the main entrypoint. - [Operations Runbook](./operations/runbook.md) - [Secure Headless Server Mode](./operations/server-mode.md) - [Security Hardening](./operations/security-hardening.md) +- [Automation Credential Security](./operations/credential-security.md) - [Logging and Correlation IDs](./operations/logging-and-correlation.md) - [CLI Commands Reference](./reference/cli-commands.md) - `codeux` management syntax, aliases, prompting, JSON payloads, and approvals - [CLI Management Surface](./operations/management-cli.md) diff --git a/docs/operations/credential-security.md b/docs/operations/credential-security.md new file mode 100644 index 0000000000..067fe67fc7 --- /dev/null +++ b/docs/operations/credential-security.md @@ -0,0 +1,30 @@ +# Automation Credential Security + +Code UX stores automation credentials through a broker rather than exposing secret values to node definitions, dashboard reads, MCP payloads, agent context, or run inspection records. Canonical node bindings reference credential metadata by ID; only the broker can resolve the value at execution time after project and capability checks. Named project binding keys use the same broker for other automation consumers. + +## Scope and policy + +- Project credentials can be managed only through their owning project. +- Global credentials are opt-in and require an explicit project allowlist containing the configuring project. +- Resolution succeeds only when both the credential and binding approve the requested capability. +- Revoked, unavailable, missing, cross-project, or insufficiently capable credentials fail closed. + +The dashboard accepts secret values only on create, rotate, and replace requests. Responses contain configuration, scope, status, key-version, and validation metadata but never stored values. Access-event rows contain identifiers, binding keys, capabilities, outcomes, and denial reasons; they never contain secret material. + +## Encryption and key custody + +The SQLite secret store uses AES-256-GCM envelope encryption. Each write generates a unique 256-bit data key, payload nonce, and key-wrapping nonce. Credential ownership and workspace context are authenticated as additional data. SQLite stores only ciphertext, authentication tags, wrapped keys, nonces, and key identifiers/versions. + +Root keys are never stored in SQLite. Headless mode requires `CODE_UX_CREDENTIAL_KEY_FILE` to identify a mounted file whose contents decode from base64 or hexadecimal to exactly 32 bytes. The environment variable contains a path, not key material. Keep the mount readable only by the Code UX process and outside the project workspace. + +Electron uses the OS-backed `safeStorage` boundary and refuses credential operations when secure OS storage is unavailable. Vault and KMS adapters expose explicit health states. No provider silently falls back to plaintext or an insecure locally derived key. + +## Recovery and rotation + +Back up root keys independently from `app.db`. Losing a required key version makes its ciphertext unrecoverable by design. Restoring only SQLite is insufficient. + +Credential rotation writes a fresh envelope with a new data key and nonces, increments the credential version, and records metadata about the transition. Root-key providers must retain old key IDs and versions until envelopes are rewrapped. Revocation prevents resolution immediately while preserving audit metadata. + +## API surface + +Project-scoped routes live under `/api/projects/:projectId/credentials`. Supported operations are create, bind, test, rotate, replace, revoke, promote, and restrict. List and health endpoints return metadata only. Existing dashboard authentication and middleware apply before these routes. diff --git a/src/app/dependency-factory/core-factory.ts b/src/app/dependency-factory/core-factory.ts index 8e9b04cd42..55108c588e 100644 --- a/src/app/dependency-factory/core-factory.ts +++ b/src/app/dependency-factory/core-factory.ts @@ -66,6 +66,11 @@ import { SprintFileBrowserRepository } from "../../repositories/sprint-file-brow import { DockerService } from "../../services/docker-service.js"; import { CustomDashboardRepository } from "../../repositories/custom-dashboard-repository.js"; import { CustomDashboardValidationService } from "../../services/custom-dashboard-validation-service.js"; +import { AutomationCredentialRepository } from "../../repositories/automation-credential-repository.js"; +import { CredentialBroker } from "../../services/credentials/credential-broker.js"; +import { MountedKeyFileProvider } from "../../infrastructure/security/mounted-key-file-provider.js"; +import { EncryptedSqliteSecretStore } from "../../infrastructure/security/encrypted-sqlite-secret-store.js"; +import { getProcessCredentialKeyProvider } from "../../services/credentials/key-provider-registry.js"; export interface CoreDependencies { providerRunner: IProviderRunner; @@ -125,6 +130,8 @@ export interface CoreDependencies { sprintFileBrowserRepository: SprintFileBrowserRepository; customDashboardRepository: CustomDashboardRepository; customDashboardValidationService: CustomDashboardValidationService; + automationCredentialRepository: AutomationCredentialRepository; + credentialBroker: CredentialBroker; } export function createCoreDependencies( @@ -180,6 +187,14 @@ export function createCoreDependencies( const subtaskRepository = new SubtaskFileRepository(); const sessionTracking = new SessionTrackingRepository(); const appDbStorage = new AppDbStorage(); + const automationCredentialRepository = new AutomationCredentialRepository(appDbStorage); + const credentialKeyProvider = getProcessCredentialKeyProvider() + ?? new MountedKeyFileProvider(process.env.CODE_UX_CREDENTIAL_KEY_FILE); + const credentialBroker = new CredentialBroker( + automationCredentialRepository, + new EncryptedSqliteSecretStore(automationCredentialRepository, credentialKeyProvider), + credentialKeyProvider, + ); const dashboardRealtimeEventRepository = new DashboardRealtimeEventRepository(appDbStorage); const dashboardRealtimeService = new DashboardRealtimeService( dashboardRealtimeEventRepository, @@ -391,5 +406,7 @@ export function createCoreDependencies( sprintFileBrowserRepository, customDashboardRepository, customDashboardValidationService, + automationCredentialRepository, + credentialBroker, }; } diff --git a/src/app/dependency-factory/dashboard-factory.ts b/src/app/dependency-factory/dashboard-factory.ts index 8c31109a1b..af70fdec29 100644 --- a/src/app/dependency-factory/dashboard-factory.ts +++ b/src/app/dependency-factory/dashboard-factory.ts @@ -34,6 +34,7 @@ import { NodeFlowService } from "../../services/node-flow-service.js"; import { resolveEffectiveDashboardSettings } from "../../services/settings-resolution-service.js"; export interface DashboardDependencies { + credentialBroker: CoreDependencies["credentialBroker"]; chatThreadRuntimeService: ChatThreadRuntimeService; chatProviderRepository: CoreDependencies["chatProviderRepository"]; chatProviderIngressService: ChatProviderIngressService; @@ -226,6 +227,7 @@ export function createDashboardDependencies( projectManagementRepository, settingsRepository, providerExecutionService, + credentialBroker: coreDeps.credentialBroker, getDashboardSettings: (projectId) => resolveDashboardSettings({ projectId }), }); const nodeFlowService = new NodeFlowService(coreDeps.nodeFlowRepository, nodeFlowRuntimeService); @@ -549,6 +551,7 @@ export function createDashboardDependencies( schedulerServiceRef.set(schedulerService); return { + credentialBroker: coreDeps.credentialBroker, chatProviderRepository, chatThreadRuntimeService, chatProviderIngressService, diff --git a/src/contracts/automation-credential-types.ts b/src/contracts/automation-credential-types.ts new file mode 100644 index 0000000000..b3136f13bb --- /dev/null +++ b/src/contracts/automation-credential-types.ts @@ -0,0 +1,86 @@ +export type AutomationCredentialScope = "project" | "global"; +export type AutomationCredentialStatus = "active" | "revoked" | "unavailable"; +export type AutomationCredentialCapability = "read" | "write" | "admin" | string; + +export interface AutomationCredentialMetadata { + id: string; + name: string; + kind: string; + scope: AutomationCredentialScope; + projectId: string | null; + allowedProjectIds: string[]; + capabilities: AutomationCredentialCapability[]; + status: AutomationCredentialStatus; + configured: boolean; + keyId: string; + keyVersion: number; + version: number; + lastValidatedAt: string | null; + validationStatus: "untested" | "valid" | "invalid" | "unavailable"; + createdAt: string; + updatedAt: string; +} + +export interface CreateAutomationCredentialInput { + name: string; + kind: string; + value: string; + scope?: AutomationCredentialScope; + allowedProjectIds?: string[]; + capabilities?: AutomationCredentialCapability[]; +} + +export interface AutomationCredentialBinding { + id: string; + credentialId: string; + projectId: string; + bindingKey: string; + requiredCapabilities: AutomationCredentialCapability[]; + createdAt: string; + updatedAt: string; +} + +export type CredentialAccessOutcome = "granted" | "denied"; +export interface AutomationCredentialAccessEvent { + id: string; + credentialId: string | null; + projectId: string; + bindingKey: string | null; + capability: string | null; + operation: string; + outcome: CredentialAccessOutcome; + reason: string | null; + createdAt: string; +} + +export interface AutomationCredentialRotation { + id: string; + credentialId: string; + fromVersion: number; + toVersion: number; + keyId: string; + keyVersion: number; + rotatedAt: string; +} + +export interface CredentialResolutionRequest { + projectId: string; + bindingKey: string; + capability: AutomationCredentialCapability; + workspaceId: string; +} + +export interface ResolvedCredential { + credentialId: string; + value: string; + version: number; +} + +export interface CredentialBackendHealth { + available: boolean; + secure: boolean; + provider: string; + keyId: string | null; + keyVersion: number | null; + reason?: string; +} diff --git a/src/electron/credential-key-persistence.ts b/src/electron/credential-key-persistence.ts new file mode 100644 index 0000000000..f8261a4610 --- /dev/null +++ b/src/electron/credential-key-persistence.ts @@ -0,0 +1,23 @@ +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import type { ProtectedKeyPersistence } from "../infrastructure/security/electron-safe-storage-key-provider.js"; + +/** Persists only the OS-encrypted root-key blob; plaintext key bytes never reach this boundary. */ +export class ElectronCredentialKeyPersistence implements ProtectedKeyPersistence { + constructor(private readonly filePath: string) {} + + async read(): Promise { + try { return await readFile(this.filePath); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + } + + async write(value: Buffer): Promise { + await mkdir(dirname(this.filePath), { recursive: true }); + const temporaryPath = `${this.filePath}.tmp`; + await writeFile(temporaryPath, value, { mode: 0o600 }); + await rename(temporaryPath, this.filePath); + } +} diff --git a/src/electron/main.ts b/src/electron/main.ts index c0c7bd4395..e9517aee04 100644 --- a/src/electron/main.ts +++ b/src/electron/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, dialog, ipcMain, nativeImage, session, shell } from "electron"; +import { app, BrowserWindow, dialog, ipcMain, nativeImage, safeStorage, session, shell } from "electron"; import * as fs from "fs"; import Module from "module"; import * as os from "os"; @@ -15,6 +15,9 @@ import { } from "./dashboard-network-policy.js"; import { openCodeUxUpdatesPage, toggleWindowMaximized } from "./window-controls.js"; import { createDebouncedSaver, loadWindowState, saveWindowState } from "./window-state.js"; +import { ElectronCredentialKeyPersistence } from "./credential-key-persistence.js"; +import { ElectronSafeStorageKeyProvider } from "../infrastructure/security/electron-safe-storage-key-provider.js"; +import { setProcessCredentialKeyProvider } from "../services/credentials/key-provider-registry.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -299,6 +302,10 @@ async function startServer(): Promise { registerPackagedNodeModules(); const dotenv = await import("dotenv"); dotenv.config({ path: path.join(projectRoot, ".env"), quiet: true }); + setProcessCredentialKeyProvider(new ElectronSafeStorageKeyProvider( + safeStorage, + new ElectronCredentialKeyPersistence(path.join(app.getPath("userData"), "credential-root-key.bin")), + )); const [{ loadAppConfig }, { CodeUxServer }] = await Promise.all([ import("../config/app-config.js"), diff --git a/src/infrastructure/security/electron-safe-storage-key-provider.ts b/src/infrastructure/security/electron-safe-storage-key-provider.ts new file mode 100644 index 0000000000..2efc89aebb --- /dev/null +++ b/src/infrastructure/security/electron-safe-storage-key-provider.ts @@ -0,0 +1,43 @@ +import { randomBytes } from "node:crypto"; +import type { CredentialBackendHealth } from "../../contracts/automation-credential-types.js"; +import { KeyProviderUnavailableError, type KeyMaterial, type KeyProvider } from "../../services/credentials/key-provider.js"; + +export interface ElectronSafeStorageBoundary { + isEncryptionAvailable(): boolean; + encryptString(value: string): Buffer; + decryptString(value: Buffer): string; +} + +export interface ProtectedKeyPersistence { + read(): Promise; + write(value: Buffer): Promise; +} + +export class ElectronSafeStorageKeyProvider implements KeyProvider { + readonly providerName = "electron-safe-storage"; + constructor(private readonly safeStorage: ElectronSafeStorageBoundary, private readonly persistence: ProtectedKeyPersistence, private readonly keyId = "electron-root", private readonly version = 1) {} + + async health(): Promise { + if (!this.safeStorage.isEncryptionAvailable()) return { available: false, secure: false, provider: this.providerName, keyId: null, keyVersion: null, reason: "OS secure storage is unavailable." }; + try { const key = await this.getActiveKey(); key.key.fill(0); return { available: true, secure: true, provider: this.providerName, keyId: this.keyId, keyVersion: this.version }; } + catch (error) { return { available: false, secure: false, provider: this.providerName, keyId: null, keyVersion: null, reason: error instanceof Error ? error.message : String(error) }; } + } + + async getActiveKey(): Promise { + if (!this.safeStorage.isEncryptionAvailable()) throw new KeyProviderUnavailableError("OS secure storage is unavailable."); + let protectedValue = await this.persistence.read(); + if (!protectedValue) { + const generated = randomBytes(32); + try { protectedValue = this.safeStorage.encryptString(generated.toString("base64")); await this.persistence.write(protectedValue); } + finally { generated.fill(0); } + } + const key = Buffer.from(this.safeStorage.decryptString(protectedValue), "base64"); + if (key.length !== 32) { key.fill(0); throw new KeyProviderUnavailableError("Protected credential root key is invalid."); } + return { key, keyId: this.keyId, version: this.version }; + } + + async getKey(keyId: string, version: number): Promise { + if (keyId !== this.keyId || version !== this.version) throw new KeyProviderUnavailableError("Requested OS-protected key version is unavailable."); + return this.getActiveKey(); + } +} diff --git a/src/infrastructure/security/encrypted-sqlite-secret-store.ts b/src/infrastructure/security/encrypted-sqlite-secret-store.ts new file mode 100644 index 0000000000..20193715aa --- /dev/null +++ b/src/infrastructure/security/encrypted-sqlite-secret-store.ts @@ -0,0 +1,21 @@ +import type { KeyProvider } from "../../services/credentials/key-provider.js"; +import type { SecretContext, SecretStore, StoredSecretEnvelope } from "../../services/credentials/secret-store.js"; +import { decryptEnvelope, encryptEnvelope } from "../../services/credentials/encryption-utils.js"; +import type { AutomationCredentialRepository } from "../../repositories/automation-credential-repository.js"; + +export class EncryptedSqliteSecretStore implements SecretStore { + constructor(private readonly repository: AutomationCredentialRepository, private readonly keyProvider: KeyProvider) {} + async put(context: SecretContext, plaintext: Buffer): Promise { + const health = await this.keyProvider.health(); + if (!health.available || !health.secure) throw new Error(health.reason ?? "Secure key provider is unavailable."); + const rootKey = await this.keyProvider.getActiveKey(); + try { const envelope=encryptEnvelope(context,plaintext,rootKey); this.repository.putEnvelope(envelope); return envelope; } + finally { rootKey.key.fill(0); } + } + async get(context: SecretContext): Promise { + const envelope=this.repository.getEnvelope(context.credentialId); if (!envelope) throw new Error("Credential secret is unavailable."); + const rootKey=await this.keyProvider.getKey(envelope.keyId,envelope.keyVersion); + try { return decryptEnvelope(context,envelope,rootKey); } finally { rootKey.key.fill(0); } + } + async delete(credentialId: string): Promise { this.repository.deleteEnvelope(credentialId); } +} diff --git a/src/infrastructure/security/external-key-provider-adapters.ts b/src/infrastructure/security/external-key-provider-adapters.ts new file mode 100644 index 0000000000..cb12b8658b --- /dev/null +++ b/src/infrastructure/security/external-key-provider-adapters.ts @@ -0,0 +1,22 @@ +import type { CredentialBackendHealth } from "../../contracts/automation-credential-types.js"; +import { KeyProviderUnavailableError, type KeyMaterial, type KeyProvider } from "../../services/credentials/key-provider.js"; + +export interface ExternalKeyServiceClient { + health(): Promise<{ available: boolean; reason?: string }>; + activeKey(): Promise; + key(keyId: string, version: number): Promise; +} + +export class ExternalKeyProviderAdapter implements KeyProvider { + constructor(readonly providerName: "vault" | "kms", private readonly client?: ExternalKeyServiceClient) {} + async health(): Promise { + if (!this.client) return { available: false, secure: true, provider: this.providerName, keyId: null, keyVersion: null, reason: `${this.providerName} integration is not configured.` }; + const health = await this.client.health(); + return { available: health.available, secure: true, provider: this.providerName, keyId: null, keyVersion: null, reason: health.reason }; + } + getActiveKey(): Promise { if (!this.client) throw new KeyProviderUnavailableError(`${this.providerName} integration is unavailable.`); return this.client.activeKey(); } + getKey(keyId: string, version: number): Promise { if (!this.client) throw new KeyProviderUnavailableError(`${this.providerName} integration is unavailable.`); return this.client.key(keyId, version); } +} + +export class VaultKeyProviderAdapter extends ExternalKeyProviderAdapter { constructor(client?: ExternalKeyServiceClient) { super("vault", client); } } +export class KmsKeyProviderAdapter extends ExternalKeyProviderAdapter { constructor(client?: ExternalKeyServiceClient) { super("kms", client); } } diff --git a/src/infrastructure/security/mounted-key-file-provider.ts b/src/infrastructure/security/mounted-key-file-provider.ts new file mode 100644 index 0000000000..d7d1061b35 --- /dev/null +++ b/src/infrastructure/security/mounted-key-file-provider.ts @@ -0,0 +1,38 @@ +import { readFile, stat } from "node:fs/promises"; +import type { CredentialBackendHealth } from "../../contracts/automation-credential-types.js"; +import { KeyProviderUnavailableError, type KeyMaterial, type KeyProvider } from "../../services/credentials/key-provider.js"; + +export class MountedKeyFileProvider implements KeyProvider { + readonly providerName = "mounted-key-file"; + constructor(private readonly filePath: string | undefined, private readonly keyId = "mounted-root", private readonly version = 1) {} + + async health(): Promise { + try { + const material = await this.read(); + material.key.fill(0); + return { available: true, secure: true, provider: this.providerName, keyId: this.keyId, keyVersion: this.version }; + } catch (error) { + const reason=error instanceof Error ? error.message : String(error); + return { available: false, secure: !reason.includes("insecure permissions"), provider: this.providerName, keyId: null, keyVersion: null, reason }; + } + } + + getActiveKey(): Promise { return this.read(); } + async getKey(keyId: string, version: number): Promise { + if (keyId !== this.keyId || version !== this.version) throw new KeyProviderUnavailableError("Requested root key version is unavailable."); + return this.read(); + } + + private async read(): Promise { + if (!this.filePath) throw new KeyProviderUnavailableError("No mounted credential key file is configured."); + try { const info=await stat(this.filePath); if ((info.mode & 0o077) !== 0) throw new KeyProviderUnavailableError("Mounted credential key file has insecure permissions; expected owner-only access."); } + catch (error) { if (error instanceof KeyProviderUnavailableError) throw error; throw new KeyProviderUnavailableError("Mounted credential key file is unavailable."); } + let raw: Buffer; + try { raw = await readFile(this.filePath); } catch { throw new KeyProviderUnavailableError("Mounted credential key file is unavailable."); } + const trimmed = raw.toString("utf8").trim(); + raw.fill(0); + const key = /^[a-f\d]{64}$/i.test(trimmed) ? Buffer.from(trimmed, "hex") : Buffer.from(trimmed, "base64"); + if (key.length !== 32) { key.fill(0); throw new KeyProviderUnavailableError("Mounted credential key must decode to exactly 32 bytes."); } + return { key, keyId: this.keyId, version: this.version }; + } +} diff --git a/src/repositories/automation-credential-repository.ts b/src/repositories/automation-credential-repository.ts new file mode 100644 index 0000000000..a37f1d0b56 --- /dev/null +++ b/src/repositories/automation-credential-repository.ts @@ -0,0 +1,88 @@ +import { randomUUID } from "node:crypto"; +import type { AutomationCredentialAccessEvent, AutomationCredentialBinding, AutomationCredentialMetadata, AutomationCredentialRotation, AutomationCredentialScope, AutomationCredentialStatus } from "../contracts/automation-credential-types.js"; +import { AppDbStorage } from "./app-db-storage.js"; +import type { DatabaseAdapter } from "./db/database-adapter.js"; +import { EntityNotFoundError, ValidationError, toNumber } from "./repository-utils.js"; +import type { StoredSecretEnvelope } from "../services/credentials/secret-store.js"; + +interface CredentialRow { id: string; name: string; kind: string; scope: AutomationCredentialScope; project_id: string | null; allowed_project_ids_json: string; capabilities_json: string; status: AutomationCredentialStatus; key_id: string; key_version: number; version: number; last_validated_at: string | null; validation_status: AutomationCredentialMetadata["validationStatus"]; created_at: string; updated_at: string; configured?: number } +interface SecretRow { credential_id: string; ciphertext: Buffer; nonce: Buffer; auth_tag: Buffer; wrapped_data_key: Buffer; wrap_nonce: Buffer; wrap_auth_tag: Buffer; key_id: string; key_version: number } +interface BindingRow { id: string; credential_id: string; project_id: string; binding_key: string; required_capabilities_json: string; created_at: string; updated_at: string } + +export class AutomationCredentialRepository { + private readonly db: DatabaseAdapter; + constructor(storage: AppDbStorage = new AppDbStorage()) { this.db = storage.getDatabase(); } + + requireProject(projectId: string): void { + if (!this.db.prepare("SELECT id FROM projects WHERE id = ?").get(projectId)) throw new EntityNotFoundError(`Project not found: ${projectId}`); + } + + list(projectId: string): AutomationCredentialMetadata[] { + this.requireProject(projectId); + const rows = this.db.prepare(`SELECT c.*, EXISTS(SELECT 1 FROM automation_credential_secrets s WHERE s.credential_id=c.id) AS configured FROM automation_credentials c WHERE c.project_id = ? OR (c.scope = 'global' AND EXISTS (SELECT 1 FROM json_each(c.allowed_project_ids_json) WHERE value = ?)) ORDER BY c.updated_at DESC`).all(projectId, projectId) as CredentialRow[]; + return rows.map((row) => this.mapCredential(row)); + } + + get(id: string): AutomationCredentialMetadata | null { + const row = this.db.prepare("SELECT c.*, EXISTS(SELECT 1 FROM automation_credential_secrets s WHERE s.credential_id=c.id) AS configured FROM automation_credentials c WHERE c.id = ?").get(id) as CredentialRow | undefined; + return row ? this.mapCredential(row) : null; + } + + create(input: { id?: string; name: string; kind: string; scope: AutomationCredentialScope; projectId: string | null; allowedProjectIds: string[]; capabilities: string[]; keyId: string; keyVersion: number }): AutomationCredentialMetadata { + if (input.scope === "project" && !input.projectId) throw new ValidationError("Project credentials require a projectId."); + if (input.scope === "global" && input.projectId) throw new ValidationError("Global credentials cannot have an owning projectId."); + if (input.projectId) this.requireProject(input.projectId); + for (const projectId of input.allowedProjectIds) this.requireProject(projectId); + const id = input.id ?? randomUUID(); const now = new Date().toISOString(); + this.db.prepare(`INSERT INTO automation_credentials (id,name,kind,scope,project_id,allowed_project_ids_json,capabilities_json,status,key_id,key_version,version,created_at,updated_at) VALUES (?,?,?,?,?,?,?,'active',?,?,1,?,?)`).run(id, input.name, input.kind, input.scope, input.projectId, JSON.stringify(input.allowedProjectIds), JSON.stringify(input.capabilities), input.keyId, input.keyVersion, now, now); + return this.get(id)!; + } + + updateSecretMetadata(id: string, keyId: string, keyVersion: number, version: number): AutomationCredentialMetadata { + const now = new Date().toISOString(); + this.db.prepare("UPDATE automation_credentials SET key_id=?, key_version=?, version=?, status='active', validation_status='untested', last_validated_at=NULL, updated_at=? WHERE id=?").run(keyId, keyVersion, version, now, id); + const result = this.get(id); if (!result) throw new EntityNotFoundError(`Credential not found: ${id}`); return result; + } + + updateStatus(id: string, status: AutomationCredentialStatus): AutomationCredentialMetadata { + this.db.prepare("UPDATE automation_credentials SET status=?, updated_at=? WHERE id=?").run(status, new Date().toISOString(), id); + const result = this.get(id); if (!result) throw new EntityNotFoundError(`Credential not found: ${id}`); return result; + } + + updateValidation(id: string, status: AutomationCredentialMetadata["validationStatus"]): AutomationCredentialMetadata { + this.db.prepare("UPDATE automation_credentials SET validation_status=?, last_validated_at=?, updated_at=? WHERE id=?").run(status, new Date().toISOString(), new Date().toISOString(), id); + const result = this.get(id); if (!result) throw new EntityNotFoundError(`Credential not found: ${id}`); return result; + } + + restrict(id: string, allowedProjectIds: string[], capabilities: string[]): AutomationCredentialMetadata { + const credential = this.get(id); if (!credential) throw new EntityNotFoundError(`Credential not found: ${id}`); + for (const projectId of allowedProjectIds) this.requireProject(projectId); + this.db.prepare("UPDATE automation_credentials SET allowed_project_ids_json=?, capabilities_json=?, updated_at=? WHERE id=?").run(JSON.stringify(allowedProjectIds), JSON.stringify(capabilities), new Date().toISOString(), id); + return this.get(id)!; + } + + promote(id: string, allowedProjectIds: string[]): AutomationCredentialMetadata { + const credential = this.get(id); if (!credential) throw new EntityNotFoundError(`Credential not found: ${id}`); + for (const projectId of allowedProjectIds) this.requireProject(projectId); + this.db.prepare("UPDATE automation_credentials SET scope='global', project_id=NULL, allowed_project_ids_json=?, updated_at=? WHERE id=?").run(JSON.stringify(allowedProjectIds), new Date().toISOString(), id); + return this.get(id)!; + } + + putEnvelope(envelope: StoredSecretEnvelope): void { + this.db.prepare(`INSERT INTO automation_credential_secrets (credential_id,ciphertext,nonce,auth_tag,wrapped_data_key,wrap_nonce,wrap_auth_tag,key_id,key_version,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(credential_id) DO UPDATE SET ciphertext=excluded.ciphertext,nonce=excluded.nonce,auth_tag=excluded.auth_tag,wrapped_data_key=excluded.wrapped_data_key,wrap_nonce=excluded.wrap_nonce,wrap_auth_tag=excluded.wrap_auth_tag,key_id=excluded.key_id,key_version=excluded.key_version,updated_at=excluded.updated_at`).run(envelope.credentialId,envelope.ciphertext,envelope.nonce,envelope.authTag,envelope.wrappedDataKey,envelope.wrapNonce,envelope.wrapAuthTag,envelope.keyId,envelope.keyVersion,new Date().toISOString()); + } + getEnvelope(credentialId: string): StoredSecretEnvelope | null { const row=this.db.prepare("SELECT * FROM automation_credential_secrets WHERE credential_id=?").get(credentialId) as SecretRow|undefined; return row ? {credentialId:row.credential_id,ciphertext:row.ciphertext,nonce:row.nonce,authTag:row.auth_tag,wrappedDataKey:row.wrapped_data_key,wrapNonce:row.wrap_nonce,wrapAuthTag:row.wrap_auth_tag,keyId:row.key_id,keyVersion:toNumber(row.key_version)} : null; } + deleteEnvelope(id: string): void { this.db.prepare("DELETE FROM automation_credential_secrets WHERE credential_id=?").run(id); } + + bind(credentialId: string, projectId: string, bindingKey: string, requiredCapabilities: string[]): AutomationCredentialBinding { + this.requireProject(projectId); const now=new Date().toISOString(); const id=randomUUID(); + this.db.prepare(`INSERT INTO automation_credential_bindings (id,credential_id,project_id,binding_key,required_capabilities_json,created_at,updated_at) VALUES (?,?,?,?,?,?,?) ON CONFLICT(project_id,binding_key) DO UPDATE SET credential_id=excluded.credential_id,required_capabilities_json=excluded.required_capabilities_json,updated_at=excluded.updated_at`).run(id,credentialId,projectId,bindingKey,JSON.stringify(requiredCapabilities),now,now); + return this.getBinding(projectId,bindingKey)!; + } + getBinding(projectId: string, bindingKey: string): AutomationCredentialBinding | null { const row=this.db.prepare("SELECT * FROM automation_credential_bindings WHERE project_id=? AND binding_key=?").get(projectId,bindingKey) as BindingRow|undefined; return row ? this.mapBinding(row):null; } + recordAccess(input: Omit): void { this.db.prepare(`INSERT INTO automation_credential_access_events (id,credential_id,project_id,binding_key,capability,operation,outcome,reason,created_at) VALUES (?,?,?,?,?,?,?,?,?)`).run(randomUUID(),input.credentialId,input.projectId,input.bindingKey,input.capability,input.operation,input.outcome,input.reason,new Date().toISOString()); } + recordRotation(input: Omit): void { this.db.prepare(`INSERT INTO automation_credential_rotations (id,credential_id,from_version,to_version,key_id,key_version,rotated_at) VALUES (?,?,?,?,?,?,?)`).run(randomUUID(),input.credentialId,input.fromVersion,input.toVersion,input.keyId,input.keyVersion,new Date().toISOString()); } + + private mapCredential(row: CredentialRow): AutomationCredentialMetadata { return {id:row.id,name:row.name,kind:row.kind,scope:row.scope,projectId:row.project_id,allowedProjectIds:JSON.parse(row.allowed_project_ids_json) as string[],capabilities:JSON.parse(row.capabilities_json) as string[],status:row.status,configured:toNumber(row.configured)===1,keyId:row.key_id,keyVersion:toNumber(row.key_version),version:toNumber(row.version),lastValidatedAt:row.last_validated_at,validationStatus:row.validation_status,createdAt:row.created_at,updatedAt:row.updated_at}; } + private mapBinding(row: BindingRow): AutomationCredentialBinding { return {id:row.id,credentialId:row.credential_id,projectId:row.project_id,bindingKey:row.binding_key,requiredCapabilities:JSON.parse(row.required_capabilities_json) as string[],createdAt:row.created_at,updatedAt:row.updated_at}; } +} diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index b874fd2de6..da475c1306 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -392,6 +392,89 @@ export function ensureCustomDashboardTables(db: DatabaseAdapter): void { ensureIndex(db, "idx_custom_dashboard_publications_project", "custom_dashboard_publications", "project_id, published_at DESC"); } +export function ensureAutomationCredentialTables(db: DatabaseAdapter): void { + db.exec(` + CREATE TABLE IF NOT EXISTS automation_credentials ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT NOT NULL, + scope TEXT NOT NULL CHECK (scope IN ('project', 'global')), + project_id TEXT, + allowed_project_ids_json TEXT NOT NULL DEFAULT '[]', + capabilities_json TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'active', + key_id TEXT NOT NULL, + key_version INTEGER NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + last_validated_at TEXT, + validation_status TEXT NOT NULL DEFAULT 'untested', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + CHECK ((scope = 'project' AND project_id IS NOT NULL) OR (scope = 'global' AND project_id IS NULL)) + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_credential_secrets ( + credential_id TEXT PRIMARY KEY, + ciphertext BLOB NOT NULL, + nonce BLOB NOT NULL, + auth_tag BLOB NOT NULL, + wrapped_data_key BLOB NOT NULL, + wrap_nonce BLOB NOT NULL, + wrap_auth_tag BLOB NOT NULL, + key_id TEXT NOT NULL, + key_version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (credential_id) REFERENCES automation_credentials(id) ON DELETE CASCADE + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_credential_bindings ( + id TEXT PRIMARY KEY, + credential_id TEXT NOT NULL, + project_id TEXT NOT NULL, + binding_key TEXT NOT NULL, + required_capabilities_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (credential_id) REFERENCES automation_credentials(id) ON DELETE CASCADE, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + UNIQUE (project_id, binding_key) + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_credential_access_events ( + id TEXT PRIMARY KEY, + credential_id TEXT, + project_id TEXT NOT NULL, + binding_key TEXT, + capability TEXT, + operation TEXT NOT NULL, + outcome TEXT NOT NULL, + reason TEXT, + created_at TEXT NOT NULL + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_credential_rotations ( + id TEXT PRIMARY KEY, + credential_id TEXT NOT NULL, + from_version INTEGER NOT NULL, + to_version INTEGER NOT NULL, + key_id TEXT NOT NULL, + key_version INTEGER NOT NULL, + rotated_at TEXT NOT NULL, + FOREIGN KEY (credential_id) REFERENCES automation_credentials(id) ON DELETE CASCADE + ) + `); + ensureIndex(db, "idx_automation_credentials_project", "automation_credentials", "project_id, status, updated_at DESC"); + ensureIndex(db, "idx_automation_credentials_global", "automation_credentials", "scope, status, updated_at DESC"); + ensureIndex(db, "idx_automation_credential_bindings_credential", "automation_credential_bindings", "credential_id, project_id"); + ensureIndex(db, "idx_automation_credential_access_events_project", "automation_credential_access_events", "project_id, created_at DESC"); + ensureIndex(db, "idx_automation_credential_rotations_credential", "automation_credential_rotations", "credential_id, rotated_at DESC"); +} + export function migrateSprintLinkedIssuesExternalSources(db: DatabaseAdapter): void { ensureColumn(db, "sprint_linked_issues", "project_key", "TEXT"); ensureColumn(db, "sprint_linked_issues", "external_id", "TEXT"); @@ -627,6 +710,7 @@ export function runMigrations(db: DatabaseAdapter): void { ensureNodeFlowTables(db); migratePersistedNodeFlowGraphs(db); ensureCustomDashboardTables(db); + ensureAutomationCredentialTables(db); ensureColumn(db, "projects", "initialization_mode", "TEXT NOT NULL DEFAULT 'existing'"); ensureColumn(db, "provider_invocations", "tool_call_count", "INTEGER NOT NULL DEFAULT 0"); diff --git a/src/server/automation-credential-routes.ts b/src/server/automation-credential-routes.ts new file mode 100644 index 0000000000..3eb0cf3b7c --- /dev/null +++ b/src/server/automation-credential-routes.ts @@ -0,0 +1,21 @@ +import type { Express } from "express"; +import type { DashboardDependencies } from "./dashboard-server.js"; +import { asyncRoute } from "./route-utils.js"; +import { requireTrimmedString } from "./request-parsers.js"; +import type { CreateAutomationCredentialInput } from "../contracts/automation-credential-types.js"; + +function broker(deps: DashboardDependencies) { if (!deps.credentialBroker) throw new Error("Credential broker is not enabled."); return deps.credentialBroker; } +const strings=(value:unknown):string[]=>Array.isArray(value)?value.filter((item):item is string=>typeof item === "string"&&item.trim().length>0).map((item)=>item.trim()):[]; + +export function registerAutomationCredentialRoutes(app: Express,deps:DashboardDependencies):void{ + app.get("/api/credentials/health",asyncRoute(async(_req,res)=>{res.json(await broker(deps).health());})); + app.get("/api/projects/:projectId/credentials",asyncRoute(async(req,res)=>{res.json(broker(deps).list(requireTrimmedString(req.params.projectId,"projectId")));})); + app.post("/api/projects/:projectId/credentials",asyncRoute(async(req,res)=>{res.status(201).json(await broker(deps).create(requireTrimmedString(req.params.projectId,"projectId"),req.body as CreateAutomationCredentialInput));})); + app.post("/api/projects/:projectId/credentials/:credentialId/bind",asyncRoute(async(req,res)=>{const body=req.body as Record;res.json(broker(deps).bind(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId"),requireTrimmedString(body.bindingKey,"bindingKey"),strings(body.capabilities)));})); + app.post("/api/projects/:projectId/credentials/:credentialId/test",asyncRoute(async(req,res)=>{res.json(await broker(deps).test(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId")));})); + app.post("/api/projects/:projectId/credentials/:credentialId/rotate",asyncRoute(async(req,res)=>{res.json(await broker(deps).rotate(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId"),requireTrimmedString((req.body as Record).value,"value")));})); + app.post("/api/projects/:projectId/credentials/:credentialId/replace",asyncRoute(async(req,res)=>{res.json(await broker(deps).replace(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId"),requireTrimmedString((req.body as Record).value,"value")));})); + app.post("/api/projects/:projectId/credentials/:credentialId/revoke",asyncRoute(async(req,res)=>{res.json(broker(deps).revoke(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId")));})); + app.post("/api/projects/:projectId/credentials/:credentialId/promote",asyncRoute(async(req,res)=>{res.json(await broker(deps).promote(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId"),strings((req.body as Record).allowedProjectIds)));})); + app.post("/api/projects/:projectId/credentials/:credentialId/restrict",asyncRoute(async(req,res)=>{const body=req.body as Record;res.json(broker(deps).restrict(requireTrimmedString(req.params.projectId,"projectId"),requireTrimmedString(req.params.credentialId,"credentialId"),strings(body.allowedProjectIds),strings(body.capabilities)));})); +} diff --git a/src/server/dashboard-route-registration.ts b/src/server/dashboard-route-registration.ts index 95e851cf0b..196a19bd7f 100644 --- a/src/server/dashboard-route-registration.ts +++ b/src/server/dashboard-route-registration.ts @@ -29,6 +29,7 @@ import { registerUpdateStatusRoutes } from "./update-status-routes.js"; import { registerMemoryRoutes } from "./memory-routes.js"; import { registerKnowledgeRoutes } from "./knowledge-routes.js"; import { registerDocsWebRoutes } from "./docs-web-routes.js"; +import { registerAutomationCredentialRoutes } from "./automation-credential-routes.js"; import { registerChatProviderRoutes } from "./chat-provider-routes.js"; import { registerChatProviderIngressRoutes } from "./chat-provider-ingress-routes.js"; import { registerSpeechRoutes } from "./speech-routes.js"; @@ -110,6 +111,7 @@ const registerPreviewRouteGroup = (app: Express, deps: DashboardDependencies): v const registerSettingsRouteGroup = (app: Express, deps: DashboardDependencies, liveActivityCacheMs: number): void => { registerSettingsRoutes(app, deps, liveActivityCacheMs); + registerAutomationCredentialRoutes(app, deps); registerChatProviderRoutes(app, deps); registerChatProviderIngressRoutes(app, deps); }; diff --git a/src/server/dashboard-server.ts b/src/server/dashboard-server.ts index bcff13a31c..2be9edfc1c 100644 --- a/src/server/dashboard-server.ts +++ b/src/server/dashboard-server.ts @@ -141,6 +141,7 @@ import type { NodeFlowService } from "../services/node-flow-service.js"; import type { CustomDashboardRepository } from "../repositories/custom-dashboard-repository.js"; import type { CustomDashboardValidationService } from "../services/custom-dashboard-validation-service.js"; import type { SkillService } from "../services/skill-service.js"; +import type { CredentialBroker } from "../services/credentials/credential-broker.js"; import type { ManagedRuntimeService } from "../services/managed-runtime-service.js"; import type { ProviderToolManager } from "../services/provider-tool-manager.js"; import { @@ -189,6 +190,7 @@ export interface DashboardServerOptions { customDashboardRepository?: CustomDashboardRepository; customDashboardValidationService?: CustomDashboardValidationService; skillService?: SkillService; + credentialBroker?: CredentialBroker; managedRuntimeService?: ManagedRuntimeService; providerToolManager?: ProviderToolManager; playwrightBrowserManager?: PlaywrightBrowserManager; diff --git a/src/services/credentials/credential-broker.ts b/src/services/credentials/credential-broker.ts new file mode 100644 index 0000000000..992262a547 --- /dev/null +++ b/src/services/credentials/credential-broker.ts @@ -0,0 +1,88 @@ +import { randomUUID } from "node:crypto"; +import type { AutomationCredentialMetadata, CreateAutomationCredentialInput, CredentialBackendHealth, CredentialResolutionRequest, ResolvedCredential } from "../../contracts/automation-credential-types.js"; +import type { AutomationCredentialRepository } from "../../repositories/automation-credential-repository.js"; +import type { KeyProvider } from "./key-provider.js"; +import type { SecretStore } from "./secret-store.js"; + +export class CredentialAccessDeniedError extends Error { + constructor(message: string) { super(message); this.name = "CredentialAccessDeniedError"; } +} + +export class CredentialBroker { + constructor(private readonly repository: AutomationCredentialRepository, private readonly secretStore: SecretStore, private readonly keyProvider: KeyProvider) {} + + health(): Promise { return this.keyProvider.health(); } + list(projectId: string): AutomationCredentialMetadata[] { return this.repository.list(projectId); } + + async create(projectId: string, input: CreateAutomationCredentialInput): Promise { + this.repository.requireProject(projectId); + const name=input.name?.trim(); const kind=input.kind?.trim(); const value=input.value; + if (!name || !kind || typeof value !== "string" || value.length === 0) throw new Error("name, kind, and a non-empty value are required."); + const scope=input.scope ?? "project"; + const allowedProjectIds=scope === "global" ? [...new Set(input.allowedProjectIds ?? [])] : []; + if (scope === "global" && !allowedProjectIds.includes(projectId)) throw new Error("Global credentials require an explicit allowlist containing the configuring project."); + const health=await this.keyProvider.health(); + if (!health.available || !health.secure || !health.keyId || health.keyVersion === null) throw new Error(health.reason ?? "Secure credential storage is unavailable."); + const id=randomUUID(); + const metadata=this.repository.create({id,name,kind,scope,projectId:scope === "project" ? projectId:null,allowedProjectIds,capabilities:[...new Set(input.capabilities ?? [])],keyId:health.keyId,keyVersion:health.keyVersion}); + const plaintext=Buffer.from(value,"utf8"); + try { await this.secretStore.put(this.context(metadata),plaintext); } + catch (error) { this.repository.updateStatus(id,"unavailable"); throw error; } + finally { plaintext.fill(0); } + return this.repository.get(id)!; + } + + bind(projectId: string, credentialId: string, bindingKey: string, capabilities: string[]) { + const credential=this.requireAccessible(projectId,credentialId); + if (credential.status !== "active") throw new CredentialAccessDeniedError("Only active credentials can be bound."); + const required=[...new Set(capabilities)]; + if (required.some((capability)=>!credential.capabilities.includes(capability))) throw new CredentialAccessDeniedError("Binding requests capabilities the credential does not grant."); + return this.repository.bind(credentialId,projectId,bindingKey.trim(),required); + } + + async test(projectId: string, credentialId: string): Promise { + const credential=this.requireAccessible(projectId,credentialId); + try { const plaintext=await this.secretStore.get(this.context(credential)); plaintext.fill(0); return this.repository.updateValidation(credentialId,"valid"); } + catch { this.repository.updateValidation(credentialId,"invalid"); throw new Error("Credential validation failed."); } + } + + async rotate(projectId: string, credentialId: string, value: string): Promise { return this.replaceValue(projectId,credentialId,value,true); } + async replace(projectId: string, credentialId: string, value: string): Promise { return this.replaceValue(projectId,credentialId,value,false); } + revoke(projectId: string, credentialId: string): AutomationCredentialMetadata { this.requireAccessible(projectId,credentialId); return this.repository.updateStatus(credentialId,"revoked"); } + async promote(projectId: string, credentialId: string, allowedProjectIds: string[]): Promise { const credential=this.requireAccessible(projectId,credentialId); if (credential.scope !== "project" || credential.projectId !== projectId) throw new CredentialAccessDeniedError("Only the owning project can promote this credential."); if (!allowedProjectIds.includes(projectId)) throw new Error("The global allowlist must retain the owning project."); const plaintext=await this.secretStore.get(this.context(credential)); const promoted=this.repository.promote(credentialId,[...new Set(allowedProjectIds)]); try { await this.secretStore.put(this.context(promoted),plaintext); return this.repository.get(credentialId)!; } catch(error){this.repository.updateStatus(credentialId,"unavailable");throw error;} finally { plaintext.fill(0); } } + restrict(projectId: string, credentialId: string, allowedProjectIds: string[], capabilities: string[]): AutomationCredentialMetadata { const credential=this.requireAccessible(projectId,credentialId); if (credential.scope === "global" && !allowedProjectIds.includes(projectId)) throw new Error("The configuring project must remain allowlisted."); return this.repository.restrict(credentialId,[...new Set(allowedProjectIds)],[...new Set(capabilities)]); } + + async resolve(request: CredentialResolutionRequest): Promise { + const binding=this.repository.getBinding(request.projectId,request.bindingKey); + if (!binding) return this.deny(request,null,"No credential binding exists."); + const credential=this.repository.get(binding.credentialId); + if (!credential) return this.deny(request,binding.credentialId,"Bound credential is missing."); + if (!this.canAccess(credential,request.projectId)) return this.deny(request,credential.id,"Credential is outside the project scope."); + if (credential.status !== "active") return this.deny(request,credential.id,"Credential is not active."); + if (!credential.capabilities.includes(request.capability) || !binding.requiredCapabilities.includes(request.capability)) return this.deny(request,credential.id,"Required capability is not approved."); + try { + const secret=await this.secretStore.get(this.context(credential)); const value=secret.toString("utf8"); secret.fill(0); + this.repository.recordAccess({credentialId:credential.id,projectId:request.projectId,bindingKey:request.bindingKey,capability:request.capability,operation:"resolve",outcome:"granted",reason:null}); + return {credentialId:credential.id,value,version:credential.version}; + } catch { return this.deny(request,credential.id,"Credential backend is unavailable or authentication failed."); } + } + + async resolveCredentialId(request: CredentialResolutionRequest & { credentialId: string }): Promise { + const credential=this.repository.get(request.credentialId); + if (!credential) return this.deny(request,request.credentialId,"Credential is missing."); + if (!this.canAccess(credential,request.projectId)) return this.deny(request,credential.id,"Credential is outside the project scope."); + if (credential.status !== "active") return this.deny(request,credential.id,"Credential is not active."); + if (!credential.capabilities.includes(request.capability)) return this.deny(request,credential.id,"Required capability is not approved."); + try { + const secret=await this.secretStore.get(this.context(credential)); const value=secret.toString("utf8"); secret.fill(0); + this.repository.recordAccess({credentialId:credential.id,projectId:request.projectId,bindingKey:request.bindingKey,capability:request.capability,operation:"resolve",outcome:"granted",reason:null}); + return {credentialId:credential.id,value,version:credential.version}; + } catch { return this.deny(request,credential.id,"Credential backend is unavailable or authentication failed."); } + } + + private async replaceValue(projectId:string,credentialId:string,value:string,rotation:boolean):Promise{ const credential=this.requireAccessible(projectId,credentialId); if (!value) throw new Error("A non-empty replacement value is required."); const plaintext=Buffer.from(value,"utf8"); try { const envelope=await this.secretStore.put(this.context(credential),plaintext); const next=this.repository.updateSecretMetadata(credentialId,envelope.keyId,envelope.keyVersion,credential.version+1); if(rotation)this.repository.recordRotation({credentialId,fromVersion:credential.version,toVersion:next.version,keyId:envelope.keyId,keyVersion:envelope.keyVersion}); return next; } finally { plaintext.fill(0); } } + private context(credential:AutomationCredentialMetadata){ return {credentialId:credential.id,projectId:credential.projectId ?? "global",workspaceId:credential.projectId ?? "global"}; } + private canAccess(credential:AutomationCredentialMetadata,projectId:string):boolean{return credential.scope === "project" ? credential.projectId === projectId : credential.allowedProjectIds.includes(projectId);} + private requireAccessible(projectId:string,credentialId:string):AutomationCredentialMetadata{this.repository.requireProject(projectId);const credential=this.repository.get(credentialId);if(!credential||!this.canAccess(credential,projectId))throw new CredentialAccessDeniedError("Credential is not available to this project.");return credential;} + private deny(request:CredentialResolutionRequest,credentialId:string|null,reason:string):never{this.repository.recordAccess({credentialId,projectId:request.projectId,bindingKey:request.bindingKey,capability:request.capability,operation:"resolve",outcome:"denied",reason});throw new CredentialAccessDeniedError(reason);} +} diff --git a/src/services/credentials/encryption-utils.ts b/src/services/credentials/encryption-utils.ts new file mode 100644 index 0000000000..2433560589 --- /dev/null +++ b/src/services/credentials/encryption-utils.ts @@ -0,0 +1,63 @@ +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import type { KeyMaterial } from "./key-provider.js"; +import type { SecretContext, StoredSecretEnvelope } from "./secret-store.js"; + +const ALGORITHM = "aes-256-gcm"; +const NONCE_BYTES = 12; + +function aad(context: SecretContext, purpose: "credential" | "data-key"): Buffer { + return Buffer.from(JSON.stringify({ + credentialId: context.credentialId, + projectId: context.projectId, + workspaceId: context.workspaceId, + purpose, + schema: 1, + }), "utf8"); +} + +function encrypt(key: Buffer, plaintext: Buffer, associatedData: Buffer): { ciphertext: Buffer; nonce: Buffer; authTag: Buffer } { + if (key.length !== 32) throw new Error("AES-256-GCM requires a 32-byte key."); + const nonce = randomBytes(NONCE_BYTES); + const cipher = createCipheriv(ALGORITHM, key, nonce); + cipher.setAAD(associatedData); + const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); + return { ciphertext, nonce, authTag: cipher.getAuthTag() }; +} + +function decrypt(key: Buffer, ciphertext: Buffer, nonce: Buffer, authTag: Buffer, associatedData: Buffer): Buffer { + if (key.length !== 32) throw new Error("AES-256-GCM requires a 32-byte key."); + const decipher = createDecipheriv(ALGORITHM, key, nonce); + decipher.setAAD(associatedData); + decipher.setAuthTag(authTag); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]); +} + +export function encryptEnvelope(context: SecretContext, plaintext: Buffer, rootKey: KeyMaterial): StoredSecretEnvelope { + const dataKey = randomBytes(32); + try { + const payload = encrypt(dataKey, plaintext, aad(context, "credential")); + const wrapped = encrypt(rootKey.key, dataKey, aad(context, "data-key")); + return { + credentialId: context.credentialId, + ciphertext: payload.ciphertext, + nonce: payload.nonce, + authTag: payload.authTag, + wrappedDataKey: wrapped.ciphertext, + wrapNonce: wrapped.nonce, + wrapAuthTag: wrapped.authTag, + keyId: rootKey.keyId, + keyVersion: rootKey.version, + }; + } finally { + dataKey.fill(0); + } +} + +export function decryptEnvelope(context: SecretContext, envelope: StoredSecretEnvelope, rootKey: KeyMaterial): Buffer { + const dataKey = decrypt(rootKey.key, envelope.wrappedDataKey, envelope.wrapNonce, envelope.wrapAuthTag, aad(context, "data-key")); + try { + return decrypt(dataKey, envelope.ciphertext, envelope.nonce, envelope.authTag, aad(context, "credential")); + } finally { + dataKey.fill(0); + } +} diff --git a/src/services/credentials/key-provider-registry.ts b/src/services/credentials/key-provider-registry.ts new file mode 100644 index 0000000000..afb200093a --- /dev/null +++ b/src/services/credentials/key-provider-registry.ts @@ -0,0 +1,13 @@ +import type { KeyProvider } from "./key-provider.js"; + +let processKeyProvider: KeyProvider | null = null; + +/** Configures a host-specific provider before dependency construction (for example Electron safeStorage). */ +export function setProcessCredentialKeyProvider(provider: KeyProvider): void { + if (processKeyProvider) throw new Error("The process credential key provider is already configured."); + processKeyProvider = provider; +} + +export function getProcessCredentialKeyProvider(): KeyProvider | null { + return processKeyProvider; +} diff --git a/src/services/credentials/key-provider.ts b/src/services/credentials/key-provider.ts new file mode 100644 index 0000000000..5ecc2ee747 --- /dev/null +++ b/src/services/credentials/key-provider.ts @@ -0,0 +1,21 @@ +import type { CredentialBackendHealth } from "../../contracts/automation-credential-types.js"; + +export interface KeyMaterial { + key: Buffer; + keyId: string; + version: number; +} + +export interface KeyProvider { + readonly providerName: string; + health(): Promise; + getActiveKey(): Promise; + getKey(keyId: string, version: number): Promise; +} + +export class KeyProviderUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = "KeyProviderUnavailableError"; + } +} diff --git a/src/services/credentials/secret-store.ts b/src/services/credentials/secret-store.ts new file mode 100644 index 0000000000..2c6935557c --- /dev/null +++ b/src/services/credentials/secret-store.ts @@ -0,0 +1,23 @@ +export interface SecretContext { + credentialId: string; + projectId: string; + workspaceId: string; +} + +export interface StoredSecretEnvelope { + credentialId: string; + ciphertext: Buffer; + nonce: Buffer; + authTag: Buffer; + wrappedDataKey: Buffer; + wrapNonce: Buffer; + wrapAuthTag: Buffer; + keyId: string; + keyVersion: number; +} + +export interface SecretStore { + put(context: SecretContext, plaintext: Buffer): Promise; + get(context: SecretContext): Promise; + delete(credentialId: string): Promise; +} diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index d99082ef0b..92c16e1afb 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -10,6 +10,7 @@ import type { SettingsRepository } from "../repositories/settings-repository.js" import type { ProviderExecutionService } from "./provider-execution-service.js"; import type { CliProviderId } from "../infrastructure/providers/cli/provider-command-specs.js"; import type { ProviderRunResult } from "../infrastructure/providers/cli/provider-runner.js"; +import type { CredentialBroker } from "./credentials/credential-broker.js"; import { buildProviderInvocationWorkspaceOptions } from "../infrastructure/providers/cli/invocation-workspace-preparer.js"; import type { DashboardSettings, @@ -40,6 +41,7 @@ interface NodeFlowRuntimeDeps { settingsRepository: SettingsRepository; providerExecutionService?: ProviderExecutionService; getDashboardSettings?: (projectId: string) => DashboardSettings; + credentialBroker?: CredentialBroker; } interface RuntimeContext { @@ -334,6 +336,7 @@ export class NodeFlowRuntimeService { throw new ValidationError(`Provider prompt node ${node.id} requires a prompt value.`); } const providerSettings = this.resolveProviderSettings(context.projectId, config); + const boundCredential = await this.resolveNodeCredential(context, node, "provider"); if (!CLI_PROVIDER_IDS.has(providerSettings.provider)) { throw new ValidationError(`Provider prompt node ${node.id} requires a CLI provider.`); } @@ -359,7 +362,7 @@ export class NodeFlowRuntimeService { maxConcurrentTasks: providerSettings.maxConcurrentTasks, prompt, model: readString(config.model) ?? providerSettings.model, - apiKey: providerSettings.apiKey, + apiKey: boundCredential ?? providerSettings.apiKey, providerMountAuth: providerSettings.mountAuth, providerAuthPath: providerSettings.authPath, providerConfigMode: providerSettings.providerConfigMode, @@ -448,6 +451,8 @@ export class NodeFlowRuntimeService { } } const headers = normalizeHeaders(readJsonObject(config.headers)); + const boundCredential = await this.resolveNodeCredential(context, node, "auth"); + if (boundCredential) headers.Authorization = boundCredential; const timeoutMs = normalizeTimeout(config.timeout ?? config.timeoutMs); const controller = new AbortController(); const abortListener = (): void => controller.abort(context.options.signal?.reason); @@ -494,6 +499,14 @@ export class NodeFlowRuntimeService { } } + private async resolveNodeCredential(context: RuntimeContext,node:NodeFlowNode,slot:string):Promise{ + const binding=node.credentialBindings?.find((candidate)=>candidate.slot===slot); + if (!binding) return undefined; + if (!this.deps.credentialBroker) throw new ValidationError("Credential broker is not configured for node flow runtime."); + const resolved=await this.deps.credentialBroker.resolveCredentialId({projectId:context.projectId,credentialId:binding.credentialId,bindingKey:`${context.flowId}:${node.id}:${slot}`,capability:"read",workspaceId:context.runId}); + return resolved.value; + } + private executeOutputNode(context: RuntimeContext, node: NodeFlowNode): NodeFlowJsonObject { const config = readNodeConfig(node); const valuePath = readString(config.path) ?? readString(config.valuePath); diff --git a/tests/backend/repositories/automation-credential-repository.test.ts b/tests/backend/repositories/automation-credential-repository.test.ts new file mode 100644 index 0000000000..9b8004c2cd --- /dev/null +++ b/tests/backend/repositories/automation-credential-repository.test.ts @@ -0,0 +1,20 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; +import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; +import { AutomationCredentialRepository } from "../../../src/repositories/automation-credential-repository.js"; +import { MountedKeyFileProvider } from "../../../src/infrastructure/security/mounted-key-file-provider.js"; +import { EncryptedSqliteSecretStore } from "../../../src/infrastructure/security/encrypted-sqlite-secret-store.js"; +import { CredentialBroker } from "../../../src/services/credentials/credential-broker.js"; + +const dirs:string[]=[]; +async function fixture(){const dir=await mkdtemp(join(tmpdir(),"credential-test-"));dirs.push(dir);const dbPath=join(dir,"app.db");const keyPath=join(dir,"root.key");await writeFile(keyPath,Buffer.alloc(32,9).toString("base64"),{mode:0o600});const storage=new AppDbStorage(dbPath);const projects=new ProjectManagementRepository(storage);const first=projects.createProject({name:"First",sourceType:"local",sourceRef:join(dir,"first")});const second=projects.createProject({name:"Second",sourceType:"local",sourceRef:join(dir,"second")});const repository=new AutomationCredentialRepository(storage);const provider=new MountedKeyFileProvider(keyPath);const broker=new CredentialBroker(repository,new EncryptedSqliteSecretStore(repository,provider),provider);return{dir,dbPath,storage,repository,broker,first,second};} +afterEach(async()=>{await Promise.all(dirs.splice(0).map((dir)=>rm(dir,{recursive:true,force:true})))}); + +describe("automation credential repository and broker",()=>{ + it("persists only encrypted material, resolves capabilities, rotates, and audits metadata",async()=>{const f=await fixture();const secret="plain-secret-marker";const created=await f.broker.create(f.first.id,{name:"Token",kind:"api-token",value:secret,capabilities:["read"]});expect(JSON.stringify(created)).not.toContain(secret);const persisted=f.storage.getDatabase().prepare("SELECT * FROM automation_credential_secrets WHERE credential_id=?").get(created.id) as Record;expect(JSON.stringify(persisted)).not.toContain(secret);f.broker.bind(f.first.id,created.id,"node.http",["read"]);expect((await f.broker.resolve({projectId:f.first.id,bindingKey:"node.http",capability:"read",workspaceId:"run"})).value).toBe(secret);const rotated=await f.broker.rotate(f.first.id,created.id,"replacement");expect(rotated.version).toBe(2);expect((await f.broker.resolve({projectId:f.first.id,bindingKey:"node.http",capability:"read",workspaceId:"run"})).value).toBe("replacement");const event=f.storage.getDatabase().prepare("SELECT * FROM automation_credential_access_events ORDER BY created_at DESC LIMIT 1").get() as Record;expect(JSON.stringify(event)).not.toContain(secret);expect(f.storage.getDatabase().prepare("SELECT COUNT(*) AS count FROM automation_credential_rotations").get()).toMatchObject({count:1});f.storage.close();}); + it("fails closed for cross-project, revoked, missing, and insecure providers",async()=>{const f=await fixture();const created=await f.broker.create(f.first.id,{name:"Token",kind:"api-token",value:"secret",capabilities:["read"]});expect(()=>f.broker.bind(f.second.id,created.id,"node.http",["read"])).toThrow(/not available/);f.broker.bind(f.first.id,created.id,"node.http",["read"]);f.broker.revoke(f.first.id,created.id);await expect(f.broker.resolve({projectId:f.first.id,bindingKey:"node.http",capability:"read",workspaceId:"run"})).rejects.toThrow(/not active/);const insecurePath=join(f.dir,"insecure.key");await writeFile(insecurePath,Buffer.alloc(32,4).toString("base64"),{mode:0o644});const insecureHealth=await new MountedKeyFileProvider(insecurePath).health();expect(insecureHealth).toMatchObject({available:false,secure:false});f.storage.close();const unavailable=new MountedKeyFileProvider(undefined);expect((await unavailable.health()).available).toBe(false);}); + it("requires explicit project allowlists and re-encrypts promoted credentials",async()=>{const f=await fixture();await expect(f.broker.create(f.first.id,{name:"Global",kind:"token",value:"secret",scope:"global",allowedProjectIds:[f.second.id],capabilities:["read"]})).rejects.toThrow(/explicit allowlist/);const projectCredential=await f.broker.create(f.first.id,{name:"Promoted",kind:"token",value:"secret",capabilities:["read"]});const promoted=await f.broker.promote(f.first.id,projectCredential.id,[f.first.id,f.second.id]);expect(promoted.scope).toBe("global");f.broker.bind(f.second.id,promoted.id,"node.global",["read"]);expect((await f.broker.resolve({projectId:f.second.id,bindingKey:"node.global",capability:"read",workspaceId:"run"})).value).toBe("secret");f.storage.close();}); +}); diff --git a/tests/backend/server/automation-credential-routes.test.ts b/tests/backend/server/automation-credential-routes.test.ts new file mode 100644 index 0000000000..922e14cb03 --- /dev/null +++ b/tests/backend/server/automation-credential-routes.test.ts @@ -0,0 +1,8 @@ +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import { registerAutomationCredentialRoutes } from "../../../src/server/automation-credential-routes.js"; + +describe("automation credential routes",()=>{ + it("passes secret values only into write operations and returns metadata",async()=>{const metadata={id:"credential-1",name:"Token",kind:"api-token",scope:"project",projectId:"project-1",allowedProjectIds:[],capabilities:["read"],status:"active",configured:true,keyId:"root",keyVersion:1,version:1,lastValidatedAt:null,validationStatus:"untested",createdAt:"now",updatedAt:"now"};const credentialBroker={create:vi.fn().mockResolvedValue(metadata)};const app=express();app.use(express.json());registerAutomationCredentialRoutes(app,{credentialBroker} as any);const response=await request(app).post("/api/projects/project-1/credentials").send({name:"Token",kind:"api-token",value:"super-secret",capabilities:["read"]});expect(response.status).toBe(201);expect(response.body).toEqual(metadata);expect(JSON.stringify(response.body)).not.toContain("super-secret");expect(credentialBroker.create).toHaveBeenCalledWith("project-1",expect.objectContaining({value:"super-secret"}));}); +}); diff --git a/tests/backend/services/credential-encryption.test.ts b/tests/backend/services/credential-encryption.test.ts new file mode 100644 index 0000000000..70a32906c6 --- /dev/null +++ b/tests/backend/services/credential-encryption.test.ts @@ -0,0 +1,12 @@ +import { randomBytes } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { decryptEnvelope, encryptEnvelope } from "../../../src/services/credentials/encryption-utils.js"; + +const context={credentialId:"credential-1",projectId:"project-1",workspaceId:"workspace-1"}; +const root={key:Buffer.alloc(32,7),keyId:"root",version:1}; + +describe("credential envelope encryption",()=>{ + it("round trips with unique payload and wrap nonces",()=>{const first=encryptEnvelope(context,Buffer.from("secret"),root);const second=encryptEnvelope(context,Buffer.from("secret"),root);expect(decryptEnvelope(context,first,root).toString()).toBe("secret");expect(first.nonce.equals(second.nonce)).toBe(false);expect(first.wrapNonce.equals(second.wrapNonce)).toBe(false);expect(first.ciphertext.toString("utf8")).not.toContain("secret");}); + it.each(["ciphertext","authTag","wrappedDataKey","wrapAuthTag"] as const)("rejects %s tampering",(field)=>{const envelope=encryptEnvelope(context,Buffer.from("secret"),root);envelope[field][0]^=1;expect(()=>decryptEnvelope(context,envelope,root)).toThrow();}); + it("rejects wrong keys and authenticated context",()=>{const envelope=encryptEnvelope(context,Buffer.from("secret"),root);expect(()=>decryptEnvelope(context,envelope,{...root,key:randomBytes(32)})).toThrow();expect(()=>decryptEnvelope({...context,workspaceId:"other"},envelope,root)).toThrow();}); +}); diff --git a/tests/backend/services/node-flow-runtime-service.test.ts b/tests/backend/services/node-flow-runtime-service.test.ts index dc1683f65a..072726e075 100644 --- a/tests/backend/services/node-flow-runtime-service.test.ts +++ b/tests/backend/services/node-flow-runtime-service.test.ts @@ -12,10 +12,11 @@ import { DEFAULT_DASHBOARD_SETTINGS } from "../../../src/repositories/settings-d import { NodeFlowRuntimeService } from "../../../src/services/node-flow-runtime-service.js"; import type { ProviderExecutionService } from "../../../src/services/provider-execution-service.js"; import type { NodeFlowGraph } from "../../../src/contracts/node-flow-types.js"; +import type { CredentialBroker } from "../../../src/services/credentials/credential-broker.js"; const tempDirs: string[] = []; -async function createRuntime(providerExecutionService?: Partial): Promise<{ +async function createRuntime(providerExecutionService?: Partial,credentialBroker?:Partial): Promise<{ dir: string; projectRepository: ProjectManagementRepository; nodeFlowRepository: NodeFlowRepository; @@ -34,6 +35,7 @@ async function createRuntime(providerExecutionService?: Partial DEFAULT_DASHBOARD_SETTINGS, }); return { dir, projectRepository, nodeFlowRepository, executionRepository, runtime }; @@ -90,14 +92,15 @@ describe("NodeFlowRuntimeService", () => { rawUsageJson: null, }, }); - const { dir, projectRepository, nodeFlowRepository, executionRepository, runtime } = await createRuntime({ executeProvider } as Partial); + const resolveCredentialId=vi.fn().mockResolvedValue({credentialId:"credential-1",value:"bound-secret",version:1}); + const { dir, projectRepository, nodeFlowRepository, executionRepository, runtime } = await createRuntime({ executeProvider } as Partial,{resolveCredentialId}); const project = projectRepository.createProject({ name: "Provider Project", sourceType: "local", sourceRef: dir }); const flow = nodeFlowRepository.createFlow(project.id, { title: "Provider", graph: { nodes: [ { id: "input", type: "input", title: "Input" }, - { id: "prompt", type: "provider_prompt", title: "Prompt", data: { provider: "mockup-cli", prompt: "Answer {{input.question}}" } }, + { id: "prompt", type: "provider_prompt", title: "Prompt", data: { provider: "mockup-cli", prompt: "Answer {{input.question}}" }, credentialBindings: [{slot:"provider",credentialId:"credential-1"}] }, { id: "output", type: "output", title: "Output" }, ], edges: [ @@ -113,10 +116,12 @@ describe("NodeFlowRuntimeService", () => { type: "node_flow_node", provider: "mockup-cli", prompt: "Answer now", + apiKey: "bound-secret", invocationId: expect.stringMatching(/^xi_/), trackPromptInInvocation: false, trackAssistantInInvocation: false, })); + expect(resolveCredentialId).toHaveBeenCalledWith(expect.objectContaining({projectId:project.id,credentialId:"credential-1",bindingKey:`${flow.id}:prompt:provider`,capability:"read"})); const promptRun = result.nodeRuns.find((nodeRun) => nodeRun.nodeId === "prompt"); expect(promptRun?.executionInvocationId).toMatch(/^xi_/); expect(promptRun?.output).toMatchObject({ text: "provider answer", nativeSessionId: "native-1" }); From 27faf92be18c1a08090280c199bb90255dfdaae0 Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 04:44:42 +0000 Subject: [PATCH 03/25] feat(task T03): implement via codex --- .../node-flow-durable-execution.md | 7 + docs-web/architecture/node-flows.md | 2 + ...chitecture-node-flow-durable-execution.mdx | 7 + .../content/docs/architecture-node-flows.mdx | 2 + docs-web/content/docs/registry.ts | 9 ++ .../content/docs/user-dashboard-scheduler.mdx | 4 +- ...cture-node-flow-durable-execution.lazy.tsx | 11 ++ docs-web/user/dashboard/scheduler.md | 4 +- docs/SUMMARY.md | 1 + .../node-flow-durable-execution.md | 15 ++ docs/architecture/node-flows.md | 8 +- docs/dashboard/scheduler.md | 4 +- docs/index.md | 2 + .../dependency-factory/dashboard-factory.ts | 4 + .../node-flow-execution-policy-types.ts | 45 ++++++ src/contracts/node-flow-types.ts | 55 ++++++- src/contracts/scheduler-types.ts | 2 + src/mcp/management/node-flow-actions.ts | 5 + src/repositories/db/app-db-migrations.ts | 51 +++++++ src/repositories/node-flow-repository.ts | 143 +++++++++++++++++- src/repositories/scheduler-repository.ts | 11 +- src/server/node-flow-routes.ts | 7 + src/services/node-flow-runtime-service.ts | 118 +++++++++++++-- src/services/node-flow-service.ts | 4 + .../node-flows/node-flow-attempt-service.ts | 21 +++ .../node-flows/node-flow-lease-service.ts | 8 + .../node-flow-publication-service.ts | 19 +++ .../node-flows/node-flow-queue-service.ts | 18 +++ .../node-flows/node-flow-recovery-service.ts | 27 ++++ src/services/scheduler-service.ts | 11 +- .../mcp/management-node-flow-actions.test.ts | 1 + .../repositories/scheduler-repository.test.ts | 2 + tests/backend/server/node-flow-routes.test.ts | 1 + .../node-flow-recovery-service.test.ts | 59 ++++++++ .../node-flow-runtime-service.test.ts | 45 ++++++ .../services/scheduler-service.test.ts | 1 + 36 files changed, 707 insertions(+), 27 deletions(-) create mode 100644 docs-web/architecture/node-flow-durable-execution.md create mode 100644 docs-web/content/docs/architecture-node-flow-durable-execution.mdx create mode 100644 docs-web/routes/docs.architecture-node-flow-durable-execution.lazy.tsx create mode 100644 docs/architecture/node-flow-durable-execution.md create mode 100644 src/contracts/node-flow-execution-policy-types.ts create mode 100644 src/services/node-flows/node-flow-attempt-service.ts create mode 100644 src/services/node-flows/node-flow-lease-service.ts create mode 100644 src/services/node-flows/node-flow-publication-service.ts create mode 100644 src/services/node-flows/node-flow-queue-service.ts create mode 100644 src/services/node-flows/node-flow-recovery-service.ts create mode 100644 tests/backend/services/node-flow-recovery-service.test.ts diff --git a/docs-web/architecture/node-flow-durable-execution.md b/docs-web/architecture/node-flow-durable-execution.md new file mode 100644 index 0000000000..e5bc80f167 --- /dev/null +++ b/docs-web/architecture/node-flow-durable-execution.md @@ -0,0 +1,7 @@ +# Node Flow Durable Execution + +Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run. + +Runs are durably queued and leased with bounded global and project concurrency. Node attempts retain attempt number, executor and invocation identity, artifact digest, redacted payloads, credential ids, failure class, and retry decision. Retryable failures use bounded exponential backoff and jitter, while cancellation and timeout signals propagate to provider and HTTP work. + +On restart, expired pre-invocation work is safely requeued. Work with an external invocation and an unknown outcome moves to `attention_required` and is never silently replayed. Credential values are resolved only for the active node and are not retained in run history or diagnostics. diff --git a/docs-web/architecture/node-flows.md b/docs-web/architecture/node-flows.md index 493c84b123..f3f15bfffa 100644 --- a/docs-web/architecture/node-flows.md +++ b/docs-web/architecture/node-flows.md @@ -14,3 +14,5 @@ Node flows are project-owned, versioned Graph v2 workflows. | `output` | Selects the result. | These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. + +Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](./node-flow-durable-execution.md). diff --git a/docs-web/content/docs/architecture-node-flow-durable-execution.mdx b/docs-web/content/docs/architecture-node-flow-durable-execution.mdx new file mode 100644 index 0000000000..e5bc80f167 --- /dev/null +++ b/docs-web/content/docs/architecture-node-flow-durable-execution.mdx @@ -0,0 +1,7 @@ +# Node Flow Durable Execution + +Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run. + +Runs are durably queued and leased with bounded global and project concurrency. Node attempts retain attempt number, executor and invocation identity, artifact digest, redacted payloads, credential ids, failure class, and retry decision. Retryable failures use bounded exponential backoff and jitter, while cancellation and timeout signals propagate to provider and HTTP work. + +On restart, expired pre-invocation work is safely requeued. Work with an external invocation and an unknown outcome moves to `attention_required` and is never silently replayed. Credential values are resolved only for the active node and are not retained in run history or diagnostics. diff --git a/docs-web/content/docs/architecture-node-flows.mdx b/docs-web/content/docs/architecture-node-flows.mdx index 493c84b123..31bf1274aa 100644 --- a/docs-web/content/docs/architecture-node-flows.mdx +++ b/docs-web/content/docs/architecture-node-flows.mdx @@ -14,3 +14,5 @@ Node flows are project-owned, versioned Graph v2 workflows. | `output` | Selects the result. | These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. + +Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](/docs/architecture-node-flow-durable-execution). diff --git a/docs-web/content/docs/registry.ts b/docs-web/content/docs/registry.ts index 522ed80858..637afe7f3a 100644 --- a/docs-web/content/docs/registry.ts +++ b/docs-web/content/docs/registry.ts @@ -108,6 +108,7 @@ export type DocsSlug = | 'user-dashboard-custom-dashboards' | 'architecture-custom-dashboard-foundation' | 'architecture-managed-container-runtime' + | 'architecture-node-flow-durable-execution' | 'architecture-node-flow-foundation' | 'architecture-node-flows' | 'architecture-speech-input' @@ -858,6 +859,13 @@ export const docsRegistry: Record = { title: "Managed Container Runtime", description: "The managed container runtime removes first-invocation Docker builds while keeping provider binaries local to each user's Docker host.", }, + 'architecture-node-flow-durable-execution': { + id: 'architecture-node-flow-durable-execution', + path: '/docs/architecture-node-flow-durable-execution', + section: 'Architecture', + title: "Node Flow Durable Execution", + description: "Node flows execute immutable published snapshots. A run explicitly pins a published version or follows the latest published version; later edits cannot change a pinned run.", + }, 'architecture-node-flow-foundation': { id: 'architecture-node-flow-foundation', path: '/docs/architecture-node-flow-foundation', @@ -1001,6 +1009,7 @@ export const orderedDocs: DocsRegistryEntry[] = [ docsRegistry['user-dashboard-custom-dashboards'], docsRegistry['architecture-custom-dashboard-foundation'], docsRegistry['architecture-managed-container-runtime'], + docsRegistry['architecture-node-flow-durable-execution'], docsRegistry['architecture-node-flow-foundation'], docsRegistry['architecture-node-flows'], docsRegistry['architecture-speech-input'], diff --git a/docs-web/content/docs/user-dashboard-scheduler.mdx b/docs-web/content/docs/user-dashboard-scheduler.mdx index aeda9453e7..3331436bb2 100644 --- a/docs-web/content/docs/user-dashboard-scheduler.mdx +++ b/docs-web/content/docs/user-dashboard-scheduler.mdx @@ -22,11 +22,13 @@ Each scheduler entry has a **target** — the thing that runs when it fires: | **Message** | Posts a project message (for example, a recurring planning or status prompt). | | **Memory remediation** | Runs the long-term memory cleanup workflow on a schedule. | -Node-flow entries store `nodeFlowTarget = { flowId, input?, flowVersion? }` inside the existing +Node-flow entries store `nodeFlowTarget = { flowId, input?, versionSelection }` inside the existing target JSON payload, validate that the flow belongs to the selected project, and run through the node-flow runtime with scheduler trigger metadata when due. Blank dashboard input is omitted, and supplied input must be a JSON object. +Choose a pinned published version when every occurrence must execute the same immutable snapshot, or latest published when each occurrence should pick up the newest publication. A legacy `flowVersion` is treated as a pinned version and affects execution, not just audit metadata. + The backend scheduler contract also supports agent-created wakeups. Agent wakeups are stored in the target JSON payload with `origin` and `source` set to `agent_scheduler`, plus `createdByAgentId` when the creating agent provides it. Agent wakeups post diff --git a/docs-web/routes/docs.architecture-node-flow-durable-execution.lazy.tsx b/docs-web/routes/docs.architecture-node-flow-durable-execution.lazy.tsx new file mode 100644 index 0000000000..033f986814 --- /dev/null +++ b/docs-web/routes/docs.architecture-node-flow-durable-execution.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import ArchitectureNodeFlowDurableExecutionContent from '../content/docs/architecture-node-flow-durable-execution.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/architecture-node-flow-durable-execution')({ + component: () => ( + + + + ) +}) diff --git a/docs-web/user/dashboard/scheduler.md b/docs-web/user/dashboard/scheduler.md index c2ebc68e24..34481144bb 100644 --- a/docs-web/user/dashboard/scheduler.md +++ b/docs-web/user/dashboard/scheduler.md @@ -22,11 +22,13 @@ Each scheduler entry has a **target** — the thing that runs when it fires: | **Message** | Posts a project message (for example, a recurring planning or status prompt). | | **Memory remediation** | Runs the long-term memory cleanup workflow on a schedule. | -Node-flow entries store `nodeFlowTarget = { flowId, input?, flowVersion? }` inside the existing +Node-flow entries store `nodeFlowTarget = { flowId, input?, versionSelection }` inside the existing target JSON payload, validate that the flow belongs to the selected project, and run through the node-flow runtime with scheduler trigger metadata when due. Blank dashboard input is omitted, and supplied input must be a JSON object. +Choose a pinned published version when every occurrence must execute the same immutable snapshot, or latest published when each occurrence should pick up the newest publication. A legacy `flowVersion` is treated as a pinned version and affects execution, not just audit metadata. + The backend scheduler contract also supports agent-created wakeups. Agent wakeups are stored in the target JSON payload with `origin` and `source` set to `agent_scheduler`, plus `createdByAgentId` when the creating agent provides it. Agent wakeups post diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5e0a632d23..933e7c124d 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -39,6 +39,7 @@ - [Agent Knowledge Base](./architecture/agent-knowledge-base.md) - [Node Flow Foundation](./architecture/node-flow-foundation.md) - [Node Flows](./architecture/node-flows.md) +- [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/docs/architecture/node-flow-durable-execution.md b/docs/architecture/node-flow-durable-execution.md new file mode 100644 index 0000000000..334447c9de --- /dev/null +++ b/docs/architecture/node-flow-durable-execution.md @@ -0,0 +1,15 @@ +# Node Flow Durable Execution + +Node-flow execution is publication based. Saving a flow appends an immutable version and publication containing the normalized graph and an immutable execution-policy snapshot. Manual, MCP, and scheduled callers select either `{ mode: "pinned", version: N }` or `{ mode: "latest_published" }`; the runtime never executes the mutable `node_flows` graph. + +## Durable lifecycle + +Runs move through `queued`, `running`, `approval_waiting`, `retry_waiting`, `attention_required`, and terminal `succeeded`, `failed`, or `cancelled` states. A queue claim assigns an executor, lease expiry, and heartbeat. Global and per-project limits bound active claims. Cancellation and node timeouts propagate through `AbortSignal`. + +Every node execution creates a numbered attempt with executor identity, optional execution invocation id, SHA-256 output digest, redacted input/output, credential ids, failure classification, and retry decision. Retryable timeout, quota, and transient failures use the publication policy's bounded exponential backoff and jitter. Credential values are resolved only at the node boundary and are never written to run, attempt, invocation, or diagnostic records. + +## Recovery contract + +Startup recovery scans queued and waiting work plus running work with expired leases. A pre-invocation attempt can be requeued safely without inserting a duplicate attempt. An expired attempt with an invocation id has an unknown externally observable outcome and moves to `attention_required`; Code UX does not silently replay it. Approval- and retry-waiting runs retain their durable state until their prerequisite becomes actionable. + +The relevant tables are `node_flow_publications`, `node_flow_runs`, `node_flow_node_runs`, and `node_flow_node_attempts`. Attempt history is available at `GET /api/node-flow-runs/:runId/attempts` and contains only redacted payloads and credential identifiers. diff --git a/docs/architecture/node-flows.md b/docs/architecture/node-flows.md index 6268a53066..bc355cf310 100644 --- a/docs/architecture/node-flows.md +++ b/docs/architecture/node-flows.md @@ -11,10 +11,12 @@ Node-flow persistence is owned by `NodeFlowRepository` and stored in SQLite: | Table | Purpose | | --- | --- | | `node_flows` | Current project-scoped flow record: id, project id, title, description, normalized `graph_json`, current version, and timestamps. | -| `node_flow_versions` | Immutable snapshots written on create and every update. Versions keep the graph saved at that point, even though current runtime execution uses the latest flow record. | +| `node_flow_versions` | Immutable edit snapshots written on create and every update. | +| `node_flow_publications` | Immutable executable graph and execution-policy snapshots selected by pinned or latest-published runs. | | `node_flow_agent_skills` | Agent attachment table keyed by flow and agent preset. It stores the skill display name and description used when exposing the flow as a repeatable agent capability. | | `node_flow_runs` | Flow run records with status, version, trigger type, redacted trigger payload, redacted input/output, error message, timestamps, and optional `execution_invocation_id`. | | `node_flow_node_runs` | Per-node run records with status, node id, redacted input/output, error message, timestamps, and optional `execution_invocation_id`. | +| `node_flow_node_attempts` | Numbered attempts with executor/invocation identity, artifact digest, credential ids, redacted payloads, failure class, and retry decision. | All graphs, widget schemas, run inputs, outputs, and trigger payloads are stored as JSON text and hydrated into typed contracts at the repository boundary. Flow, version, run, and attachment records belong to a project. Agent attachment operations verify that the target agent preset belongs to the same project as the flow. @@ -37,7 +39,7 @@ Dashboard-only editable canvas state lives in `dashboard/src/v2/lib/nodes-canvas ## Runtime -`NodeFlowRuntimeService.runFlow(projectId, flowId, input, options)` revalidates the saved graph, checks project ownership, and then executes nodes in the validator's topological order. +`NodeFlowRuntimeService.runFlow(projectId, flowId, input, options)` resolves an explicit pinned or latest-published snapshot, revalidates that immutable graph, claims a durable lease, and executes nodes in topological order. See [Node Flow Durable Execution](./node-flow-durable-execution.md) for queue, retry, lease, recovery, quota, and redaction guarantees. Runtime-supported node types are: @@ -75,7 +77,7 @@ Cancellation records cancelled node rows for the current and remaining nodes. At ## Scheduling -Scheduler entries with `targetType: "node_flow"` persist `nodeFlowTarget = { flowId, input?, flowVersion? }` inside `scheduler_entries.target_json`. Ownership is validated when entries are created or updated and again before due-run execution. +Scheduler entries with `targetType: "node_flow"` persist an explicit `versionSelection`: pinned schedules continue to execute version N after N+1 is published, while latest-published schedules resolve the newest publication at dispatch time. Legacy `flowVersion` values normalize to pinned selection and are executable semantics, not audit-only metadata. Ownership is validated when entries are created or updated and again before due-run execution. Due runs call `NodeFlowRuntimeService.runFlow` with `triggerType = "scheduler"` and trigger payload metadata for the scheduler entry id, scheduled occurrence time, target type, and persisted flow version when present. Node-flow schedules advance only when `runFlow` returns a run status of `succeeded`. Returned `failed` or `cancelled` runs mark the scheduler entry `failed` with the run error and still count the attempted occurrence in `lastRunAt` and `runCount`; runtime startup rejections mark failure without creating a false successful schedule run. diff --git a/docs/dashboard/scheduler.md b/docs/dashboard/scheduler.md index b7f31b8dcd..583234204c 100644 --- a/docs/dashboard/scheduler.md +++ b/docs/dashboard/scheduler.md @@ -78,7 +78,7 @@ The target payload keys are: - `chatTarget`: `{ bodyMarkdown, threadId?, title?, connectionId? }` - `memoryRemediationTarget`: `{ mode, source? }` - `taskTarget`: `{ taskId, provider?, origin: "agent_scheduler", source: "agent_scheduler", createdByAgentId? }` -- `nodeFlowTarget`: `{ flowId, input?, flowVersion? }` +- `nodeFlowTarget`: `{ flowId, input?, versionSelection }`; legacy `flowVersion` normalizes to pinned selection - `agentWakeupTarget`: `{ bodyMarkdown, threadId?, title?, connectionId?, origin: "agent_scheduler", source: "agent_scheduler", createdByAgentId? }` `node_flow` entries keep their flow id and optional input in `target_json`; ownership is checked when entries are created or updated and again before due-run execution. The persisted `flowVersion` is target metadata and is passed in scheduler trigger payloads for auditability; the current runtime executes through the latest node-flow runtime API. Due-run handling treats the returned node-flow run status as authoritative: only `succeeded` advances the schedule as successful, while `failed` and `cancelled` mark the scheduler entry `failed`, persist the run error, and record the attempted occurrence in `lastRunAt` and `runCount`. `agent_wakeup` and `task` entries always normalize `origin` and `source` to `agent_scheduler` in `target_json`. When the creator supplies `createdByAgentId`, it is preserved with the target payload for later authorization, audit, and notification work. Existing sprint, quicksprint, chat, memory remediation, recurrence, pause/resume, and `after_sprint_end` anchor rows continue to hydrate from the same JSON payload without a schema migration. @@ -156,7 +156,7 @@ For sprint targets, failures from either automatic planning or direct orchestrat ### Node-Flow Schedules -Node-flow schedules use `targetType: "node_flow"` and `nodeFlowTarget = { flowId, input?, flowVersion? }`. +Node-flow schedules use `targetType: "node_flow"` and `nodeFlowTarget = { flowId, input?, versionSelection }`. A pinned selection always executes that published graph and policy snapshot after newer versions are published; `latest_published` resolves the newest publication per occurrence. Behavior: diff --git a/docs/index.md b/docs/index.md index 201c31ab40..21d4ab0ebf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -66,6 +66,7 @@ Use this page as the main entrypoint. 28. [Agent Knowledge Base](./architecture/agent-knowledge-base.md) 29. [Node Flow Foundation](./architecture/node-flow-foundation.md) 30. [Node Flows](./architecture/node-flows.md) +31. [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) 31. [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) 32. [Memory Claims and Evidence](./architecture/memory-claims.md) 33. [Speech Input Architecture](./architecture/speech-input.md) @@ -158,6 +159,7 @@ Use this page as the main entrypoint. - [Agent Knowledge Base](./architecture/agent-knowledge-base.md) - [Node Flow Foundation](./architecture/node-flow-foundation.md) - [Node Flows](./architecture/node-flows.md) +- [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/src/app/dependency-factory/dashboard-factory.ts b/src/app/dependency-factory/dashboard-factory.ts index af70fdec29..da4d85983f 100644 --- a/src/app/dependency-factory/dashboard-factory.ts +++ b/src/app/dependency-factory/dashboard-factory.ts @@ -31,6 +31,7 @@ import { SpeechSynthesisService } from "../../services/speech-synthesis-service. import { SpeechModelManager } from "../../services/speech-model-manager.js"; import { NodeFlowRuntimeService } from "../../services/node-flow-runtime-service.js"; import { NodeFlowService } from "../../services/node-flow-service.js"; +import { NodeFlowRecoveryService } from "../../services/node-flows/node-flow-recovery-service.js"; import { resolveEffectiveDashboardSettings } from "../../services/settings-resolution-service.js"; export interface DashboardDependencies { @@ -230,6 +231,9 @@ export function createDashboardDependencies( credentialBroker: coreDeps.credentialBroker, getDashboardSettings: (projectId) => resolveDashboardSettings({ projectId }), }); + if (coreDeps.nodeFlowRepository) { + new NodeFlowRecoveryService(coreDeps.nodeFlowRepository).recover(); + } const nodeFlowService = new NodeFlowService(coreDeps.nodeFlowRepository, nodeFlowRuntimeService); const activityCacheService = new ActivityCacheService( diff --git a/src/contracts/node-flow-execution-policy-types.ts b/src/contracts/node-flow-execution-policy-types.ts new file mode 100644 index 0000000000..151a316e9e --- /dev/null +++ b/src/contracts/node-flow-execution-policy-types.ts @@ -0,0 +1,45 @@ +export type NodeFlowVersionSelection = + | { mode: "latest_published" } + | { mode: "pinned"; version: number }; + +export type NodeFlowFailureClassification = + | "cancelled" + | "timeout" + | "quota" + | "validation" + | "credential" + | "transient" + | "permanent" + | "unknown_side_effect"; + +export interface NodeFlowRetryPolicySnapshot { + maxAttempts: number; + backoffMs: number; + maxBackoffMs: number; + jitterRatio: number; + retryableClasses: NodeFlowFailureClassification[]; +} + +export interface NodeFlowExecutionPolicySnapshot { + maxConcurrentRuns: number; + maxConcurrentRunsPerProject: number; + leaseDurationMs: number; + heartbeatIntervalMs: number; + defaultTimeoutMs: number; + retry: NodeFlowRetryPolicySnapshot; +} + +export const DEFAULT_NODE_FLOW_EXECUTION_POLICY: Readonly = Object.freeze({ + maxConcurrentRuns: 4, + maxConcurrentRunsPerProject: 2, + leaseDurationMs: 30_000, + heartbeatIntervalMs: 10_000, + defaultTimeoutMs: 60_000, + retry: Object.freeze({ + maxAttempts: 1, + backoffMs: 500, + maxBackoffMs: 30_000, + jitterRatio: 0.2, + retryableClasses: Object.freeze(["timeout", "quota", "transient"]) as NodeFlowFailureClassification[], + }), +}); diff --git a/src/contracts/node-flow-types.ts b/src/contracts/node-flow-types.ts index 8e19d7ab7a..cc6471c28b 100644 --- a/src/contracts/node-flow-types.ts +++ b/src/contracts/node-flow-types.ts @@ -160,6 +160,17 @@ export interface NodeFlowVersionRecord { createdAt: string; } +export interface NodeFlowPublicationRecord { + id: string; + flowId: string; + projectId: string; + version: number; + graph: NodeFlowGraph; + policy: import("./node-flow-execution-policy-types.js").NodeFlowExecutionPolicySnapshot; + publishedBy: string; + createdAt: string; +} + export interface CreateNodeFlowInput { id?: string; title: string; @@ -202,15 +213,25 @@ export interface AttachNodeFlowSkillInput { description?: string; } -export type NodeFlowRunStatus = "queued" | "running" | "succeeded" | "failed" | "cancelled"; -export type NodeFlowNodeRunStatus = "pending" | "running" | "succeeded" | "failed" | "skipped" | "cancelled"; +export type NodeFlowRunStatus = + | "queued" | "running" | "approval_waiting" | "retry_waiting" + | "attention_required" | "succeeded" | "failed" | "cancelled"; +export type NodeFlowNodeRunStatus = + | "pending" | "running" | "retry_waiting" | "attention_required" + | "succeeded" | "failed" | "skipped" | "cancelled"; export interface NodeFlowRunRecord { id: string; flowId: string; projectId: string; version: number; + publicationId: string | null; status: NodeFlowRunStatus; + policy: import("./node-flow-execution-policy-types.js").NodeFlowExecutionPolicySnapshot; + leaseOwner: string | null; + leaseExpiresAt: string | null; + heartbeatAt: string | null; + cancelRequestedAt: string | null; executionInvocationId: string | null; triggerType: string; triggerPayload: NodeFlowJsonObject | null; @@ -240,6 +261,27 @@ export interface NodeFlowNodeRunRecord { updatedAt: string; } +export interface NodeFlowNodeAttemptRecord { + id: string; + runId: string; + nodeRunId: string; + nodeId: string; + attemptNumber: number; + status: NodeFlowNodeRunStatus; + executorId: string; + invocationId: string | null; + artifactDigest: string | null; + input: NodeFlowJsonObject | null; + output: NodeFlowJsonObject | null; + credentialIds: string[]; + failureClassification: import("./node-flow-execution-policy-types.js").NodeFlowFailureClassification | null; + retryDecision: "retry" | "stop" | "attention_required" | null; + errorMessage: string | null; + startedAt: string; + finishedAt: string | null; + createdAt: string; +} + export interface NodeFlowListResponse { flows: NodeFlowRecord[]; } @@ -256,6 +298,8 @@ export interface CreateNodeFlowRunInput { flowId: string; projectId: string; version: number; + publicationId?: string | null; + policy?: import("./node-flow-execution-policy-types.js").NodeFlowExecutionPolicySnapshot; status?: NodeFlowRunStatus; executionInvocationId?: string | null; triggerType?: string; @@ -274,6 +318,10 @@ export interface UpdateNodeFlowRunInput { errorMessage?: string | null; startedAt?: string | null; finishedAt?: string | null; + leaseOwner?: string | null; + leaseExpiresAt?: string | null; + heartbeatAt?: string | null; + cancelRequestedAt?: string | null; } export interface CreateNodeFlowNodeRunInput { @@ -304,10 +352,13 @@ export interface RunNodeFlowOptions { triggerType?: string; triggerPayload?: NodeFlowJsonObject; signal?: AbortSignal; + versionSelection?: import("./node-flow-execution-policy-types.js").NodeFlowVersionSelection; + executorId?: string; } export interface NodeFlowRunSummaryResponse { run: NodeFlowRunRecord; nodeRuns: NodeFlowNodeRunRecord[]; + attempts?: NodeFlowNodeAttemptRecord[]; output: NodeFlowJsonObject | null; } diff --git a/src/contracts/scheduler-types.ts b/src/contracts/scheduler-types.ts index 366f3d9968..12bcc88f64 100644 --- a/src/contracts/scheduler-types.ts +++ b/src/contracts/scheduler-types.ts @@ -1,6 +1,7 @@ import type { QuicksprintExecutionInput } from "./quicksprint-types.js"; import type { ProviderId } from "./app-types.js"; import type { NodeFlowJsonObject } from "./node-flow-types.js"; +import type { NodeFlowVersionSelection } from "./node-flow-execution-policy-types.js"; export type ScheduleTargetType = "sprint" | "quicksprint" | "chat" | "memory_remediation" | "agent_wakeup" | "task" | "node_flow"; export type ScheduleStatus = "scheduled" | "paused" | "completed" | "failed" | "cancelled"; @@ -79,6 +80,7 @@ export interface ScheduleNodeFlowTarget { flowId: string; input?: NodeFlowJsonObject; flowVersion?: number; + versionSelection?: NodeFlowVersionSelection; } export interface SchedulerEntryRecord { diff --git a/src/mcp/management/node-flow-actions.ts b/src/mcp/management/node-flow-actions.ts index cdb6cd138e..a35437f0b5 100644 --- a/src/mcp/management/node-flow-actions.ts +++ b/src/mcp/management/node-flow-actions.ts @@ -16,6 +16,7 @@ import type { NodeFlowService } from "../../services/node-flow-service.js"; import { managementValidationError, parseOptionalObject, + parseOptionalNumber, parseOptionalString, parseRequiredObject, parseRequiredString, @@ -139,8 +140,12 @@ export class NodeFlowActions { const projectId = parseRequiredString(payload, "projectId"); const flowId = parseRequiredString(payload, "flowId"); const input = parseOptionalObject(payload, "input") ?? {}; + const flowVersion = parseOptionalNumber(payload, "flowVersion", 1); const result = await this.nodeFlowService.runFlow(projectId, flowId, input, { triggerType: "mcp_management", + versionSelection: flowVersion === undefined + ? { mode: "latest_published" } + : { mode: "pinned", version: Math.floor(flowVersion) }, }); return { result: formatRunSummary(result) }; } diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index da475c1306..d54064de0e 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -257,7 +257,54 @@ export function ensureNodeFlowTables(db: DatabaseAdapter): void { ) `); + db.exec(` + CREATE TABLE IF NOT EXISTS node_flow_publications ( + id TEXT PRIMARY KEY, + flow_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL, + graph_json TEXT NOT NULL, + policy_json TEXT NOT NULL, + published_by TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (flow_id) REFERENCES node_flows(id) ON DELETE CASCADE, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + UNIQUE (flow_id, version) + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS node_flow_node_attempts ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + node_run_id TEXT NOT NULL, + node_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + status TEXT NOT NULL, + executor_id TEXT NOT NULL, + invocation_id TEXT, + artifact_digest TEXT, + input_json TEXT, + output_json TEXT, + credential_ids_json TEXT NOT NULL DEFAULT '[]', + failure_classification TEXT, + retry_decision TEXT, + error_message TEXT, + started_at TEXT NOT NULL, + finished_at TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (run_id) REFERENCES node_flow_runs(id) ON DELETE CASCADE, + FOREIGN KEY (node_run_id) REFERENCES node_flow_node_runs(id) ON DELETE CASCADE, + UNIQUE (run_id, node_id, attempt_number) + ) + `); + ensureColumn(db, "node_flow_runs", "execution_invocation_id", "TEXT"); + ensureColumn(db, "node_flow_runs", "publication_id", "TEXT"); + ensureColumn(db, "node_flow_runs", "policy_json", "TEXT NOT NULL DEFAULT '{}'"); + ensureColumn(db, "node_flow_runs", "lease_owner", "TEXT"); + ensureColumn(db, "node_flow_runs", "lease_expires_at", "TEXT"); + ensureColumn(db, "node_flow_runs", "heartbeat_at", "TEXT"); + ensureColumn(db, "node_flow_runs", "cancel_requested_at", "TEXT"); ensureColumn(db, "node_flow_node_runs", "execution_invocation_id", "TEXT"); ensureIndex(db, "idx_node_flows_project_updated", "node_flows", "project_id, updated_at DESC"); @@ -266,6 +313,10 @@ export function ensureNodeFlowTables(db: DatabaseAdapter): void { ensureIndex(db, "idx_node_flow_runs_flow_created", "node_flow_runs", "flow_id, created_at DESC"); ensureIndex(db, "idx_node_flow_runs_project_created", "node_flow_runs", "project_id, created_at DESC"); ensureIndex(db, "idx_node_flow_node_runs_run_created", "node_flow_node_runs", "run_id, created_at ASC"); + ensureIndex(db, "idx_node_flow_publications_latest", "node_flow_publications", "flow_id, version DESC"); + ensureIndex(db, "idx_node_flow_runs_queue", "node_flow_runs", "status, lease_expires_at, created_at ASC"); + ensureIndex(db, "idx_node_flow_runs_project_status", "node_flow_runs", "project_id, status"); + ensureIndex(db, "idx_node_flow_attempts_run_node", "node_flow_node_attempts", "run_id, node_id, attempt_number"); } interface LegacyNodeFlowRow { diff --git a/src/repositories/node-flow-repository.ts b/src/repositories/node-flow-repository.ts index faf2e5ae48..c3648e3606 100644 --- a/src/repositories/node-flow-repository.ts +++ b/src/repositories/node-flow-repository.ts @@ -12,6 +12,8 @@ import type { NodeFlowJsonObject, NodeFlowJsonValue, NodeFlowNodeRunRecord, + NodeFlowNodeAttemptRecord, + NodeFlowPublicationRecord, NodeFlowRecord, NodeFlowRunRecord, NodeFlowSkillAttachment, @@ -20,6 +22,8 @@ import type { UpdateNodeFlowInput, UpdateNodeFlowRunInput, } from "../contracts/node-flow-types.js"; +import { DEFAULT_NODE_FLOW_EXECUTION_POLICY } from "../contracts/node-flow-execution-policy-types.js"; +import type { NodeFlowExecutionPolicySnapshot, NodeFlowFailureClassification } from "../contracts/node-flow-execution-policy-types.js"; import { migratePersistedNodeFlowGraphs } from "./db/app-db-migrations.js"; interface NodeFlowRow { @@ -59,7 +63,13 @@ interface NodeFlowRunRow { flow_id: string; project_id: string; version: number | string; + publication_id: string | null; + policy_json: string; status: string; + lease_owner: string | null; + lease_expires_at: string | null; + heartbeat_at: string | null; + cancel_requested_at: string | null; execution_invocation_id: string | null; trigger_type: string; trigger_payload_json: string | null; @@ -72,6 +82,19 @@ interface NodeFlowRunRow { updated_at: string; } +interface NodeFlowPublicationRow { + id: string; flow_id: string; project_id: string; version: number | string; + graph_json: string; policy_json: string; published_by: string; created_at: string; +} + +interface NodeFlowAttemptRow { + id: string; run_id: string; node_run_id: string; node_id: string; attempt_number: number | string; + status: string; executor_id: string; invocation_id: string | null; artifact_digest: string | null; + input_json: string | null; output_json: string | null; credential_ids_json: string; + failure_classification: string | null; retry_decision: string | null; error_message: string | null; + started_at: string; finished_at: string | null; created_at: string; +} + interface NodeFlowNodeRunRow { id: string; run_id: string; @@ -98,6 +121,7 @@ export class NodeFlowRepository { ) { this.db = storage.getDatabase(); migratePersistedNodeFlowGraphs(this.db); + this.backfillPublications(); } listFlows(projectId: string): NodeFlowRecord[] { @@ -142,6 +166,7 @@ export class NodeFlowRepository { graphJson, createdAt: now, }); + this.insertPublication(id, projectId, 1, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, "system"); }); const created = this.requireFlow(id); @@ -173,6 +198,7 @@ export class NodeFlowRepository { graphJson, createdAt: now, }); + this.insertPublication(flowId, current.projectId, nextVersion, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, "system"); }); const updated = this.requireFlow(flowId); @@ -207,6 +233,26 @@ export class NodeFlowRepository { return row ? this.mapVersionRow(row) : null; } + listPublications(flowId: string): NodeFlowPublicationRecord[] { + this.requireFlow(flowId); + return (this.db.prepare(`SELECT * FROM node_flow_publications WHERE flow_id = ? ORDER BY version DESC`).all(flowId) as unknown as NodeFlowPublicationRow[]) + .map((row) => this.mapPublicationRow(row)); + } + + getPublication(flowId: string, version?: number): NodeFlowPublicationRecord | null { + const row = version === undefined + ? this.db.prepare(`SELECT * FROM node_flow_publications WHERE flow_id = ? ORDER BY version DESC LIMIT 1`).get(flowId) + : this.db.prepare(`SELECT * FROM node_flow_publications WHERE flow_id = ? AND version = ?`).get(flowId, Math.floor(version)); + return row ? this.mapPublicationRow(row as NodeFlowPublicationRow) : null; + } + + publishVersion(flowId: string, version: number, policy: NodeFlowExecutionPolicySnapshot = DEFAULT_NODE_FLOW_EXECUTION_POLICY, publishedBy = "system"): NodeFlowPublicationRecord { + const snapshot = this.getVersion(flowId, version); + if (!snapshot) throw new EntityNotFoundError(`Node flow version not found: ${flowId}@${version}`); + this.insertPublication(flowId, snapshot.projectId, snapshot.version, this.serializeJson(snapshot.graph), policy, publishedBy); + return requireRecord(this.getPublication(flowId, version), "Node flow publication", `${flowId}@${version}`); + } + attachToAgent(flowId: string, input: AttachNodeFlowSkillInput): NodeFlowSkillAttachment { const flow = this.requireFlow(flowId); const agentPresetId = input.agentPresetId?.trim(); @@ -317,14 +363,16 @@ export class NodeFlowRepository { const id = randomUUID(); this.db.prepare(` INSERT INTO node_flow_runs ( - id, flow_id, project_id, version, status, execution_invocation_id, trigger_type, + id, flow_id, project_id, version, publication_id, policy_json, status, execution_invocation_id, trigger_type, trigger_payload_json, input_json, output_json, error_message, started_at, finished_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( id, input.flowId, input.projectId, Math.max(1, Math.floor(input.version)), + input.publicationId ?? null, + this.serializeJson(input.policy ?? DEFAULT_NODE_FLOW_EXECUTION_POLICY), input.status || "running", input.executionInvocationId ?? null, input.triggerType?.trim() || "manual", @@ -350,7 +398,7 @@ export class NodeFlowRepository { output_json = ?, error_message = ?, started_at = ?, - finished_at = ?, + finished_at = ?, lease_owner = ?, lease_expires_at = ?, heartbeat_at = ?, cancel_requested_at = ?, updated_at = ? WHERE id = ? `).run( @@ -360,12 +408,62 @@ export class NodeFlowRepository { input.errorMessage === undefined ? current.errorMessage : input.errorMessage, input.startedAt === undefined ? current.startedAt : input.startedAt, input.finishedAt === undefined ? current.finishedAt : input.finishedAt, + input.leaseOwner === undefined ? current.leaseOwner : input.leaseOwner, + input.leaseExpiresAt === undefined ? current.leaseExpiresAt : input.leaseExpiresAt, + input.heartbeatAt === undefined ? current.heartbeatAt : input.heartbeatAt, + input.cancelRequestedAt === undefined ? current.cancelRequestedAt : input.cancelRequestedAt, now, runId, ); return requireRecord(this.getRun(runId), "Node flow run", runId); } + claimQueuedRun(runId: string, executorId: string, leaseDurationMs: number, now = new Date()): NodeFlowRunRecord | null { + const expiresAt = new Date(now.getTime() + leaseDurationMs).toISOString(); + const result = this.db.prepare(`UPDATE node_flow_runs SET status = 'running', lease_owner = ?, lease_expires_at = ?, heartbeat_at = ?, started_at = COALESCE(started_at, ?), updated_at = ? WHERE id = ? AND status IN ('queued','retry_waiting') AND (lease_expires_at IS NULL OR lease_expires_at <= ?)`) + .run(executorId, expiresAt, now.toISOString(), now.toISOString(), now.toISOString(), runId, now.toISOString()); + return result.changes > 0 ? this.getRun(runId) : null; + } + + heartbeatRun(runId: string, executorId: string, leaseDurationMs: number, now = new Date()): boolean { + const result = this.db.prepare(`UPDATE node_flow_runs SET heartbeat_at = ?, lease_expires_at = ?, updated_at = ? WHERE id = ? AND status = 'running' AND lease_owner = ?`) + .run(now.toISOString(), new Date(now.getTime() + leaseDurationMs).toISOString(), now.toISOString(), runId, executorId); + return result.changes > 0; + } + + countActiveRuns(projectId?: string): number { + const row = projectId + ? this.db.prepare(`SELECT COUNT(*) AS count FROM node_flow_runs WHERE project_id = ? AND status = 'running'`).get(projectId) + : this.db.prepare(`SELECT COUNT(*) AS count FROM node_flow_runs WHERE status = 'running'`).get(); + return toNumber((row as { count: number | string }).count); + } + + listRecoverableRuns(nowIso = new Date().toISOString()): NodeFlowRunRecord[] { + return (this.db.prepare(`SELECT * FROM node_flow_runs WHERE status IN ('queued','retry_waiting','approval_waiting') OR (status = 'running' AND lease_expires_at <= ?) ORDER BY created_at ASC`).all(nowIso) as unknown as NodeFlowRunRow[]).map((row) => this.mapRunRow(row)); + } + + requestCancellation(runId: string): NodeFlowRunRecord { + return this.updateRun(runId, { cancelRequestedAt: new Date().toISOString() }); + } + + createNodeAttempt(input: Omit): NodeFlowNodeAttemptRecord { + const id = randomUUID(); const now = new Date().toISOString(); + this.db.prepare(`INSERT INTO node_flow_node_attempts (id, run_id, node_run_id, node_id, attempt_number, status, executor_id, invocation_id, artifact_digest, input_json, output_json, credential_ids_json, failure_classification, retry_decision, error_message, started_at, finished_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run(id, input.runId, input.nodeRunId, input.nodeId, input.attemptNumber, input.status, input.executorId, input.invocationId, input.artifactDigest, this.serializeNullableJson(input.input), this.serializeNullableJson(input.output), JSON.stringify(input.credentialIds), input.failureClassification, input.retryDecision, input.errorMessage, input.startedAt, input.finishedAt, now); + return requireRecord(this.getNodeAttempt(id), "Node flow node attempt", id); + } + + updateNodeAttempt(id: string, input: Partial>): NodeFlowNodeAttemptRecord { + const current = requireRecord(this.getNodeAttempt(id), "Node flow node attempt", id); + this.db.prepare(`UPDATE node_flow_node_attempts SET status = ?, invocation_id = ?, artifact_digest = ?, output_json = ?, failure_classification = ?, retry_decision = ?, error_message = ?, finished_at = ? WHERE id = ?`) + .run(input.status ?? current.status, input.invocationId === undefined ? current.invocationId : input.invocationId, input.artifactDigest === undefined ? current.artifactDigest : input.artifactDigest, input.output === undefined ? this.serializeNullableJson(current.output) : this.serializeNullableJson(input.output), input.failureClassification === undefined ? current.failureClassification : input.failureClassification, input.retryDecision === undefined ? current.retryDecision : input.retryDecision, input.errorMessage === undefined ? current.errorMessage : input.errorMessage, input.finishedAt === undefined ? current.finishedAt : input.finishedAt, id); + return requireRecord(this.getNodeAttempt(id), "Node flow node attempt", id); + } + + listNodeAttempts(runId: string): NodeFlowNodeAttemptRecord[] { + return (this.db.prepare(`SELECT * FROM node_flow_node_attempts WHERE run_id = ? ORDER BY node_id, attempt_number`).all(runId) as unknown as NodeFlowAttemptRow[]).map((row) => this.mapAttemptRow(row)); + } + createNodeRun(input: CreateNodeFlowNodeRunInput): NodeFlowNodeRunRecord { const run = requireRecord(this.getRun(input.runId), "Node flow run", input.runId); if (run.flowId !== input.flowId || run.projectId !== input.projectId) { @@ -450,6 +548,19 @@ export class NodeFlowRepository { ); } + private insertPublication(flowId: string, projectId: string, version: number, graphJson: string, policy: NodeFlowExecutionPolicySnapshot, publishedBy: string): void { + this.db.prepare(`INSERT OR IGNORE INTO node_flow_publications (id, flow_id, project_id, version, graph_json, policy_json, published_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`) + .run(randomUUID(), flowId, projectId, version, graphJson, this.serializeJson(policy), publishedBy, new Date().toISOString()); + } + + private backfillPublications(): void { + const versions = this.db.prepare(`SELECT flow_id, project_id, version, graph_json, created_at FROM node_flow_versions`).all() as Array<{ flow_id: string; project_id: string; version: number | string; graph_json: string; created_at: string }>; + for (const version of versions) { + this.db.prepare(`INSERT OR IGNORE INTO node_flow_publications (id, flow_id, project_id, version, graph_json, policy_json, published_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`) + .run(randomUUID(), version.flow_id, version.project_id, toNumber(version.version), version.graph_json, this.serializeJson(DEFAULT_NODE_FLOW_EXECUTION_POLICY), "migration", version.created_at); + } + } + private requireProject(projectId: string): void { requireRecord(this.db.prepare(`SELECT id FROM projects WHERE id = ?`).get(projectId), "Project", projectId); } @@ -489,6 +600,11 @@ export class NodeFlowRepository { return row ? this.mapNodeRunRow(row) : null; } + private getNodeAttempt(id: string): NodeFlowNodeAttemptRecord | null { + const row = this.db.prepare(`SELECT * FROM node_flow_node_attempts WHERE id = ?`).get(id) as NodeFlowAttemptRow | undefined; + return row ? this.mapAttemptRow(row) : null; + } + private requireTitle(title: string | undefined): string { const normalized = title?.trim(); if (!normalized) { @@ -527,6 +643,11 @@ export class NodeFlowRepository { } } + private parsePolicy(value: string): NodeFlowExecutionPolicySnapshot { + try { return { ...DEFAULT_NODE_FLOW_EXECUTION_POLICY, ...JSON.parse(value) as NodeFlowExecutionPolicySnapshot }; } + catch { return { ...DEFAULT_NODE_FLOW_EXECUTION_POLICY }; } + } + private isJsonValue(value: unknown): value is NodeFlowJsonValue { if (value === null) { return true; @@ -594,7 +715,13 @@ export class NodeFlowRepository { flowId: row.flow_id, projectId: row.project_id, version: toNumber(row.version), + publicationId: row.publication_id, status: row.status as NodeFlowRunRecord["status"], + policy: this.parsePolicy(row.policy_json), + leaseOwner: row.lease_owner, + leaseExpiresAt: row.lease_expires_at, + heartbeatAt: row.heartbeat_at, + cancelRequestedAt: row.cancel_requested_at, executionInvocationId: row.execution_invocation_id, triggerType: row.trigger_type, triggerPayload: this.parseObject(row.trigger_payload_json), @@ -627,6 +754,16 @@ export class NodeFlowRepository { }; } + private mapPublicationRow(row: NodeFlowPublicationRow): NodeFlowPublicationRecord { + return { id: row.id, flowId: row.flow_id, projectId: row.project_id, version: toNumber(row.version), graph: this.parseGraph(row.graph_json), policy: this.parsePolicy(row.policy_json), publishedBy: row.published_by, createdAt: row.created_at }; + } + + private mapAttemptRow(row: NodeFlowAttemptRow): NodeFlowNodeAttemptRecord { + let credentialIds: string[] = []; + try { const parsed = JSON.parse(row.credential_ids_json) as unknown; if (Array.isArray(parsed)) credentialIds = parsed.filter((item): item is string => typeof item === "string"); } catch { /* legacy row */ } + return { id: row.id, runId: row.run_id, nodeRunId: row.node_run_id, nodeId: row.node_id, attemptNumber: toNumber(row.attempt_number), status: row.status as NodeFlowNodeRunRecord["status"], executorId: row.executor_id, invocationId: row.invocation_id, artifactDigest: row.artifact_digest, input: this.parseObject(row.input_json), output: this.parseObject(row.output_json), credentialIds, failureClassification: row.failure_classification as NodeFlowFailureClassification | null, retryDecision: row.retry_decision as NodeFlowNodeAttemptRecord["retryDecision"], errorMessage: row.error_message, startedAt: row.started_at, finishedAt: row.finished_at, createdAt: row.created_at }; + } + private publishProjectStructureRefresh(projectId: string): void { this.realtimeNotifier?.scheduleProjectStructureRefresh(projectId, { includeProjects: false }); } diff --git a/src/repositories/scheduler-repository.ts b/src/repositories/scheduler-repository.ts index fff9c56e63..a073fa9159 100644 --- a/src/repositories/scheduler-repository.ts +++ b/src/repositories/scheduler-repository.ts @@ -364,8 +364,17 @@ export class SchedulerRepository { target.input = normalizedInput; } const flowVersion = this.normalizeOptionalPositiveInteger(input.nodeFlowTarget?.flowVersion, "nodeFlowTarget.flowVersion"); - if (flowVersion !== undefined) { + const versionSelection = input.nodeFlowTarget?.versionSelection; + if (versionSelection?.mode === "pinned") { + target.versionSelection = { mode: "pinned", version: this.normalizeOptionalPositiveInteger(versionSelection.version, "nodeFlowTarget.versionSelection.version")! }; + target.flowVersion = target.versionSelection.version; + } else if (versionSelection?.mode === "latest_published") { + target.versionSelection = { mode: "latest_published" }; + } else if (flowVersion !== undefined) { target.flowVersion = flowVersion; + target.versionSelection = { mode: "pinned", version: flowVersion }; + } else { + target.versionSelection = { mode: "latest_published" }; } return { nodeFlowTarget: target }; } diff --git a/src/server/node-flow-routes.ts b/src/server/node-flow-routes.ts index 4d9a3805cd..a608686dd4 100644 --- a/src/server/node-flow-routes.ts +++ b/src/server/node-flow-routes.ts @@ -66,6 +66,7 @@ export function registerNodeFlowRoutes(app: Express, deps: DashboardDependencies input?: Record; triggerType?: string; triggerPayload?: NodeFlowJsonObject; + flowVersion?: number; }; const result = await requireNodeFlowService(deps).runFlow( requireTrimmedString(body.projectId, "projectId"), @@ -74,6 +75,9 @@ export function registerNodeFlowRoutes(app: Express, deps: DashboardDependencies { triggerType: body.triggerType, triggerPayload: body.triggerPayload, + versionSelection: body.flowVersion === undefined + ? { mode: "latest_published" } + : { mode: "pinned", version: body.flowVersion }, }, ); res.status(201).json(result); @@ -121,4 +125,7 @@ export function registerNodeFlowRoutes(app: Express, deps: DashboardDependencies app.get("/api/node-flow-runs/:runId/node-runs", syncRoute((req, res) => { res.json(requireNodeFlowService(deps).listNodeRuns(requireTrimmedString(req.params.runId, "runId"))); })); + app.get("/api/node-flow-runs/:runId/attempts", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).listNodeAttempts(requireTrimmedString(req.params.runId, "runId"))); + })); } diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index 92c16e1afb..57a8f95138 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -11,6 +11,11 @@ import type { ProviderExecutionService } from "./provider-execution-service.js"; import type { CliProviderId } from "../infrastructure/providers/cli/provider-command-specs.js"; import type { ProviderRunResult } from "../infrastructure/providers/cli/provider-runner.js"; import type { CredentialBroker } from "./credentials/credential-broker.js"; +import { NodeFlowPublicationService } from "./node-flows/node-flow-publication-service.js"; +import { NodeFlowQueueService } from "./node-flows/node-flow-queue-service.js"; +import { NodeFlowAttemptService } from "./node-flows/node-flow-attempt-service.js"; +import { NodeFlowLeaseService } from "./node-flows/node-flow-lease-service.js"; +import type { NodeFlowFailureClassification } from "../contracts/node-flow-execution-policy-types.js"; import { buildProviderInvocationWorkspaceOptions } from "../infrastructure/providers/cli/invocation-workspace-preparer.js"; import type { DashboardSettings, @@ -55,6 +60,8 @@ interface RuntimeContext { predecessors: Map; descendants: Map>; options: RunNodeFlowOptions; + executorId: string; + currentAttemptId?: string; } interface NodeExecutionResult { @@ -79,7 +86,9 @@ export class NodeFlowRuntimeService { throw new ValidationError("Node flow does not belong to the requested project."); } - const { graph, executionOrder } = normalizeNodeFlowGraph(flow.graph); + const selection = options.versionSelection ?? { mode: "latest_published" }; + const publication = new NodeFlowPublicationService(this.deps.nodeFlowRepository).resolve(flow.id, selection); + const { graph, executionOrder } = normalizeNodeFlowGraph(publication.graph); this.requireSupportedNodes(graph); const sanitizedInput = maskSecrets(input); const startedAt = new Date().toISOString(); @@ -92,29 +101,39 @@ export class NodeFlowRuntimeService { }); this.deps.executionRepository.appendExecutionInvocationMessage(parentInvocation.id, { role: "system", - contentMarkdown: `Node flow run started for flow ${flow.id} at version ${flow.version}.`, + contentMarkdown: `Node flow run started for flow ${flow.id} at published version ${publication.version}.`, metadata: { flowId: flow.id, - flowVersion: flow.version, + flowVersion: publication.version, + publicationId: publication.id, }, }); const run = this.deps.nodeFlowRepository.createRun({ flowId: flow.id, projectId, - version: flow.version, - status: "running", + version: publication.version, + publicationId: publication.id, + policy: publication.policy, + status: "queued", executionInvocationId: parentInvocation.id, triggerType: options.triggerType, triggerPayload: options.triggerPayload ? maskSecrets(options.triggerPayload) : null, input: sanitizedInput, - startedAt, + startedAt: null, }); + const executorId = options.executorId?.trim() || `node-flow-runtime:${process.pid}:${randomUUID()}`; + const claimedRun = new NodeFlowQueueService(this.deps.nodeFlowRepository).claim(run, executorId); + const leaseService = new NodeFlowLeaseService(this.deps.nodeFlowRepository); + const heartbeatTimer = setInterval(() => { + leaseService.heartbeat(claimedRun.id, executorId, publication.policy.leaseDurationMs); + }, publication.policy.heartbeatIntervalMs); + heartbeatTimer.unref?.(); const context: RuntimeContext = { projectId, flowId: flow.id, - runId: run.id, + runId: claimedRun.id, graph, order: executionOrder, input, @@ -122,6 +141,7 @@ export class NodeFlowRuntimeService { predecessors: buildPredecessors(graph), descendants: buildDescendants(graph), options, + executorId, }; const blockedNodes = new Set(); @@ -136,7 +156,7 @@ export class NodeFlowRuntimeService { } if (options.signal?.aborted) { terminalStatus = "cancelled"; - terminalError = "Node flow run was cancelled."; + terminalError ??= "Node flow run was cancelled."; await this.persistSkippedNode(context, node, "cancelled", terminalError); for (const remainingNodeId of executionOrder.slice(executionOrder.indexOf(nodeId) + 1)) { const remaining = graph.nodes.find((candidate) => candidate.id === remainingNodeId); @@ -155,28 +175,59 @@ export class NodeFlowRuntimeService { continue; } + const nodeInput = maskSecrets(this.buildNodeInput(context, node.id)); const nodeRun = this.deps.nodeFlowRepository.createNodeRun({ runId: run.id, flowId: flow.id, projectId, nodeId: node.id, status: "running", - input: maskSecrets(this.buildNodeInput(context, node.id)), + input: nodeInput, startedAt: new Date().toISOString(), }); - try { + const attemptService = new NodeFlowAttemptService(this.deps.nodeFlowRepository); + const retryPolicy = { + ...publication.policy.retry, + ...(node.policy?.retry ?? {}), + }; + let attemptNumber = 0; + while (attemptNumber < retryPolicy.maxAttempts) { + attemptNumber += 1; + const attempt = attemptService.start(nodeRun, executorId, nodeInput, (node.credentialBindings ?? []).map((binding) => binding.credentialId)); + context.currentAttemptId = attempt.id; + const timeoutMs = node.policy?.timeout?.timeoutMs ?? publication.policy.defaultTimeoutMs; + const timeoutController = new AbortController(); + const parentAbort = (): void => timeoutController.abort(options.signal?.reason); + options.signal?.addEventListener("abort", parentAbort, { once: true }); + const timeout = setTimeout(() => timeoutController.abort(new Error(`Node ${node.id} timed out after ${timeoutMs}ms.`)), timeoutMs); + const previousOptions = context.options; + context.options = { ...options, signal: timeoutController.signal }; + try { const result = await this.executeNode(context, node, nodeRun); context.outputs.set(node.id, result.output); + attemptService.succeed(attempt, maskSecrets(result.output), result.invocationId); this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { status: "succeeded", executionInvocationId: result.invocationId ?? nodeRun.executionInvocationId, output: maskSecrets(result.output), finishedAt: new Date().toISOString(), }); + clearTimeout(timeout); options.signal?.removeEventListener("abort", parentAbort); context.options = previousOptions; + break; } catch (error) { + clearTimeout(timeout); options.signal?.removeEventListener("abort", parentAbort); context.options = previousOptions; const message = error instanceof Error ? error.message : String(error); - const wasCancelled = options.signal?.aborted === true; + const classification = classifyFailure(error, options.signal?.aborted === true, timeoutController.signal.aborted); + const wasCancelled = classification === "cancelled"; + const retryable = retryPolicy.retryableClasses.includes(classification) && attemptNumber < retryPolicy.maxAttempts; + attemptService.fail(attempt, classification, message, retryable, this.deps.nodeFlowRepository.listNodeAttempts(run.id).find((candidate) => candidate.id === attempt.id)?.invocationId); + if (retryable) { + this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { status: "retry_waiting", errorMessage: message }); + await delay(retryDelay(retryPolicy.backoffMs, retryPolicy.maxBackoffMs ?? retryPolicy.backoffMs, retryPolicy.jitterRatio ?? 0, attemptNumber), options.signal); + this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { status: "running", errorMessage: null }); + continue; + } const continueOnError = node.data?.continueOnError === true; const failureOutput = { error: message }; context.outputs.set(node.id, failureOutput); @@ -186,7 +237,10 @@ export class NodeFlowRuntimeService { errorMessage: message, finishedAt: new Date().toISOString(), }); - if (wasCancelled) { + if (classification === "unknown_side_effect") { + terminalStatus = "attention_required"; + terminalError = message; + } else if (wasCancelled) { terminalStatus = "cancelled"; terminalError ??= message || "Node flow run was cancelled."; for (const remainingNodeId of executionOrder.slice(nodeIndex + 1)) { @@ -204,6 +258,11 @@ export class NodeFlowRuntimeService { blockedNodes.add(descendant); } } + break; + } + } + if (terminalStatus === "cancelled" || terminalStatus === "attention_required") { + break; } } @@ -214,9 +273,11 @@ export class NodeFlowRuntimeService { output: maskSecrets(output), errorMessage: terminalError, finishedAt, + leaseOwner: null, + leaseExpiresAt: null, }); this.deps.executionRepository.updateExecutionInvocation(parentInvocation.id, { - status: terminalStatus === "succeeded" ? "completed" : terminalStatus, + status: terminalStatus === "succeeded" ? "completed" : terminalStatus === "attention_required" ? "failed" : terminalStatus, errorMessage: terminalError, finishedAt, }); @@ -232,9 +293,11 @@ export class NodeFlowRuntimeService { }, }); + clearInterval(heartbeatTimer); return { run: updatedRun, nodeRuns: this.deps.nodeFlowRepository.listNodeRuns(run.id), + attempts: this.deps.nodeFlowRepository.listNodeAttempts(run.id), output: updatedRun.output, }; } @@ -263,6 +326,9 @@ export class NodeFlowRuntimeService { startedAt: new Date().toISOString(), }); this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { executionInvocationId: invocation.id }); + if (context.currentAttemptId) { + this.deps.nodeFlowRepository.updateNodeAttempt(context.currentAttemptId, { invocationId: invocation.id }); + } try { const result = node.type === "provider_prompt" ? await this.executeProviderPromptNode(context, node, invocation.id) @@ -761,6 +827,32 @@ function redactUrl(url: URL): string { return clone.toString(); } +function classifyFailure(error: unknown, parentAborted: boolean, attemptAborted: boolean): NodeFlowFailureClassification { + if (parentAborted) return "cancelled"; + const message = error instanceof Error ? error.message : String(error); + if (attemptAborted || /timed? out|timeout/i.test(message)) return "timeout"; + if (/quota|rate.?limit|429/i.test(message)) return "quota"; + if (/credential|secret|access denied/i.test(message)) return "credential"; + if (error instanceof ValidationError || /requires|unsupported|must /i.test(message)) return "validation"; + if (/ECONNRESET|ECONNREFUSED|temporar|unavailable|502|503|504/i.test(message)) return "transient"; + return "permanent"; +} + +function retryDelay(baseMs: number, maxMs: number, jitterRatio: number, attemptNumber: number): number { + const exponential = Math.min(maxMs, Math.max(0, baseMs) * (2 ** Math.max(0, attemptNumber - 1))); + const jitter = exponential * Math.max(0, Math.min(1, jitterRatio)); + return Math.max(0, Math.round(exponential - jitter + (Math.random() * jitter * 2))); +} + +async function delay(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0) return; + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, ms); + const abort = (): void => { clearTimeout(timer); reject(new Error("Node flow run was cancelled.")); }; + signal?.addEventListener("abort", abort, { once: true }); + }); +} + function providerFailureMessage(result: ProviderRunResult): string { const output = [result.stderr, result.stdout] .map((stream) => stream?.trim()) diff --git a/src/services/node-flow-service.ts b/src/services/node-flow-service.ts index e6462ceb72..066e300c57 100644 --- a/src/services/node-flow-service.ts +++ b/src/services/node-flow-service.ts @@ -107,6 +107,10 @@ export class NodeFlowService { return { nodeRuns: this.repository.listNodeRuns(runId) }; } + listNodeAttempts(runId: string) { + return { attempts: this.repository.listNodeAttempts(runId) }; + } + async runFlow( projectId: string, flowId: string, diff --git a/src/services/node-flows/node-flow-attempt-service.ts b/src/services/node-flows/node-flow-attempt-service.ts new file mode 100644 index 0000000000..cad3afb9f6 --- /dev/null +++ b/src/services/node-flows/node-flow-attempt-service.ts @@ -0,0 +1,21 @@ +import { createHash } from "crypto"; +import type { NodeFlowRepository } from "../../repositories/node-flow-repository.js"; +import type { NodeFlowJsonObject, NodeFlowNodeAttemptRecord, NodeFlowNodeRunRecord } from "../../contracts/node-flow-types.js"; +import type { NodeFlowFailureClassification } from "../../contracts/node-flow-execution-policy-types.js"; + +export class NodeFlowAttemptService { + constructor(private readonly repository: NodeFlowRepository) {} + + start(nodeRun: NodeFlowNodeRunRecord, executorId: string, input: NodeFlowJsonObject, credentialIds: string[]): NodeFlowNodeAttemptRecord { + const attemptNumber = this.repository.listNodeAttempts(nodeRun.runId).filter((attempt) => attempt.nodeId === nodeRun.nodeId).length + 1; + return this.repository.createNodeAttempt({ runId: nodeRun.runId, nodeRunId: nodeRun.id, nodeId: nodeRun.nodeId, attemptNumber, status: "running", executorId, invocationId: null, artifactDigest: null, input, output: null, credentialIds, failureClassification: null, retryDecision: null, errorMessage: null, startedAt: new Date().toISOString(), finishedAt: null }); + } + + succeed(attempt: NodeFlowNodeAttemptRecord, output: NodeFlowJsonObject, invocationId?: string | null): NodeFlowNodeAttemptRecord { + return this.repository.updateNodeAttempt(attempt.id, { status: "succeeded", invocationId: invocationId ?? null, output, artifactDigest: createHash("sha256").update(JSON.stringify(output)).digest("hex"), retryDecision: "stop", finishedAt: new Date().toISOString() }); + } + + fail(attempt: NodeFlowNodeAttemptRecord, classification: NodeFlowFailureClassification, errorMessage: string, retry: boolean, invocationId?: string | null): NodeFlowNodeAttemptRecord { + return this.repository.updateNodeAttempt(attempt.id, { status: classification === "cancelled" ? "cancelled" : "failed", invocationId: invocationId ?? attempt.invocationId, failureClassification: classification, retryDecision: classification === "unknown_side_effect" ? "attention_required" : retry ? "retry" : "stop", errorMessage, finishedAt: new Date().toISOString() }); + } +} diff --git a/src/services/node-flows/node-flow-lease-service.ts b/src/services/node-flows/node-flow-lease-service.ts new file mode 100644 index 0000000000..721b058e43 --- /dev/null +++ b/src/services/node-flows/node-flow-lease-service.ts @@ -0,0 +1,8 @@ +import type { NodeFlowRepository } from "../../repositories/node-flow-repository.js"; + +export class NodeFlowLeaseService { + constructor(private readonly repository: NodeFlowRepository) {} + heartbeat(runId: string, executorId: string, leaseDurationMs: number): boolean { + return this.repository.heartbeatRun(runId, executorId, leaseDurationMs); + } +} diff --git a/src/services/node-flows/node-flow-publication-service.ts b/src/services/node-flows/node-flow-publication-service.ts new file mode 100644 index 0000000000..78433d8c58 --- /dev/null +++ b/src/services/node-flows/node-flow-publication-service.ts @@ -0,0 +1,19 @@ +import { EntityNotFoundError } from "../../repositories/repository-utils.js"; +import type { NodeFlowRepository } from "../../repositories/node-flow-repository.js"; +import type { NodeFlowPublicationRecord } from "../../contracts/node-flow-types.js"; +import type { NodeFlowVersionSelection } from "../../contracts/node-flow-execution-policy-types.js"; + +export class NodeFlowPublicationService { + constructor(private readonly repository: NodeFlowRepository) {} + + resolve(flowId: string, selection: NodeFlowVersionSelection): NodeFlowPublicationRecord { + const publication = selection.mode === "pinned" + ? this.repository.getPublication(flowId, selection.version) + : this.repository.getPublication(flowId); + if (!publication) { + const suffix = selection.mode === "pinned" ? ` at version ${selection.version}` : ""; + throw new EntityNotFoundError(`Published node flow not found: ${flowId}${suffix}`); + } + return publication; + } +} diff --git a/src/services/node-flows/node-flow-queue-service.ts b/src/services/node-flows/node-flow-queue-service.ts new file mode 100644 index 0000000000..f96241ebfc --- /dev/null +++ b/src/services/node-flows/node-flow-queue-service.ts @@ -0,0 +1,18 @@ +import type { NodeFlowRepository } from "../../repositories/node-flow-repository.js"; +import type { NodeFlowRunRecord } from "../../contracts/node-flow-types.js"; + +export class NodeFlowQuotaExceededError extends Error {} + +export class NodeFlowQueueService { + constructor(private readonly repository: NodeFlowRepository) {} + + claim(run: NodeFlowRunRecord, executorId: string): NodeFlowRunRecord { + if (this.repository.countActiveRuns() >= run.policy.maxConcurrentRuns + || this.repository.countActiveRuns(run.projectId) >= run.policy.maxConcurrentRunsPerProject) { + throw new NodeFlowQuotaExceededError("Node flow concurrency quota is exhausted."); + } + const claimed = this.repository.claimQueuedRun(run.id, executorId, run.policy.leaseDurationMs); + if (!claimed) throw new Error("Node flow run could not be claimed."); + return claimed; + } +} diff --git a/src/services/node-flows/node-flow-recovery-service.ts b/src/services/node-flows/node-flow-recovery-service.ts new file mode 100644 index 0000000000..2af71113cb --- /dev/null +++ b/src/services/node-flows/node-flow-recovery-service.ts @@ -0,0 +1,27 @@ +import type { NodeFlowRepository } from "../../repositories/node-flow-repository.js"; +import type { NodeFlowRunRecord } from "../../contracts/node-flow-types.js"; + +export class NodeFlowRecoveryService { + constructor(private readonly repository: NodeFlowRepository) {} + + recover(now = new Date()): NodeFlowRunRecord[] { + const recoverable = this.repository.listRecoverableRuns(now.toISOString()); + return recoverable.map((run) => { + if (run.status !== "running") return run; + const attempts = this.repository.listNodeAttempts(run.id); + const active = attempts.find((attempt) => attempt.status === "running"); + if (active) { + const nextStatus = active.invocationId ? "attention_required" : "queued"; + return this.repository.updateRun(run.id, { + status: nextStatus, + errorMessage: active.invocationId + ? "An externally observable attempt lost its lease; its outcome is unknown and requires attention." + : "Lease expired before an external invocation began; the run was safely requeued.", + leaseOwner: null, + leaseExpiresAt: null, + }); + } + return this.repository.updateRun(run.id, { status: "queued", leaseOwner: null, leaseExpiresAt: null }); + }); + } +} diff --git a/src/services/scheduler-service.ts b/src/services/scheduler-service.ts index 70d8ab7c9e..8fc5dd55f8 100644 --- a/src/services/scheduler-service.ts +++ b/src/services/scheduler-service.ts @@ -38,7 +38,7 @@ export interface SchedulerServiceDeps { taskRerunService?: TaskRerunService; memoryRemediationService?: MemoryRemediationService; nodeFlowRuntimeService?: NodeFlowRuntimeService; - nodeFlowRepository?: Pick; + nodeFlowRepository?: Pick; logger: Logger; tickIntervalMs?: number; } @@ -403,6 +403,9 @@ export class SchedulerService { target.input ?? {}, { triggerType: "scheduler", + versionSelection: target.versionSelection ?? (target.flowVersion !== undefined + ? { mode: "pinned", version: target.flowVersion } + : { mode: "latest_published" }), triggerPayload: { schedulerEntryId: entry.id, scheduledFor: occurrenceIso, @@ -440,6 +443,12 @@ export class SchedulerService { throw new Error("nodeFlowTarget.flowId is required."); } this.validateNodeFlowTargetOwnership(projectId, flowId); + const selection = input.nodeFlowTarget?.versionSelection + ?? (input.nodeFlowTarget?.flowVersion !== undefined ? { mode: "pinned" as const, version: input.nodeFlowTarget.flowVersion } : { mode: "latest_published" as const }); + if (selection.mode === "pinned" && typeof this.deps.nodeFlowRepository?.getPublication === "function" + && !this.deps.nodeFlowRepository.getPublication(flowId, selection.version)) { + throw new Error("Scheduled node flow version must reference a published version."); + } return; } diff --git a/tests/backend/mcp/management-node-flow-actions.test.ts b/tests/backend/mcp/management-node-flow-actions.test.ts index 54711b8284..dbd69abd52 100644 --- a/tests/backend/mcp/management-node-flow-actions.test.ts +++ b/tests/backend/mcp/management-node-flow-actions.test.ts @@ -112,6 +112,7 @@ describe("manage_node_flows", () => { expect(nodeFlowService.runFlow).toHaveBeenCalledWith("project-1", "flow-1", { prompt: "Ship" }, { triggerType: "mcp_management", + versionSelection: { mode: "latest_published" }, }); expect(parsed.result.run.id).toBe("run-1"); expect(parsed.result.output).toEqual({ ok: true }); diff --git a/tests/backend/repositories/scheduler-repository.test.ts b/tests/backend/repositories/scheduler-repository.test.ts index 1bac634aa3..6a3b1cbf78 100644 --- a/tests/backend/repositories/scheduler-repository.test.ts +++ b/tests/backend/repositories/scheduler-repository.test.ts @@ -390,6 +390,7 @@ describe("SchedulerRepository", () => { flowId: "flow-1", input: { prompt: "Ship it", count: 2, nested: { ok: true } }, flowVersion: 3, + versionSelection: { mode: "pinned", version: 3 }, }); expect(schedulerRepository.getEntry(entry.id)?.nodeFlowTarget).toEqual(entry.nodeFlowTarget); @@ -411,6 +412,7 @@ describe("SchedulerRepository", () => { expect(updated.nodeFlowTarget).toEqual({ flowId: "flow-2", input: { next: true }, + versionSelection: { mode: "latest_published" }, }); }); diff --git a/tests/backend/server/node-flow-routes.test.ts b/tests/backend/server/node-flow-routes.test.ts index add15bec4d..662fa498a1 100644 --- a/tests/backend/server/node-flow-routes.test.ts +++ b/tests/backend/server/node-flow-routes.test.ts @@ -138,6 +138,7 @@ describe("node flow routes", () => { expect(nodeFlowService.runFlow).toHaveBeenCalledWith("project-1", "flow-1", { prompt: "Ship" }, { triggerType: "manual", triggerPayload: undefined, + versionSelection: { mode: "latest_published" }, }); }); }); diff --git a/tests/backend/services/node-flow-recovery-service.test.ts b/tests/backend/services/node-flow-recovery-service.test.ts new file mode 100644 index 0000000000..918f0e4719 --- /dev/null +++ b/tests/backend/services/node-flow-recovery-service.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it } from "vitest"; +import * as fs from "fs/promises"; +import * as os from "os"; +import * as path from "path"; +import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; +import { NodeFlowRepository } from "../../../src/repositories/node-flow-repository.js"; +import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; +import { NodeFlowRecoveryService } from "../../../src/services/node-flows/node-flow-recovery-service.js"; +import { NodeFlowQueueService, NodeFlowQuotaExceededError } from "../../../src/services/node-flows/node-flow-queue-service.js"; +import { DEFAULT_NODE_FLOW_EXECUTION_POLICY } from "../../../src/contracts/node-flow-execution-policy-types.js"; + +const dirs: string[] = []; +afterEach(async () => Promise.all(dirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })))); + +describe("NodeFlowRecoveryService", () => { + it("enforces the immutable project concurrency quota before claiming", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-quota-")); dirs.push(dir); + const storage = new AppDbStorage(path.join(dir, "app.db")); + const projects = new ProjectManagementRepository(storage); const repository = new NodeFlowRepository(storage); + const project = projects.createProject({ name: "Quota", sourceType: "local", sourceRef: dir }); + const flow = repository.createFlow(project.id, { title: "Quota", graph: { nodes: [{ id: "input", type: "input", title: "Input" }], edges: [] } }); + const run = repository.createRun({ flowId: flow.id, projectId: project.id, version: 1, status: "queued", policy: { ...DEFAULT_NODE_FLOW_EXECUTION_POLICY, maxConcurrentRunsPerProject: 0 } }); + + expect(() => new NodeFlowQueueService(repository).claim(run, "executor")).toThrow(NodeFlowQuotaExceededError); + expect(repository.getRun(run.id)?.status).toBe("queued"); + }); + + it("requeues an expired pre-invocation attempt without creating a duplicate attempt", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-recovery-")); dirs.push(dir); + const storage = new AppDbStorage(path.join(dir, "app.db")); + const projects = new ProjectManagementRepository(storage); const repository = new NodeFlowRepository(storage); + const project = projects.createProject({ name: "Recovery", sourceType: "local", sourceRef: dir }); + const flow = repository.createFlow(project.id, { title: "Recover", graph: { nodes: [{ id: "input", type: "input", title: "Input" }], edges: [] } }); + const run = repository.createRun({ flowId: flow.id, projectId: project.id, version: 1, status: "running" }); + repository.updateRun(run.id, { leaseOwner: "dead", leaseExpiresAt: "2020-01-01T00:00:00.000Z" }); + const nodeRun = repository.createNodeRun({ runId: run.id, flowId: flow.id, projectId: project.id, nodeId: "input", status: "running" }); + repository.createNodeAttempt({ runId: run.id, nodeRunId: nodeRun.id, nodeId: "input", attemptNumber: 1, status: "running", executorId: "dead", invocationId: null, artifactDigest: null, input: {}, output: null, credentialIds: [], failureClassification: null, retryDecision: null, errorMessage: null, startedAt: "2020-01-01T00:00:00.000Z", finishedAt: null }); + + const [recovered] = new NodeFlowRecoveryService(repository).recover(); + expect(recovered?.status).toBe("queued"); + expect(repository.listNodeAttempts(run.id)).toHaveLength(1); + }); + + it("requires attention when an expired attempt has an external invocation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-recovery-")); dirs.push(dir); + const storage = new AppDbStorage(path.join(dir, "app.db")); + const projects = new ProjectManagementRepository(storage); const repository = new NodeFlowRepository(storage); + const project = projects.createProject({ name: "Recovery External", sourceType: "local", sourceRef: dir }); + const flow = repository.createFlow(project.id, { title: "Recover", graph: { nodes: [{ id: "http", type: "http_request", title: "HTTP" }], edges: [] } }); + const run = repository.createRun({ flowId: flow.id, projectId: project.id, version: 1, status: "running" }); + repository.updateRun(run.id, { leaseOwner: "dead", leaseExpiresAt: "2020-01-01T00:00:00.000Z" }); + const nodeRun = repository.createNodeRun({ runId: run.id, flowId: flow.id, projectId: project.id, nodeId: "http", status: "running" }); + repository.createNodeAttempt({ runId: run.id, nodeRunId: nodeRun.id, nodeId: "http", attemptNumber: 1, status: "running", executorId: "dead", invocationId: "external-1", artifactDigest: null, input: {}, output: null, credentialIds: [], failureClassification: null, retryDecision: null, errorMessage: null, startedAt: "2020-01-01T00:00:00.000Z", finishedAt: null }); + + const [recovered] = new NodeFlowRecoveryService(repository).recover(); + expect(recovered?.status).toBe("attention_required"); + expect(recovered?.errorMessage).toMatch(/outcome is unknown/i); + }); +}); diff --git a/tests/backend/services/node-flow-runtime-service.test.ts b/tests/backend/services/node-flow-runtime-service.test.ts index 072726e075..6a94053222 100644 --- a/tests/backend/services/node-flow-runtime-service.test.ts +++ b/tests/backend/services/node-flow-runtime-service.test.ts @@ -46,6 +46,51 @@ afterEach(async () => { }); describe("NodeFlowRuntimeService", () => { + it("executes an explicitly pinned publication while latest selection follows the newest publication", async () => { + const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(); + const project = projectRepository.createProject({ name: "Version Project", sourceType: "local", sourceRef: dir }); + const flow = nodeFlowRepository.createFlow(project.id, { title: "Versioned", graph: { nodes: [{ id: "set", type: "set_fields", title: "Set", data: { fields: { release: "v1" } } }], edges: [] } }); + nodeFlowRepository.updateFlow(flow.id, { graph: { nodes: [{ id: "set", type: "set_fields", title: "Set", data: { fields: { release: "v2" } } }], edges: [] } }); + + const pinned = await runtime.runFlow(project.id, flow.id, {}, { versionSelection: { mode: "pinned", version: 1 } }); + const latest = await runtime.runFlow(project.id, flow.id, {}, { versionSelection: { mode: "latest_published" } }); + + expect(pinned.run.version).toBe(1); + expect(pinned.output).toEqual({ release: "v1" }); + expect(latest.run.version).toBe(2); + expect(latest.output).toEqual({ release: "v2" }); + }); + + it("retries classified transient failures and persists redacted numbered attempts", async () => { + const executeProvider = vi.fn() + .mockRejectedValueOnce(new Error("503 temporarily unavailable")) + .mockResolvedValue({ ok: true, stdout: "ok", stderr: "", code: 0, text: "ok", nativeSessionId: null, usageTelemetry: { transcriptText: "ok", inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningOutputTokens: 0, totalTokens: 0, usageSource: "reported", rawUsageJson: null } }); + const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime({ executeProvider } as Partial); + const project = projectRepository.createProject({ name: "Retry Project", sourceType: "local", sourceRef: dir }); + const flow = nodeFlowRepository.createFlow(project.id, { title: "Retry", graph: { nodes: [{ id: "prompt", type: "provider_prompt", title: "Prompt", data: { provider: "mockup-cli", prompt: "{{input.apiToken}}" }, policy: { retry: { maxAttempts: 2, backoffMs: 0, maxBackoffMs: 0 } } }], edges: [] } }); + + const result = await runtime.runFlow(project.id, flow.id, { apiToken: "never-store" }); + + expect(result.run.status).toBe("succeeded"); + expect(executeProvider).toHaveBeenCalledTimes(2); + expect(result.attempts?.map((attempt) => [attempt.attemptNumber, attempt.retryDecision])).toEqual([[1, "retry"], [2, "stop"]]); + expect(JSON.stringify(result.attempts)).not.toContain("never-store"); + }); + + it("propagates node timeouts to the executor and classifies the attempt", async () => { + const executeProvider = vi.fn().mockImplementation(({ signal }: { signal?: AbortSignal }) => new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + })); + const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime({ executeProvider } as Partial); + const project = projectRepository.createProject({ name: "Timeout Project", sourceType: "local", sourceRef: dir }); + const flow = nodeFlowRepository.createFlow(project.id, { title: "Timeout", graph: { nodes: [{ id: "prompt", type: "provider_prompt", title: "Prompt", data: { provider: "mockup-cli", prompt: "wait" }, policy: { timeout: { timeoutMs: 5 } } }], edges: [] } }); + + const result = await runtime.runFlow(project.id, flow.id, {}); + + expect(result.run.status).toBe("failed"); + expect(result.attempts?.[0]).toMatchObject({ failureClassification: "timeout", retryDecision: "stop" }); + }); + it("executes deterministic nodes in topological order and persists the succeeded run", async () => { const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(); const project = projectRepository.createProject({ name: "Runtime Project", sourceType: "local", sourceRef: dir }); diff --git a/tests/backend/services/scheduler-service.test.ts b/tests/backend/services/scheduler-service.test.ts index 4bac0cd5b2..cf21d93cb2 100644 --- a/tests/backend/services/scheduler-service.test.ts +++ b/tests/backend/services/scheduler-service.test.ts @@ -533,6 +533,7 @@ describe("SchedulerService", () => { expect(nodeFlowRuntimeService.runFlow).toHaveBeenCalledWith("project-1", "flow-1", { prompt: "Ship" }, { triggerType: "scheduler", + versionSelection: { mode: "pinned", version: 2 }, triggerPayload: { schedulerEntryId: "entry-1", scheduledFor: "2026-05-18T09:00:00.000Z", From d53fe2f507ed3485ae0af8ec52a2e7a65871068f Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 05:17:37 +0000 Subject: [PATCH 04/25] feat(task T04): implement via codex --- .../node-flow-builtins-and-security.md | 25 ++ docs-web/architecture/node-flows.md | 10 +- ...ecture-node-flow-builtins-and-security.mdx | 25 ++ .../content/docs/architecture-node-flows.mdx | 10 +- docs-web/content/docs/registry.ts | 9 + ...e-node-flow-builtins-and-security.lazy.tsx | 11 + docs/SUMMARY.md | 1 + .../node-flow-builtins-and-security.md | 58 +++++ docs/architecture/node-flows.md | 12 +- docs/index.md | 2 + src/app/dependency-factory/core-factory.ts | 12 + .../dependency-factory/dashboard-factory.ts | 20 ++ src/contracts/node-flow-types.ts | 4 +- .../node-flows/node-definition-registry.ts | 51 ++++ .../automation-approval-repository.ts | 112 +++++++++ .../automation-outbox-repository.ts | 83 +++++++ .../automation-webhook-trigger-repository.ts | 51 ++++ src/repositories/db/app-db-migrations.ts | 71 ++++++ src/server/dashboard-route-registration.ts | 2 + src/server/dashboard-server.ts | 4 + src/server/node-flow-routes.ts | 29 +++ src/server/node-flow-webhook-routes.ts | 26 ++ src/services/node-flow-runtime-service.ts | 121 +++++++++- src/services/node-flows/approval-service.ts | 37 +++ .../node-flows/builtins/builtin-executors.ts | 142 +++++++++++ src/services/node-flows/builtins/index.ts | 1 + .../node-flows/egress-policy-service.ts | 224 ++++++++++++++++++ src/services/node-flows/oauth-broker.ts | 107 +++++++++ src/services/node-flows/outbox-service.ts | 42 ++++ .../node-definition-registry.test.ts | 11 +- ...automation-governance-repositories.test.ts | 50 ++++ .../server/node-flow-webhook-routes.test.ts | 20 ++ .../services/node-flow-builtins.test.ts | 28 +++ .../node-flow-egress-policy-service.test.ts | 42 ++++ .../services/node-flow-oauth-broker.test.ts | 32 +++ .../node-flow-runtime-service.test.ts | 56 +++-- 36 files changed, 1499 insertions(+), 42 deletions(-) create mode 100644 docs-web/architecture/node-flow-builtins-and-security.md create mode 100644 docs-web/content/docs/architecture-node-flow-builtins-and-security.mdx create mode 100644 docs-web/routes/docs.architecture-node-flow-builtins-and-security.lazy.tsx create mode 100644 docs/architecture/node-flow-builtins-and-security.md create mode 100644 src/repositories/automation-approval-repository.ts create mode 100644 src/repositories/automation-outbox-repository.ts create mode 100644 src/repositories/automation-webhook-trigger-repository.ts create mode 100644 src/server/node-flow-webhook-routes.ts create mode 100644 src/services/node-flows/approval-service.ts create mode 100644 src/services/node-flows/builtins/builtin-executors.ts create mode 100644 src/services/node-flows/builtins/index.ts create mode 100644 src/services/node-flows/egress-policy-service.ts create mode 100644 src/services/node-flows/oauth-broker.ts create mode 100644 src/services/node-flows/outbox-service.ts create mode 100644 tests/backend/repositories/automation-governance-repositories.test.ts create mode 100644 tests/backend/server/node-flow-webhook-routes.test.ts create mode 100644 tests/backend/services/node-flow-builtins.test.ts create mode 100644 tests/backend/services/node-flow-egress-policy-service.test.ts create mode 100644 tests/backend/services/node-flow-oauth-broker.test.ts diff --git a/docs-web/architecture/node-flow-builtins-and-security.md b/docs-web/architecture/node-flow-builtins-and-security.md new file mode 100644 index 0000000000..74a54d6688 --- /dev/null +++ b/docs-web/architecture/node-flow-builtins-and-security.md @@ -0,0 +1,25 @@ +# Node Flow Built-ins and External-Effect Security + +The governed catalog adds deterministic branches, bounded collection processing, durable approvals, and replay-safe external effects while keeping the versioned definition registry as the executable authority. + +## Control and integration nodes + +- `condition` selects `true` or `false`; `switch` selects one named case or `default`. Unselected branches persist as skipped node runs. +- `foreach` accepts no more than 1,000 items. `merge` supports `object`, `array`, and `first` strategies. +- `delay` is cancellable and capped at one hour. `execute_subflow` requires same-project ownership, rejects direct self-reference, and caps depth at eight. +- `approval` persists an operator decision. `email_draft` never sends. `email_send` requires approval and uses the idempotent outbox. +- `webhook_trigger` emits payloads accepted through secret-authenticated webhook ingress. + +## Network policy + +HTTP nodes and future custom nodes use the same `EgressPolicyService`. HTTPS is required unless HTTP is explicitly enabled. Private, loopback, link-local, metadata, multicast, and other non-public addresses remain blocked in both modes. Credentials in URLs and raw restricted headers are rejected. + +Every redirect is manually revalidated. DNS is checked for private results and rebinding. Host and port allowlists, response-size and content-type limits, propagated cancellation and timeouts, capped retries, idempotency requirements for unsafe retry, normalized headers, and per-key rate windows keep requests bounded. + +## OAuth, approvals, and outbox + +OAuth authorization uses PKCE S256 and short-lived AES-256-GCM state tied to an allowlisted callback origin. Tokens live behind the connection store, rotate on refresh, enforce scopes and expiry, and are never written into graph JSON or agent-visible output. Revocation, reconnect, and health checks expose no token values. + +Approvals are unique per run, node, and logical item. Outbox entries use a unique key derived from publication, run, node, and logical item, and store the provider message id after success. A restart while an entry is sending changes it to `attention_required`; Code UX does not automatically replay an unknown provider outcome. + +Webhook configuration returns a newly rotated path token and secret while persisting only their hashes. Ingress requires `x-codeux-webhook-secret` and dispatches the latest published flow version. diff --git a/docs-web/architecture/node-flows.md b/docs-web/architecture/node-flows.md index f3f15bfffa..ed7653f450 100644 --- a/docs-web/architecture/node-flows.md +++ b/docs-web/architecture/node-flows.md @@ -11,8 +11,16 @@ Node flows are project-owned, versioned Graph v2 workflows. | `template` | Renders text templates. | | `provider_prompt` | Invokes a configured CLI provider. | | `http_request` | Performs a bounded HTTP/HTTPS request. | +| `condition`, `switch` | Selects one explicit output branch and persists unselected branches as skipped. | +| `foreach`, `merge` | Bounds item fan-out and combines active inputs with an explicit strategy. | +| `delay`, `approval` | Waits with cancellation or persists an operator decision gate. | +| `email_draft`, `email_send` | Produces a draft, or sends only after approval through the idempotent outbox. | +| `execute_subflow` | Executes a same-project published flow with recursion bounds. | +| `webhook_trigger` | Emits secret-authenticated webhook input. | | `output` | Selects the result. | -These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. +These are the executable definitions. Other custom palette concepts remain non-executable until a versioned handler is registered. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](./node-flow-durable-execution.md). + +HTTP and future custom-node requests share one HTTPS-first egress policy with URL-credential rejection, DNS and redirect revalidation, private-network and metadata blocking, host/port allowlists, bounded content, retries, timeouts, and rate limits. See [Built-ins and External-Effect Security](./node-flow-builtins-and-security.md). diff --git a/docs-web/content/docs/architecture-node-flow-builtins-and-security.mdx b/docs-web/content/docs/architecture-node-flow-builtins-and-security.mdx new file mode 100644 index 0000000000..74a54d6688 --- /dev/null +++ b/docs-web/content/docs/architecture-node-flow-builtins-and-security.mdx @@ -0,0 +1,25 @@ +# Node Flow Built-ins and External-Effect Security + +The governed catalog adds deterministic branches, bounded collection processing, durable approvals, and replay-safe external effects while keeping the versioned definition registry as the executable authority. + +## Control and integration nodes + +- `condition` selects `true` or `false`; `switch` selects one named case or `default`. Unselected branches persist as skipped node runs. +- `foreach` accepts no more than 1,000 items. `merge` supports `object`, `array`, and `first` strategies. +- `delay` is cancellable and capped at one hour. `execute_subflow` requires same-project ownership, rejects direct self-reference, and caps depth at eight. +- `approval` persists an operator decision. `email_draft` never sends. `email_send` requires approval and uses the idempotent outbox. +- `webhook_trigger` emits payloads accepted through secret-authenticated webhook ingress. + +## Network policy + +HTTP nodes and future custom nodes use the same `EgressPolicyService`. HTTPS is required unless HTTP is explicitly enabled. Private, loopback, link-local, metadata, multicast, and other non-public addresses remain blocked in both modes. Credentials in URLs and raw restricted headers are rejected. + +Every redirect is manually revalidated. DNS is checked for private results and rebinding. Host and port allowlists, response-size and content-type limits, propagated cancellation and timeouts, capped retries, idempotency requirements for unsafe retry, normalized headers, and per-key rate windows keep requests bounded. + +## OAuth, approvals, and outbox + +OAuth authorization uses PKCE S256 and short-lived AES-256-GCM state tied to an allowlisted callback origin. Tokens live behind the connection store, rotate on refresh, enforce scopes and expiry, and are never written into graph JSON or agent-visible output. Revocation, reconnect, and health checks expose no token values. + +Approvals are unique per run, node, and logical item. Outbox entries use a unique key derived from publication, run, node, and logical item, and store the provider message id after success. A restart while an entry is sending changes it to `attention_required`; Code UX does not automatically replay an unknown provider outcome. + +Webhook configuration returns a newly rotated path token and secret while persisting only their hashes. Ingress requires `x-codeux-webhook-secret` and dispatches the latest published flow version. diff --git a/docs-web/content/docs/architecture-node-flows.mdx b/docs-web/content/docs/architecture-node-flows.mdx index 31bf1274aa..b19d3c6608 100644 --- a/docs-web/content/docs/architecture-node-flows.mdx +++ b/docs-web/content/docs/architecture-node-flows.mdx @@ -11,8 +11,16 @@ Node flows are project-owned, versioned Graph v2 workflows. | `template` | Renders text templates. | | `provider_prompt` | Invokes a configured CLI provider. | | `http_request` | Performs a bounded HTTP/HTTPS request. | +| `condition`, `switch` | Selects one explicit output branch and persists unselected branches as skipped. | +| `foreach`, `merge` | Bounds item fan-out and combines active inputs with an explicit strategy. | +| `delay`, `approval` | Waits with cancellation or persists an operator decision gate. | +| `email_draft`, `email_send` | Produces a draft, or sends only after approval through the idempotent outbox. | +| `execute_subflow` | Executes a same-project published flow with recursion bounds. | +| `webhook_trigger` | Emits secret-authenticated webhook input. | | `output` | Selects the result. | -These are the only executable definitions. Trigger, agent-router, task, condition, notification, and other palette concepts are planned entries without runtime handlers. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. +These are the executable definitions. Other custom palette concepts remain non-executable until a versioned handler is registered. Graph v1 migration preserves the legacy snapshot and appends deterministic v2. Execution uses immutable publications rather than the mutable editor row. Runs select a pinned publication or the latest published version, then use durable queue claims, leases, bounded quotas, timeout/cancellation propagation, and numbered retry attempts. Expired external attempts with unknown outcomes require operator attention and are not silently replayed. See [Node Flow Durable Execution](/docs/architecture-node-flow-durable-execution). + +HTTP and future custom-node requests share one HTTPS-first egress policy with URL-credential rejection, DNS and redirect revalidation, private-network and metadata blocking, host/port allowlists, bounded content, retries, timeouts, and rate limits. See [Built-ins and External-Effect Security](/docs/architecture-node-flow-builtins-and-security). diff --git a/docs-web/content/docs/registry.ts b/docs-web/content/docs/registry.ts index 637afe7f3a..42d840a534 100644 --- a/docs-web/content/docs/registry.ts +++ b/docs-web/content/docs/registry.ts @@ -108,6 +108,7 @@ export type DocsSlug = | 'user-dashboard-custom-dashboards' | 'architecture-custom-dashboard-foundation' | 'architecture-managed-container-runtime' + | 'architecture-node-flow-builtins-and-security' | 'architecture-node-flow-durable-execution' | 'architecture-node-flow-foundation' | 'architecture-node-flows' @@ -859,6 +860,13 @@ export const docsRegistry: Record = { title: "Managed Container Runtime", description: "The managed container runtime removes first-invocation Docker builds while keeping provider binaries local to each user's Docker host.", }, + 'architecture-node-flow-builtins-and-security': { + id: 'architecture-node-flow-builtins-and-security', + path: '/docs/architecture-node-flow-builtins-and-security', + section: 'Architecture', + title: "Node Flow Built-ins and External-Effect Security", + description: "The governed catalog adds deterministic branches, bounded collection processing, durable approvals, and replay-safe external effects while keeping the versioned definition registry as the executable authority.", + }, 'architecture-node-flow-durable-execution': { id: 'architecture-node-flow-durable-execution', path: '/docs/architecture-node-flow-durable-execution', @@ -1009,6 +1017,7 @@ export const orderedDocs: DocsRegistryEntry[] = [ docsRegistry['user-dashboard-custom-dashboards'], docsRegistry['architecture-custom-dashboard-foundation'], docsRegistry['architecture-managed-container-runtime'], + docsRegistry['architecture-node-flow-builtins-and-security'], docsRegistry['architecture-node-flow-durable-execution'], docsRegistry['architecture-node-flow-foundation'], docsRegistry['architecture-node-flows'], diff --git a/docs-web/routes/docs.architecture-node-flow-builtins-and-security.lazy.tsx b/docs-web/routes/docs.architecture-node-flow-builtins-and-security.lazy.tsx new file mode 100644 index 0000000000..85e5318bd6 --- /dev/null +++ b/docs-web/routes/docs.architecture-node-flow-builtins-and-security.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import ArchitectureNodeFlowBuiltinsAndSecurityContent from '../content/docs/architecture-node-flow-builtins-and-security.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/architecture-node-flow-builtins-and-security')({ + component: () => ( + + + + ) +}) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 933e7c124d..820eada147 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -40,6 +40,7 @@ - [Node Flow Foundation](./architecture/node-flow-foundation.md) - [Node Flows](./architecture/node-flows.md) - [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) +- [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/docs/architecture/node-flow-builtins-and-security.md b/docs/architecture/node-flow-builtins-and-security.md new file mode 100644 index 0000000000..1156cb9ca5 --- /dev/null +++ b/docs/architecture/node-flow-builtins-and-security.md @@ -0,0 +1,58 @@ +# Node Flow Built-ins and External-Effect Security + +The governed built-in catalog extends publication-based node-flow execution with deterministic control nodes and durable boundaries for external effects. The definition registry remains the executable authority; a graph can only run a node when its versioned manifest is registered and executable. + +## Built-in catalog + +| Node | Contract | +| --- | --- | +| `condition` | Evaluates a bounded operator and selects exactly the `true` or `false` output port. Unselected branches persist as skipped node runs. | +| `switch` | Evaluates no more than 100 configured cases and selects one named case or `default`. | +| `foreach` | Validates an array and emits at most 1,000 items. Inputs above the configured bound fail before fan-out. | +| `merge` | Combines active upstream values with `object`, `array`, or `first` strategy. | +| `delay` | Waits for a cancellable duration from zero through one hour. | +| `approval` | Creates or reuses a durable approval keyed by run, node, and logical item. | +| `email_draft` | Produces a draft only and never contacts a provider. | +| `email_send` | Requires an approved decision, then dispatches through the idempotent outbox. | +| `execute_subflow` | Executes a published flow owned by the same project, rejects direct self-reference, and caps nesting at eight. | +| `webhook_trigger` | Emits input accepted by a secret-authenticated webhook configuration. | + +The existing `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output` nodes retain their previous contracts. Typed manifest ports identify branch handles, many-valued merge inputs, and trigger outputs. Branch routing only runs a node when at least one incoming edge is active, allowing merges to join a selected path without treating an unselected sibling as a failure. + +## Governed egress + +`EgressPolicyService` is the single request boundary for HTTP nodes and future custom-node network calls. HTTPS is required by default. A node must explicitly opt into HTTP, and even then private networking remains blocked. The service rejects credentials embedded in URLs; loopback, private, link-local, carrier-grade NAT, benchmarking, multicast, and cloud-metadata addresses; metadata hostnames; restricted raw headers; and ports or hosts outside configured allowlists. + +Each redirect is handled manually and fully revalidated. DNS is resolved twice before dispatch, and a changed or newly private result is treated as rebinding. Cross-origin redirects remove credential headers. Response bodies are streamed into a bounded buffer, content types are allowlisted, timeouts and caller cancellation propagate, retry counts are capped, unsafe methods require an idempotency key before retry, and an in-process rate window bounds requests per project and host. + +## OAuth boundary + +`OAuthBroker` implements authorization-code flow with PKCE S256. Authorization state is authenticated AES-256-GCM ciphertext containing a short expiry, callback origin, redirect URI, verifier, connection id, and nonce. Callback origins must be explicitly allowlisted and match the state. Token exchange and refresh results are stored behind an `OAuthConnectionStore`; access and refresh tokens are returned only to provider-bound execution code, never to graph JSON or agent-visible output. + +Refresh happens shortly before expiry and rotates the stored refresh token when the provider returns one. Required scopes are checked before access. Revocation deletes local state after provider revocation; reconnect begins from a revoked local connection; health checks refresh when necessary and expose only health, expiry, and scopes. + +## Approvals and outbox + +`automation_approvals` persists pending and terminal decisions. Repeating the same run, node, and logical item returns the existing decision, so restarts do not create a second prompt. Email sending is approval-gated by default; `email_draft` is the non-irreversible default. + +`automation_outbox` has a unique SHA-256 idempotency key derived from publication id, run id, node id, and logical item. Provider message ids are stored after success. A process restart while an entry is `sending` changes it to `attention_required`, because the provider may have accepted the operation; Code UX does not replay an unknown external outcome automatically. + +## Webhook routes + +Creating `POST /api/node-flows/:flowId/webhook` rotates and returns a path token and secret once. Only their hashes are persisted. `POST /api/webhooks/node-flows/:pathToken` requires the secret in `x-codeux-webhook-secret`, uses constant-time digest comparison, and dispatches the latest published version with `triggerType: webhook`. The response returns only run identity and status. + +Example condition edges use explicit handles: + +```json +{ + "nodes": [ + { "id": "check", "type": "condition", "title": "Check", "data": { "path": "input.enabled" } }, + { "id": "draft", "type": "email_draft", "title": "Draft", "data": { "to": "owner@example.test", "subject": "Ready", "body": "Review this draft." } }, + { "id": "done", "type": "output", "title": "Done" } + ], + "edges": [ + { "fromNodeId": "check", "fromHandle": "true", "toNodeId": "draft" }, + { "fromNodeId": "check", "fromHandle": "false", "toNodeId": "done" } + ] +} +``` diff --git a/docs/architecture/node-flows.md b/docs/architecture/node-flows.md index bc355cf310..f8ba39cec3 100644 --- a/docs/architecture/node-flows.md +++ b/docs/architecture/node-flows.md @@ -50,11 +50,19 @@ Runtime-supported node types are: | `template` | Renders `template` or `prompt` into `outputKey` (default `text`). | | `provider_prompt` | Renders a prompt and calls an existing CLI provider configuration through `ProviderExecutionService`. | | `http_request` | Performs bounded HTTP/HTTPS requests with method, URL, headers, query, body, timeout, and optional JSON path extraction. | +| `condition`, `switch` | Select one explicit output branch; non-selected branches are persisted as skipped. | +| `foreach` | Validates and emits a bounded item list. | +| `merge` | Combines active inputs with `object`, `array`, or `first` strategy. | +| `delay` | Waits for a bounded cancellable duration. | +| `approval` | Persists an operator decision gate. | +| `email_draft`, `email_send` | Creates a draft, or sends only after approval through the idempotent outbox. | +| `execute_subflow` | Executes a same-project published subflow with recursion bounds. | +| `webhook_trigger` | Emits authenticated webhook input. | | `output` | Selects final output from a path, configured fields, or upstream output. | Template interpolation reads from `{{ input.path }}` and `{{ nodes.nodeId.path }}`. Node config is built from widget defaults, node `data`, and optional `data.values`, with later values overriding defaults. -Provider prompt nodes require a configured CLI provider. HTTP nodes require `http` or `https` URLs and support `GET`, `POST`, `PUT`, `PATCH`, `DELETE`, and `HEAD`. HTTP timeout defaults to 30 seconds and is capped at 60 seconds. +Provider prompt nodes require a configured CLI provider. HTTP nodes require HTTPS unless HTTP is explicitly enabled and support `GET`, `POST`, `PUT`, `PATCH`, `DELETE`, and `HEAD`. Requests pass through the shared SSRF, redirect, DNS, response-size, content-type, retry, timeout, and rate-limit policy described in [Node Flow Built-ins and External-Effect Security](./node-flow-builtins-and-security.md). ## Invocation Tracking @@ -105,6 +113,6 @@ Use these rules: Graph v2 is the single workflow model used by backend, MCP, runtime, and dashboard. It adds `schemaVersion: 2`, stable definition references, typed ports and flow schemas, credential-id bindings, retry and timeout policies, capability and side-effect metadata, disabled state, and optional immutable publication metadata. Plaintext credentials, secret-shaped fields, generated source, and custom code are not valid graph data. -The executable registry is exactly `input`, `set_fields`, `template`, `provider_prompt`, `http_request`, and `output`. Planned trigger, agent-router, task, condition, notification, and integration entries are not executable until handlers exist. +The executable registry contains the original deterministic/provider/HTTP nodes plus `condition`, `switch`, `foreach`, `merge`, `delay`, `approval`, `email_draft`, `email_send`, `execute_subflow`, and `webhook_trigger`. Unregistered custom types remain non-executable. Backend Graph v1 migration retains the exact prior version and appends deterministic v2. Browser canvas v1 migration returns the untouched legacy snapshot separately from the normalized graph. diff --git a/docs/index.md b/docs/index.md index 21d4ab0ebf..d459e6830d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -67,6 +67,7 @@ Use this page as the main entrypoint. 29. [Node Flow Foundation](./architecture/node-flow-foundation.md) 30. [Node Flows](./architecture/node-flows.md) 31. [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) +32. [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) 31. [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) 32. [Memory Claims and Evidence](./architecture/memory-claims.md) 33. [Speech Input Architecture](./architecture/speech-input.md) @@ -160,6 +161,7 @@ Use this page as the main entrypoint. - [Node Flow Foundation](./architecture/node-flow-foundation.md) - [Node Flows](./architecture/node-flows.md) - [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) +- [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/src/app/dependency-factory/core-factory.ts b/src/app/dependency-factory/core-factory.ts index 55108c588e..4e56765e25 100644 --- a/src/app/dependency-factory/core-factory.ts +++ b/src/app/dependency-factory/core-factory.ts @@ -67,6 +67,9 @@ import { DockerService } from "../../services/docker-service.js"; import { CustomDashboardRepository } from "../../repositories/custom-dashboard-repository.js"; import { CustomDashboardValidationService } from "../../services/custom-dashboard-validation-service.js"; import { AutomationCredentialRepository } from "../../repositories/automation-credential-repository.js"; +import { AutomationApprovalRepository } from "../../repositories/automation-approval-repository.js"; +import { AutomationOutboxRepository } from "../../repositories/automation-outbox-repository.js"; +import { AutomationWebhookTriggerRepository } from "../../repositories/automation-webhook-trigger-repository.js"; import { CredentialBroker } from "../../services/credentials/credential-broker.js"; import { MountedKeyFileProvider } from "../../infrastructure/security/mounted-key-file-provider.js"; import { EncryptedSqliteSecretStore } from "../../infrastructure/security/encrypted-sqlite-secret-store.js"; @@ -131,6 +134,9 @@ export interface CoreDependencies { customDashboardRepository: CustomDashboardRepository; customDashboardValidationService: CustomDashboardValidationService; automationCredentialRepository: AutomationCredentialRepository; + automationApprovalRepository: AutomationApprovalRepository; + automationOutboxRepository: AutomationOutboxRepository; + automationWebhookTriggerRepository: AutomationWebhookTriggerRepository; credentialBroker: CredentialBroker; } @@ -188,6 +194,9 @@ export function createCoreDependencies( const sessionTracking = new SessionTrackingRepository(); const appDbStorage = new AppDbStorage(); const automationCredentialRepository = new AutomationCredentialRepository(appDbStorage); + const automationApprovalRepository = new AutomationApprovalRepository(appDbStorage); + const automationOutboxRepository = new AutomationOutboxRepository(appDbStorage); + const automationWebhookTriggerRepository = new AutomationWebhookTriggerRepository(appDbStorage); const credentialKeyProvider = getProcessCredentialKeyProvider() ?? new MountedKeyFileProvider(process.env.CODE_UX_CREDENTIAL_KEY_FILE); const credentialBroker = new CredentialBroker( @@ -407,6 +416,9 @@ export function createCoreDependencies( customDashboardRepository, customDashboardValidationService, automationCredentialRepository, + automationApprovalRepository, + automationOutboxRepository, + automationWebhookTriggerRepository, credentialBroker, }; } diff --git a/src/app/dependency-factory/dashboard-factory.ts b/src/app/dependency-factory/dashboard-factory.ts index da4d85983f..494ba30a63 100644 --- a/src/app/dependency-factory/dashboard-factory.ts +++ b/src/app/dependency-factory/dashboard-factory.ts @@ -33,6 +33,12 @@ import { NodeFlowRuntimeService } from "../../services/node-flow-runtime-service import { NodeFlowService } from "../../services/node-flow-service.js"; import { NodeFlowRecoveryService } from "../../services/node-flows/node-flow-recovery-service.js"; import { resolveEffectiveDashboardSettings } from "../../services/settings-resolution-service.js"; +import { ApprovalService } from "../../services/node-flows/approval-service.js"; +import { MockSideEffectProvider, OutboxService } from "../../services/node-flows/outbox-service.js"; +import { EgressPolicyService } from "../../services/node-flows/egress-policy-service.js"; +import { AutomationApprovalRepository } from "../../repositories/automation-approval-repository.js"; +import { AutomationOutboxRepository } from "../../repositories/automation-outbox-repository.js"; +import { AutomationWebhookTriggerRepository } from "../../repositories/automation-webhook-trigger-repository.js"; export interface DashboardDependencies { credentialBroker: CoreDependencies["credentialBroker"]; @@ -44,6 +50,8 @@ export interface DashboardDependencies { speechSynthesisService: SpeechSynthesisService; speechModelManager: SpeechModelManager; nodeFlowService: CoreDependencies["nodeFlowService"]; + approvalService: ApprovalService; + automationWebhookTriggerRepository: CoreDependencies["automationWebhookTriggerRepository"]; activityCacheService: ActivityCacheService; taskRerunService: TaskRerunService; executionControlService: ExecutionControlService; @@ -222,6 +230,13 @@ export function createDashboardDependencies( const speechModelManager = new SpeechModelManager( logger.child({ component: "speech-model-manager" }), ); + const approvalRepository = coreDeps.automationApprovalRepository + ?? new AutomationApprovalRepository(coreDeps.appDbStorage); + const outboxRepository = coreDeps.automationOutboxRepository + ?? new AutomationOutboxRepository(coreDeps.appDbStorage); + const webhookTriggerRepository = coreDeps.automationWebhookTriggerRepository + ?? new AutomationWebhookTriggerRepository(coreDeps.appDbStorage); + const approvalService = new ApprovalService(approvalRepository); const nodeFlowRuntimeService = new NodeFlowRuntimeService({ nodeFlowRepository: coreDeps.nodeFlowRepository, executionRepository, @@ -229,6 +244,9 @@ export function createDashboardDependencies( settingsRepository, providerExecutionService, credentialBroker: coreDeps.credentialBroker, + egressPolicyService: new EgressPolicyService(), + approvalService, + outboxService: new OutboxService(outboxRepository, new MockSideEffectProvider()), getDashboardSettings: (projectId) => resolveDashboardSettings({ projectId }), }); if (coreDeps.nodeFlowRepository) { @@ -564,6 +582,8 @@ export function createDashboardDependencies( speechSynthesisService, speechModelManager, nodeFlowService, + approvalService, + automationWebhookTriggerRepository: webhookTriggerRepository, activityCacheService, taskRerunService, executionControlService, diff --git a/src/contracts/node-flow-types.ts b/src/contracts/node-flow-types.ts index cc6471c28b..d675f327d9 100644 --- a/src/contracts/node-flow-types.ts +++ b/src/contracts/node-flow-types.ts @@ -217,7 +217,7 @@ export type NodeFlowRunStatus = | "queued" | "running" | "approval_waiting" | "retry_waiting" | "attention_required" | "succeeded" | "failed" | "cancelled"; export type NodeFlowNodeRunStatus = - | "pending" | "running" | "retry_waiting" | "attention_required" + | "pending" | "running" | "approval_waiting" | "retry_waiting" | "attention_required" | "succeeded" | "failed" | "skipped" | "cancelled"; export interface NodeFlowRunRecord { @@ -353,6 +353,8 @@ export interface RunNodeFlowOptions { triggerPayload?: NodeFlowJsonObject; signal?: AbortSignal; versionSelection?: import("./node-flow-execution-policy-types.js").NodeFlowVersionSelection; + /** Internal recursion guard propagated only by Execute Subflow. */ + subflowDepth?: number; executorId?: string; } diff --git a/src/domain/node-flows/node-definition-registry.ts b/src/domain/node-flows/node-definition-registry.ts index 498c0fc1d4..f32a23f8d5 100644 --- a/src/domain/node-flows/node-definition-registry.ts +++ b/src/domain/node-flows/node-definition-registry.ts @@ -22,6 +22,27 @@ const field = ( required = false, ): NodeWidgetField => ({ id, label, type, required }); +const builtin = (input: { + type: string; label: string; description: string; category: string; + properties?: Record; fields?: NodeWidgetField[]; + ports?: NodeFlowPort[]; sideEffect?: NodeDefinitionManifest["sideEffect"]; + capabilities?: string[]; +}): NodeDefinitionManifest => ({ + type: input.type, + version: 1, + executable: true, + executionKind: "local", + configurationSchema: objectSchema([], input.properties), + ui: { label: input.label, description: input.description, category: input.category, widgetSchema: { fields: input.fields ?? [] } }, + ports: input.ports ?? [dataPort("input", "input"), dataPort("output", "output")], + credentials: [], + capabilities: input.capabilities ?? [], + sideEffect: input.sideEffect ?? "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 } }, + documentation: "docs/architecture/node-flow-builtins-and-security.md", + deprecation: { deprecated: false }, +}); + const manifests: NodeDefinitionManifest[] = [ { type: "input", version: 1, executable: true, executionKind: "local", @@ -67,6 +88,36 @@ const manifests: NodeDefinitionManifest[] = [ defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 }, timeout: { timeoutMs: 30_000 } }, documentation: "docs/architecture/node-flows.md#runtime", deprecation: { deprecated: false }, }, + builtin({ type: "condition", label: "Condition", description: "Selects one explicit boolean branch.", category: "control", + properties: { path: { type: "string" }, operator: { type: "string" }, value: { type: "any" } }, + fields: [field("path", "Value path", "text"), field("operator", "Operator", "select")], + ports: [dataPort("input", "input"), dataPort("true", "output"), dataPort("false", "output")] }), + builtin({ type: "switch", label: "Switch", description: "Selects one named case or the default branch.", category: "control", + properties: { path: { type: "string" }, cases: { type: "array", items: { type: "object" } } }, + fields: [field("path", "Value path", "text"), field("cases", "Cases", "json")], + ports: [dataPort("input", "input"), { ...dataPort("case", "output"), cardinality: "many" }, dataPort("default", "output")] }), + builtin({ type: "foreach", label: "Foreach", description: "Emits a bounded list for deterministic fan-out.", category: "control", + properties: { path: { type: "string" }, maxItems: { type: "number" } }, fields: [field("path", "Items path", "text"), field("maxItems", "Maximum items", "number")], + ports: [dataPort("input", "input"), { ...dataPort("items", "output"), schema: { type: "array", items: { type: "any" } } }, dataPort("empty", "output")] }), + builtin({ type: "merge", label: "Merge", description: "Combines upstream values with an explicit strategy.", category: "transform", + properties: { strategy: { type: "string" } }, fields: [field("strategy", "Strategy", "select")], + ports: [{ ...dataPort("input", "input"), cardinality: "many" }, dataPort("output", "output")] }), + builtin({ type: "delay", label: "Delay", description: "Waits for a bounded duration with cancellation.", category: "control", + properties: { delayMs: { type: "number" } }, fields: [field("delayMs", "Delay (ms)", "number", true)] }), + builtin({ type: "approval", label: "Approval", description: "Persists an operator decision gate.", category: "control", + properties: { summary: { type: "string" }, logicalItem: { type: "string" } }, fields: [field("summary", "Summary", "textarea")], + ports: [dataPort("input", "input"), dataPort("approved", "output"), dataPort("rejected", "output")], sideEffect: "write" }), + builtin({ type: "email_draft", label: "Email Draft", description: "Creates an email draft without sending it.", category: "integration", + properties: { to: { type: "any" }, subject: { type: "string" }, body: { type: "string" } }, + fields: [field("to", "To", "text", true), field("subject", "Subject", "text", true), field("body", "Body", "textarea", true)] }), + builtin({ type: "email_send", label: "Email Send", description: "Sends an approved email through the idempotent outbox.", category: "integration", + properties: { to: { type: "any" }, subject: { type: "string" }, body: { type: "string" }, logicalItem: { type: "string" } }, + fields: [field("to", "To", "text", true), field("subject", "Subject", "text", true), field("body", "Body", "textarea", true)], + sideEffect: "external", capabilities: ["email.send"] }), + builtin({ type: "execute_subflow", label: "Execute Subflow", description: "Executes a project-owned published flow.", category: "control", + properties: { flowId: { type: "string" }, input: { type: "object" } }, fields: [field("flowId", "Flow ID", "text", true), field("input", "Input", "json")] }), + builtin({ type: "webhook_trigger", label: "Webhook Trigger", description: "Emits authenticated webhook input.", category: "trigger", + ports: [dataPort("output", "output")], capabilities: ["webhook.receive"] }), { type: "output", version: 1, executable: true, executionKind: "local", configurationSchema: objectSchema(), diff --git a/src/repositories/automation-approval-repository.ts b/src/repositories/automation-approval-repository.ts new file mode 100644 index 0000000000..b623b3494d --- /dev/null +++ b/src/repositories/automation-approval-repository.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto"; +import { AppDbStorage } from "./app-db-storage.js"; +import type { DatabaseAdapter } from "./db/database-adapter.js"; +import { EntityNotFoundError, ValidationError } from "./repository-utils.js"; +import type { NodeFlowJsonObject } from "../contracts/node-flow-types.js"; + +export type AutomationApprovalStatus = "pending" | "approved" | "rejected" | "expired"; + +export interface AutomationApprovalRecord { + id: string; + projectId: string; + flowId: string; + runId: string; + nodeId: string; + logicalItem: string; + status: AutomationApprovalStatus; + request: NodeFlowJsonObject; + decision: NodeFlowJsonObject | null; + requestedAt: string; + decidedAt: string | null; + decidedBy: string | null; + expiresAt: string | null; + createdAt: string; + updatedAt: string; +} + +interface ApprovalRow { + id: string; project_id: string; flow_id: string; run_id: string; node_id: string; + logical_item: string; status: AutomationApprovalStatus; request_json: string; + decision_json: string | null; requested_at: string; decided_at: string | null; + decided_by: string | null; expires_at: string | null; created_at: string; updated_at: string; +} + +export class AutomationApprovalRepository { + private readonly db: DatabaseAdapter; + + constructor(storage: AppDbStorage = new AppDbStorage()) { + this.db = storage.getDatabase(); + } + + request(input: { + projectId: string; flowId: string; runId: string; nodeId: string; + logicalItem: string; request: NodeFlowJsonObject; expiresAt?: string | null; + }): AutomationApprovalRecord { + const logicalItem = input.logicalItem.trim() || "default"; + const existing = this.getForItem(input.runId, input.nodeId, logicalItem); + if (existing) return existing; + const id = randomUUID(); + const now = new Date().toISOString(); + this.db.prepare(`INSERT INTO automation_approvals + (id, project_id, flow_id, run_id, node_id, logical_item, status, request_json, + requested_at, expires_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?, ?, ?)`) + .run(id, input.projectId, input.flowId, input.runId, input.nodeId, logicalItem, + JSON.stringify(input.request), now, input.expiresAt ?? null, now, now); + return this.require(id); + } + + get(id: string): AutomationApprovalRecord | null { + const row = this.db.prepare("SELECT * FROM automation_approvals WHERE id = ?").get(id) as ApprovalRow | undefined; + return row ? this.map(row) : null; + } + + getForItem(runId: string, nodeId: string, logicalItem = "default"): AutomationApprovalRecord | null { + const row = this.db.prepare("SELECT * FROM automation_approvals WHERE run_id = ? AND node_id = ? AND logical_item = ?") + .get(runId, nodeId, logicalItem) as ApprovalRow | undefined; + return row ? this.map(row) : null; + } + + listForRun(runId: string): AutomationApprovalRecord[] { + return (this.db.prepare("SELECT * FROM automation_approvals WHERE run_id = ? ORDER BY created_at").all(runId) as ApprovalRow[]) + .map((row) => this.map(row)); + } + + decide(id: string, input: { status: "approved" | "rejected"; decidedBy: string; decision?: NodeFlowJsonObject }): AutomationApprovalRecord { + const current = this.require(id); + if (current.status !== "pending") { + if (current.status === input.status) return current; + throw new ValidationError(`Approval ${id} has already been decided.`); + } + const decidedBy = input.decidedBy.trim(); + if (!decidedBy) throw new ValidationError("decidedBy is required."); + const now = new Date().toISOString(); + this.db.prepare(`UPDATE automation_approvals SET status = ?, decision_json = ?, decided_at = ?, + decided_by = ?, updated_at = ? WHERE id = ? AND status = 'pending'`) + .run(input.status, JSON.stringify(input.decision ?? {}), now, decidedBy, now, id); + return this.require(id); + } + + expireDue(now = new Date()): number { + return this.db.prepare(`UPDATE automation_approvals SET status = 'expired', updated_at = ? + WHERE status = 'pending' AND expires_at IS NOT NULL AND expires_at <= ?`) + .run(now.toISOString(), now.toISOString()).changes; + } + + private require(id: string): AutomationApprovalRecord { + const record = this.get(id); + if (!record) throw new EntityNotFoundError(`Automation approval not found: ${id}`); + return record; + } + + private map(row: ApprovalRow): AutomationApprovalRecord { + return { + id: row.id, projectId: row.project_id, flowId: row.flow_id, runId: row.run_id, + nodeId: row.node_id, logicalItem: row.logical_item, status: row.status, + request: JSON.parse(row.request_json) as NodeFlowJsonObject, + decision: row.decision_json ? JSON.parse(row.decision_json) as NodeFlowJsonObject : null, + requestedAt: row.requested_at, decidedAt: row.decided_at, decidedBy: row.decided_by, + expiresAt: row.expires_at, createdAt: row.created_at, updatedAt: row.updated_at, + }; + } +} diff --git a/src/repositories/automation-outbox-repository.ts b/src/repositories/automation-outbox-repository.ts new file mode 100644 index 0000000000..686b69e3ff --- /dev/null +++ b/src/repositories/automation-outbox-repository.ts @@ -0,0 +1,83 @@ +import { createHash, randomUUID } from "node:crypto"; +import { AppDbStorage } from "./app-db-storage.js"; +import type { DatabaseAdapter } from "./db/database-adapter.js"; +import type { NodeFlowJsonObject } from "../contracts/node-flow-types.js"; + +export type AutomationOutboxStatus = "pending" | "sending" | "sent" | "failed" | "attention_required"; +export interface AutomationOutboxRecord { + id: string; idempotencyKey: string; projectId: string; flowId: string; publicationId: string; + runId: string; nodeId: string; logicalItem: string; effectType: string; status: AutomationOutboxStatus; + payload: NodeFlowJsonObject; providerMessageId: string | null; attemptCount: number; + lastError: string | null; createdAt: string; updatedAt: string; sentAt: string | null; +} +interface OutboxRow { + id: string; idempotency_key: string; project_id: string; flow_id: string; publication_id: string; + run_id: string; node_id: string; logical_item: string; effect_type: string; status: AutomationOutboxStatus; + payload_json: string; provider_message_id: string | null; attempt_count: number; + last_error: string | null; created_at: string; updated_at: string; sent_at: string | null; +} + +export function deriveOutboxIdempotencyKey(input: { publicationId: string; runId: string; nodeId: string; logicalItem: string }): string { + return createHash("sha256").update([input.publicationId, input.runId, input.nodeId, input.logicalItem].join("\0")).digest("hex"); +} + +export class AutomationOutboxRepository { + private readonly db: DatabaseAdapter; + constructor(storage: AppDbStorage = new AppDbStorage()) { this.db = storage.getDatabase(); } + + enqueue(input: Omit): AutomationOutboxRecord { + const idempotencyKey = deriveOutboxIdempotencyKey(input); + const existing = this.getByKey(idempotencyKey); + if (existing) return existing; + const id = randomUUID(); const now = new Date().toISOString(); + this.db.prepare(`INSERT INTO automation_outbox + (id, idempotency_key, project_id, flow_id, publication_id, run_id, node_id, logical_item, + effect_type, status, payload_json, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`) + .run(id, idempotencyKey, input.projectId, input.flowId, input.publicationId, input.runId, + input.nodeId, input.logicalItem, input.effectType, JSON.stringify(input.payload), now, now); + return this.getByKey(idempotencyKey)!; + } + + get(id: string): AutomationOutboxRecord | null { + const row = this.db.prepare("SELECT * FROM automation_outbox WHERE id = ?").get(id) as OutboxRow | undefined; + return row ? this.map(row) : null; + } + getByKey(key: string): AutomationOutboxRecord | null { + const row = this.db.prepare("SELECT * FROM automation_outbox WHERE idempotency_key = ?").get(key) as OutboxRow | undefined; + return row ? this.map(row) : null; + } + listForRun(runId: string): AutomationOutboxRecord[] { + return (this.db.prepare("SELECT * FROM automation_outbox WHERE run_id = ? ORDER BY created_at").all(runId) as OutboxRow[]).map((row) => this.map(row)); + } + claim(id: string): AutomationOutboxRecord | null { + const now = new Date().toISOString(); + const result = this.db.prepare(`UPDATE automation_outbox SET status = 'sending', attempt_count = attempt_count + 1, + updated_at = ? WHERE id = ? AND status IN ('pending', 'failed')`).run(now, id); + return result.changes > 0 ? this.get(id) : null; + } + markSent(id: string, providerMessageId: string): AutomationOutboxRecord { + const now = new Date().toISOString(); + this.db.prepare(`UPDATE automation_outbox SET status = 'sent', provider_message_id = ?, last_error = NULL, + sent_at = ?, updated_at = ? WHERE id = ?`).run(providerMessageId, now, now, id); + return this.get(id)!; + } + markFailed(id: string, error: string, unknownOutcome = false): AutomationOutboxRecord { + this.db.prepare("UPDATE automation_outbox SET status = ?, last_error = ?, updated_at = ? WHERE id = ?") + .run(unknownOutcome ? "attention_required" : "failed", error, new Date().toISOString(), id); + return this.get(id)!; + } + recoverSending(): number { + return this.db.prepare(`UPDATE automation_outbox SET status = 'attention_required', + last_error = COALESCE(last_error, 'Process restarted while provider outcome was unknown.'), updated_at = ? + WHERE status = 'sending'`).run(new Date().toISOString()).changes; + } + private map(row: OutboxRow): AutomationOutboxRecord { + return { id: row.id, idempotencyKey: row.idempotency_key, projectId: row.project_id, + flowId: row.flow_id, publicationId: row.publication_id, runId: row.run_id, nodeId: row.node_id, + logicalItem: row.logical_item, effectType: row.effect_type, status: row.status, + payload: JSON.parse(row.payload_json) as NodeFlowJsonObject, providerMessageId: row.provider_message_id, + attemptCount: Number(row.attempt_count), lastError: row.last_error, createdAt: row.created_at, + updatedAt: row.updated_at, sentAt: row.sent_at }; + } +} diff --git a/src/repositories/automation-webhook-trigger-repository.ts b/src/repositories/automation-webhook-trigger-repository.ts new file mode 100644 index 0000000000..4ecc801405 --- /dev/null +++ b/src/repositories/automation-webhook-trigger-repository.ts @@ -0,0 +1,51 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import { AppDbStorage } from "./app-db-storage.js"; +import type { DatabaseAdapter } from "./db/database-adapter.js"; +import { EntityNotFoundError, ValidationError } from "./repository-utils.js"; + +export interface AutomationWebhookTriggerRecord { + id: string; projectId: string; flowId: string; enabled: boolean; + createdAt: string; updatedAt: string; lastTriggeredAt: string | null; +} +interface WebhookRow { id: string; project_id: string; flow_id: string; path_token_hash: string; secret_hash: string; enabled: number; created_at: string; updated_at: string; last_triggered_at: string | null } +const digest = (value: string): string => createHash("sha256").update(value).digest("hex"); + +export class AutomationWebhookTriggerRepository { + private readonly db: DatabaseAdapter; + constructor(storage: AppDbStorage = new AppDbStorage()) { this.db = storage.getDatabase(); } + create(projectId: string, flowId: string): { trigger: AutomationWebhookTriggerRecord; pathToken: string; secret: string } { + const flow = this.db.prepare("SELECT project_id FROM node_flows WHERE id = ?").get(flowId) as { project_id: string } | undefined; + if (!flow) throw new EntityNotFoundError(`Node flow not found: ${flowId}`); + if (flow.project_id !== projectId) throw new ValidationError("Node flow does not belong to the requested project."); + const pathToken = randomBytes(24).toString("base64url"); const secret = randomBytes(32).toString("base64url"); + const id = randomUUID(); const now = new Date().toISOString(); + this.db.prepare(`INSERT INTO automation_webhook_triggers + (id, project_id, flow_id, path_token_hash, secret_hash, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 1, ?, ?) + ON CONFLICT(project_id, flow_id) DO UPDATE SET path_token_hash=excluded.path_token_hash, + secret_hash=excluded.secret_hash, enabled=1, updated_at=excluded.updated_at`) + .run(id, projectId, flowId, digest(pathToken), digest(secret), now, now); + return { trigger: this.getByFlow(flowId)!, pathToken, secret }; + } + getByFlow(flowId: string): AutomationWebhookTriggerRecord | null { + const row = this.db.prepare("SELECT * FROM automation_webhook_triggers WHERE flow_id = ?").get(flowId) as WebhookRow | undefined; + return row ? this.map(row) : null; + } + authenticate(pathToken: string, secret: string): AutomationWebhookTriggerRecord | null { + const row = this.db.prepare("SELECT * FROM automation_webhook_triggers WHERE path_token_hash = ? AND enabled = 1") + .get(digest(pathToken)) as WebhookRow | undefined; + if (!row) return null; + const expected = Buffer.from(row.secret_hash, "hex"); const actual = Buffer.from(digest(secret), "hex"); + if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) return null; + this.db.prepare("UPDATE automation_webhook_triggers SET last_triggered_at = ?, updated_at = ? WHERE id = ?") + .run(new Date().toISOString(), new Date().toISOString(), row.id); + return this.map(row); + } + setEnabled(flowId: string, enabled: boolean): AutomationWebhookTriggerRecord { + this.db.prepare("UPDATE automation_webhook_triggers SET enabled = ?, updated_at = ? WHERE flow_id = ?") + .run(enabled ? 1 : 0, new Date().toISOString(), flowId); + const record = this.getByFlow(flowId); if (!record) throw new EntityNotFoundError(`Webhook trigger not found for flow: ${flowId}`); + return record; + } + private map(row: WebhookRow): AutomationWebhookTriggerRecord { return { id: row.id, projectId: row.project_id, flowId: row.flow_id, enabled: Boolean(row.enabled), createdAt: row.created_at, updatedAt: row.updated_at, lastTriggeredAt: row.last_triggered_at }; } +} diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index d54064de0e..3206e63557 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -319,6 +319,76 @@ export function ensureNodeFlowTables(db: DatabaseAdapter): void { ensureIndex(db, "idx_node_flow_attempts_run_node", "node_flow_node_attempts", "run_id, node_id, attempt_number"); } +export function ensureAutomationGovernanceTables(db: DatabaseAdapter): void { + db.exec(` + CREATE TABLE IF NOT EXISTS automation_approvals ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + flow_id TEXT NOT NULL, + run_id TEXT NOT NULL, + node_id TEXT NOT NULL, + logical_item TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + request_json TEXT NOT NULL DEFAULT '{}', + decision_json TEXT, + requested_at TEXT NOT NULL, + decided_at TEXT, + decided_by TEXT, + expires_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + FOREIGN KEY (flow_id) REFERENCES node_flows(id) ON DELETE CASCADE, + FOREIGN KEY (run_id) REFERENCES node_flow_runs(id) ON DELETE CASCADE, + UNIQUE (run_id, node_id, logical_item) + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_outbox ( + id TEXT PRIMARY KEY, + idempotency_key TEXT NOT NULL UNIQUE, + project_id TEXT NOT NULL, + flow_id TEXT NOT NULL, + publication_id TEXT NOT NULL, + run_id TEXT NOT NULL, + node_id TEXT NOT NULL, + logical_item TEXT NOT NULL, + effect_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + payload_json TEXT NOT NULL, + provider_message_id TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + sent_at TEXT, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + FOREIGN KEY (flow_id) REFERENCES node_flows(id) ON DELETE CASCADE, + FOREIGN KEY (run_id) REFERENCES node_flow_runs(id) ON DELETE CASCADE + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_webhook_triggers ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + flow_id TEXT NOT NULL, + path_token_hash TEXT NOT NULL UNIQUE, + secret_hash TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + last_triggered_at TEXT, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + FOREIGN KEY (flow_id) REFERENCES node_flows(id) ON DELETE CASCADE, + UNIQUE (project_id, flow_id) + ) + `); + ensureIndex(db, "idx_automation_approvals_run_status", "automation_approvals", "run_id, status, created_at"); + ensureIndex(db, "idx_automation_outbox_status", "automation_outbox", "status, updated_at"); + ensureIndex(db, "idx_automation_outbox_run", "automation_outbox", "run_id, node_id"); + ensureIndex(db, "idx_automation_webhooks_flow", "automation_webhook_triggers", "flow_id, enabled"); +} + interface LegacyNodeFlowRow { id: string; project_id: string; @@ -759,6 +829,7 @@ export function runMigrations(db: DatabaseAdapter): void { ensureTaskSelfReflectionRatingTables(db); ensureConversationDraftTables(db); ensureNodeFlowTables(db); + ensureAutomationGovernanceTables(db); migratePersistedNodeFlowGraphs(db); ensureCustomDashboardTables(db); ensureAutomationCredentialTables(db); diff --git a/src/server/dashboard-route-registration.ts b/src/server/dashboard-route-registration.ts index 196a19bd7f..aea5368746 100644 --- a/src/server/dashboard-route-registration.ts +++ b/src/server/dashboard-route-registration.ts @@ -34,6 +34,7 @@ import { registerChatProviderRoutes } from "./chat-provider-routes.js"; import { registerChatProviderIngressRoutes } from "./chat-provider-ingress-routes.js"; import { registerSpeechRoutes } from "./speech-routes.js"; import { registerNodeFlowRoutes } from "./node-flow-routes.js"; +import { registerNodeFlowWebhookRoutes } from "./node-flow-webhook-routes.js"; import { registerCustomDashboardRoutes } from "./custom-dashboard-routes.js"; export interface DashboardRouteRegistrationOptions { @@ -121,6 +122,7 @@ const registerProjectConfigurationRouteGroup = (app: Express, deps: DashboardDep registerAgentPresetRoutes(app, deps); registerInstructionFileRoutes(app, deps); registerNodeFlowRoutes(app, deps); + registerNodeFlowWebhookRoutes(app, deps); registerCustomDashboardRoutes(app, deps); }; diff --git a/src/server/dashboard-server.ts b/src/server/dashboard-server.ts index 2be9edfc1c..720b41f26b 100644 --- a/src/server/dashboard-server.ts +++ b/src/server/dashboard-server.ts @@ -142,6 +142,8 @@ import type { CustomDashboardRepository } from "../repositories/custom-dashboard import type { CustomDashboardValidationService } from "../services/custom-dashboard-validation-service.js"; import type { SkillService } from "../services/skill-service.js"; import type { CredentialBroker } from "../services/credentials/credential-broker.js"; +import type { ApprovalService } from "../services/node-flows/approval-service.js"; +import type { AutomationWebhookTriggerRepository } from "../repositories/automation-webhook-trigger-repository.js"; import type { ManagedRuntimeService } from "../services/managed-runtime-service.js"; import type { ProviderToolManager } from "../services/provider-tool-manager.js"; import { @@ -187,6 +189,8 @@ export interface DashboardServerOptions { speechSynthesisService?: SpeechSynthesisService; speechModelManager?: SpeechModelManager; nodeFlowService?: NodeFlowService; + approvalService?: ApprovalService; + automationWebhookTriggerRepository?: AutomationWebhookTriggerRepository; customDashboardRepository?: CustomDashboardRepository; customDashboardValidationService?: CustomDashboardValidationService; skillService?: SkillService; diff --git a/src/server/node-flow-routes.ts b/src/server/node-flow-routes.ts index a608686dd4..4032685d0b 100644 --- a/src/server/node-flow-routes.ts +++ b/src/server/node-flow-routes.ts @@ -128,4 +128,33 @@ export function registerNodeFlowRoutes(app: Express, deps: DashboardDependencies app.get("/api/node-flow-runs/:runId/attempts", syncRoute((req, res) => { res.json(requireNodeFlowService(deps).listNodeAttempts(requireTrimmedString(req.params.runId, "runId"))); })); + + app.get("/api/node-flow-runs/:runId/approvals", syncRoute((req, res) => { + if (!deps.approvalService) throw new HttpRouteError(404, "Approval service is not enabled."); + res.json({ approvals: deps.approvalService.listForRun(requireTrimmedString(req.params.runId, "runId")) }); + })); + + app.post("/api/automation-approvals/:approvalId/decision", syncRoute((req, res) => { + if (!deps.approvalService) throw new HttpRouteError(404, "Approval service is not enabled."); + const body = req.body as { decision?: string; decidedBy?: string; metadata?: NodeFlowJsonObject }; + const approvalId = requireTrimmedString(req.params.approvalId, "approvalId"); + const decidedBy = requireTrimmedString(body.decidedBy, "decidedBy"); + if (body.decision === "approve") res.json(deps.approvalService.approve(approvalId, decidedBy, body.metadata)); + else if (body.decision === "reject") res.json(deps.approvalService.reject(approvalId, decidedBy, body.metadata)); + else throw new HttpRouteError(400, "decision must be approve or reject."); + })); + + app.get("/api/node-flows/:flowId/webhook", syncRoute((req, res) => { + if (!deps.automationWebhookTriggerRepository) throw new HttpRouteError(404, "Webhook triggers are not enabled."); + res.json(deps.automationWebhookTriggerRepository.getByFlow(requireTrimmedString(req.params.flowId, "flowId"))); + })); + + app.post("/api/node-flows/:flowId/webhook", syncRoute((req, res) => { + if (!deps.automationWebhookTriggerRepository) throw new HttpRouteError(404, "Webhook triggers are not enabled."); + const flowId = requireTrimmedString(req.params.flowId, "flowId"); + const flow = requireNodeFlowService(deps).get(flowId); + if (!flow) throw new HttpRouteError(404, `Node flow not found: ${flowId}`); + const configured = deps.automationWebhookTriggerRepository.create(flow.projectId, flow.id); + res.status(201).json({ ...configured.trigger, pathToken: configured.pathToken, secret: configured.secret }); + })); } diff --git a/src/server/node-flow-webhook-routes.ts b/src/server/node-flow-webhook-routes.ts new file mode 100644 index 0000000000..7795ed7b9d --- /dev/null +++ b/src/server/node-flow-webhook-routes.ts @@ -0,0 +1,26 @@ +import type { Express } from "express"; +import type { DashboardDependencies } from "./dashboard-server.js"; +import { asyncRoute } from "./route-utils.js"; +import { HttpRouteError } from "./http-errors.js"; +import { requireTrimmedString } from "./request-parsers.js"; +import type { NodeFlowJsonObject } from "../contracts/node-flow-types.js"; + +export function registerNodeFlowWebhookRoutes(app: Express, deps: DashboardDependencies): void { + if (!deps.automationWebhookTriggerRepository || !deps.nodeFlowService) return; + app.post("/api/webhooks/node-flows/:pathToken", asyncRoute(async (req, res) => { + const secretHeader = req.headers["x-codeux-webhook-secret"]; + const secret = Array.isArray(secretHeader) ? secretHeader[0] : secretHeader; + if (typeof secret !== "string" || !secret.trim()) throw new HttpRouteError(401, "Webhook authentication failed."); + const trigger = deps.automationWebhookTriggerRepository!.authenticate( + requireTrimmedString(req.params.pathToken, "pathToken"), + secret.trim(), + ); + if (!trigger) throw new HttpRouteError(401, "Webhook authentication failed."); + const payload = req.body && typeof req.body === "object" && !Array.isArray(req.body) + ? req.body as NodeFlowJsonObject : {}; + const result = await deps.nodeFlowService!.runFlow(trigger.projectId, trigger.flowId, payload, { + triggerType: "webhook", triggerPayload: payload, versionSelection: { mode: "latest_published" }, + }); + res.status(202).json({ runId: result.run.id, status: result.run.status }); + })); +} diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index 57a8f95138..a3f662e047 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -15,6 +15,11 @@ import { NodeFlowPublicationService } from "./node-flows/node-flow-publication-s import { NodeFlowQueueService } from "./node-flows/node-flow-queue-service.js"; import { NodeFlowAttemptService } from "./node-flows/node-flow-attempt-service.js"; import { NodeFlowLeaseService } from "./node-flows/node-flow-lease-service.js"; +import { EgressPolicyService } from "./node-flows/egress-policy-service.js"; +import { BuiltinExecutors } from "./node-flows/builtins/builtin-executors.js"; +import type { ApprovalService } from "./node-flows/approval-service.js"; +import { ApprovalRequiredError } from "./node-flows/approval-service.js"; +import type { OutboxService } from "./node-flows/outbox-service.js"; import type { NodeFlowFailureClassification } from "../contracts/node-flow-execution-policy-types.js"; import { buildProviderInvocationWorkspaceOptions } from "../infrastructure/providers/cli/invocation-workspace-preparer.js"; import type { @@ -47,6 +52,9 @@ interface NodeFlowRuntimeDeps { providerExecutionService?: ProviderExecutionService; getDashboardSettings?: (projectId: string) => DashboardSettings; credentialBroker?: CredentialBroker; + egressPolicyService?: EgressPolicyService; + approvalService?: ApprovalService; + outboxService?: OutboxService; } interface RuntimeContext { @@ -57,20 +65,40 @@ interface RuntimeContext { order: string[]; input: NodeFlowJsonObject; outputs: Map; + selectedPorts: Map>; predecessors: Map; descendants: Map>; options: RunNodeFlowOptions; executorId: string; currentAttemptId?: string; + publicationId: string; + subflowDepth: number; } interface NodeExecutionResult { output: NodeFlowJsonObject; invocationId?: string | null; + selectedPorts?: string[]; } export class NodeFlowRuntimeService { - constructor(private readonly deps: NodeFlowRuntimeDeps) {} + private readonly egressPolicyService: EgressPolicyService; + private readonly builtins: BuiltinExecutors; + + constructor(private readonly deps: NodeFlowRuntimeDeps) { + this.egressPolicyService = deps.egressPolicyService ?? new EgressPolicyService(); + this.builtins = new BuiltinExecutors({ + approvalService: deps.approvalService, + outboxService: deps.outboxService, + executeSubflow: async ({ projectId, flowId, input, depth, signal }) => { + const summary = await this.runFlow(projectId, flowId, input, { signal, subflowDepth: depth, triggerType: "subflow" }); + if (summary.run.status !== "succeeded") { + throw new Error(summary.run.errorMessage ?? `Subflow ${flowId} ended with status ${summary.run.status}.`); + } + return summary.output ?? {}; + }, + }); + } async runFlow( projectId: string, @@ -138,10 +166,13 @@ export class NodeFlowRuntimeService { order: executionOrder, input, outputs: new Map(), + selectedPorts: new Map(), predecessors: buildPredecessors(graph), descendants: buildDescendants(graph), options, executorId, + publicationId: publication.id, + subflowDepth: options.subflowDepth ?? 0, }; const blockedNodes = new Set(); @@ -170,6 +201,10 @@ export class NodeFlowRuntimeService { await this.persistSkippedNode(context, node, "skipped", "Skipped because an upstream node failed."); continue; } + if (this.isInactiveBranch(context, node.id)) { + await this.persistSkippedNode(context, node, "skipped", "Skipped because its incoming branch was not selected."); + continue; + } if (node.disabled) { await this.persistSkippedNode(context, node, "skipped", "Skipped because the node is disabled."); continue; @@ -206,6 +241,7 @@ export class NodeFlowRuntimeService { try { const result = await this.executeNode(context, node, nodeRun); context.outputs.set(node.id, result.output); + if (result.selectedPorts) context.selectedPorts.set(node.id, new Set(result.selectedPorts)); attemptService.succeed(attempt, maskSecrets(result.output), result.invocationId); this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { status: "succeeded", @@ -219,6 +255,15 @@ export class NodeFlowRuntimeService { clearTimeout(timeout); options.signal?.removeEventListener("abort", parentAbort); context.options = previousOptions; const message = error instanceof Error ? error.message : String(error); const classification = classifyFailure(error, options.signal?.aborted === true, timeoutController.signal.aborted); + if (error instanceof ApprovalRequiredError) { + attemptService.fail(attempt, "permanent", message, false); + this.deps.nodeFlowRepository.updateNodeRun(nodeRun.id, { + status: "approval_waiting", errorMessage: message, finishedAt: null, + }); + terminalStatus = "approval_waiting"; + terminalError = message; + break; + } const wasCancelled = classification === "cancelled"; const retryable = retryPolicy.retryableClasses.includes(classification) && attemptNumber < retryPolicy.maxAttempts; attemptService.fail(attempt, classification, message, retryable, this.deps.nodeFlowRepository.listNodeAttempts(run.id).find((candidate) => candidate.id === attempt.id)?.invocationId); @@ -261,13 +306,13 @@ export class NodeFlowRuntimeService { break; } } - if (terminalStatus === "cancelled" || terminalStatus === "attention_required") { + if (terminalStatus === "cancelled" || terminalStatus === "attention_required" || terminalStatus === "approval_waiting") { break; } } const output = this.buildFlowOutput(context); - const finishedAt = new Date().toISOString(); + const finishedAt = terminalStatus === "approval_waiting" ? null : new Date().toISOString(); const updatedRun = this.deps.nodeFlowRepository.updateRun(run.id, { status: terminalStatus, output: maskSecrets(output), @@ -277,8 +322,8 @@ export class NodeFlowRuntimeService { leaseExpiresAt: null, }); this.deps.executionRepository.updateExecutionInvocation(parentInvocation.id, { - status: terminalStatus === "succeeded" ? "completed" : terminalStatus === "attention_required" ? "failed" : terminalStatus, - errorMessage: terminalError, + status: terminalStatus === "succeeded" ? "completed" : terminalStatus === "attention_required" ? "failed" : terminalStatus === "approval_waiting" ? "running" : terminalStatus, + errorMessage: terminalStatus === "approval_waiting" ? null : terminalError, finishedAt, }); this.deps.executionRepository.appendExecutionInvocationMessage(parentInvocation.id, { @@ -364,7 +409,13 @@ export class NodeFlowRuntimeService { case "output": return { output: this.executeOutputNode(context, node) }; default: - throw new ValidationError(`Unsupported node flow node type: ${node.type}.`); + return this.builtins.execute(node.type, { + projectId: context.projectId, flowId: context.flowId, publicationId: context.publicationId, + runId: context.runId, nodeId: node.id, + config: evaluateTemplates(readNodeConfig(node), context) as NodeFlowJsonObject, + upstream: this.buildUpstreamObject(context, node.id), flowInput: context.input, + signal: context.options.signal, subflowDepth: context.subflowDepth, + }); } } @@ -518,7 +569,7 @@ export class NodeFlowRuntimeService { } const headers = normalizeHeaders(readJsonObject(config.headers)); const boundCredential = await this.resolveNodeCredential(context, node, "auth"); - if (boundCredential) headers.Authorization = boundCredential; + const credentialHeaders = boundCredential ? { authorization: boundCredential } : undefined; const timeoutMs = normalizeTimeout(config.timeout ?? config.timeoutMs); const controller = new AbortController(); const abortListener = (): void => controller.abort(context.options.signal?.reason); @@ -530,18 +581,31 @@ export class NodeFlowRuntimeService { metadata: { flowId: context.flowId, runId: context.runId, nodeId: node.id }, }); try { - const response = await fetch(url, { + const response = await this.egressPolicyService.request({ + url, method, headers, + credentialHeaders, body: buildHttpBody(method, headers, config.body), signal: controller.signal, + rateLimitKey: `${context.projectId}:${url.hostname}`, + policy: { + allowHttp: config.allowHttp === true, + allowedHosts: readStringArray(config.allowedHosts), + allowedPorts: readNumberArray(config.allowedPorts), + maxRedirects: readOptionalNumber(config.maxRedirects), + maxResponseBytes: readOptionalNumber(config.maxResponseBytes), + allowedContentTypes: readStringArray(config.allowedContentTypes), + timeoutMs, + maxRetries: readOptionalNumber(config.maxRetries), + requestsPerMinute: readOptionalNumber(config.requestsPerMinute), + }, }); - const contentType = response.headers.get("content-type") ?? ""; - const body = contentType.includes("application/json") - ? await response.json() as NodeFlowJsonValue - : await response.text(); + const body = response.contentType.includes("application/json") + ? response.json() as NodeFlowJsonValue + : response.text(); if (!response.ok) { - throw new Error(`HTTP node ${node.id} failed with ${response.status} ${response.statusText}.`); + throw new Error(`HTTP node ${node.id} failed with status ${response.status}.`); } const responsePath = readString(config.responsePath) ?? readString(config.extractJsonPath); const extracted = responsePath ? readPath(body, responsePath) : body; @@ -597,6 +661,21 @@ export class NodeFlowRuntimeService { }; } + private buildUpstreamObject(context: RuntimeContext, nodeId: string): NodeFlowJsonObject { + return Object.fromEntries((context.predecessors.get(nodeId) ?? []) + .filter((id) => context.outputs.has(id)).map((id) => [id, context.outputs.get(id) ?? {}])); + } + + private isInactiveBranch(context: RuntimeContext, nodeId: string): boolean { + const incoming = context.graph.edges.filter((edge) => edge.toNodeId === nodeId); + if (incoming.length === 0) return false; + return !incoming.some((edge) => { + if (!context.outputs.has(edge.fromNodeId)) return false; + const selected = context.selectedPorts.get(edge.fromNodeId); + return !selected || !edge.fromHandle || selected.has(edge.fromHandle); + }); + } + private firstUpstreamObject(context: RuntimeContext, nodeId: string): NodeFlowJsonObject { const predecessorIds = context.predecessors.get(nodeId) ?? []; if (predecessorIds.length === 1) { @@ -762,6 +841,22 @@ function readBoolean(value: unknown): boolean { return value === true; } +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + return value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0).map((entry) => entry.trim()); +} + +function readNumberArray(value: unknown): number[] | undefined { + if (!Array.isArray(value)) return undefined; + const values = value.map(Number).filter((entry) => Number.isInteger(entry) && entry > 0 && entry <= 65_535); + return values.length > 0 ? values : undefined; +} + +function readOptionalNumber(value: unknown): number | undefined { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + function toJsonValue(value: unknown): NodeFlowJsonValue { if (value === undefined) { return null; diff --git a/src/services/node-flows/approval-service.ts b/src/services/node-flows/approval-service.ts new file mode 100644 index 0000000000..4391688f35 --- /dev/null +++ b/src/services/node-flows/approval-service.ts @@ -0,0 +1,37 @@ +import { ValidationError } from "../../repositories/repository-utils.js"; +import { AutomationApprovalRepository, type AutomationApprovalRecord } from "../../repositories/automation-approval-repository.js"; +import type { NodeFlowJsonObject } from "../../contracts/node-flow-types.js"; + +export class ApprovalRequiredError extends Error { + constructor(public readonly approval: AutomationApprovalRecord) { + super(`Approval is required: ${approval.id}`); + this.name = "ApprovalRequiredError"; + } +} + +export class ApprovalService { + constructor(private readonly repository: AutomationApprovalRepository) {} + + requireApproval(input: { + projectId: string; flowId: string; runId: string; nodeId: string; + logicalItem?: string; request: NodeFlowJsonObject; expiresAt?: string | null; + }): AutomationApprovalRecord { + const logicalItem = input.logicalItem?.trim() || "default"; + this.repository.expireDue(); + const approval = this.repository.getForItem(input.runId, input.nodeId, logicalItem) + ?? this.repository.request({ ...input, logicalItem }); + if (approval.status === "approved") return approval; + if (approval.status === "rejected" || approval.status === "expired") { + throw new ValidationError(`Approval ${approval.id} is ${approval.status}.`); + } + throw new ApprovalRequiredError(approval); + } + + approve(id: string, decidedBy: string, decision: NodeFlowJsonObject = {}): AutomationApprovalRecord { + return this.repository.decide(id, { status: "approved", decidedBy, decision }); + } + reject(id: string, decidedBy: string, decision: NodeFlowJsonObject = {}): AutomationApprovalRecord { + return this.repository.decide(id, { status: "rejected", decidedBy, decision }); + } + listForRun(runId: string): AutomationApprovalRecord[] { return this.repository.listForRun(runId); } +} diff --git a/src/services/node-flows/builtins/builtin-executors.ts b/src/services/node-flows/builtins/builtin-executors.ts new file mode 100644 index 0000000000..5a5530a138 --- /dev/null +++ b/src/services/node-flows/builtins/builtin-executors.ts @@ -0,0 +1,142 @@ +import { ValidationError } from "../../../repositories/repository-utils.js"; +import type { NodeFlowJsonObject, NodeFlowJsonValue } from "../../../contracts/node-flow-types.js"; +import type { ApprovalService } from "../approval-service.js"; +import type { OutboxService } from "../outbox-service.js"; + +export const MAX_FOREACH_ITEMS = 1_000; +export const MAX_SUBFLOW_DEPTH = 8; +export const MAX_DELAY_MS = 60 * 60_000; + +export interface BuiltinExecutionContext { + projectId: string; flowId: string; publicationId: string; runId: string; nodeId: string; + config: NodeFlowJsonObject; upstream: NodeFlowJsonObject; flowInput: NodeFlowJsonObject; + signal?: AbortSignal; subflowDepth: number; +} +export interface BuiltinExecutionResult { output: NodeFlowJsonObject; selectedPorts?: string[] } +export interface BuiltinExecutorDependencies { + approvalService?: ApprovalService; + outboxService?: OutboxService; + executeSubflow?: (input: { projectId: string; flowId: string; input: NodeFlowJsonObject; depth: number; signal?: AbortSignal }) => Promise; +} + +export class BuiltinExecutors { + constructor(private readonly deps: BuiltinExecutorDependencies = {}) {} + + async execute(type: string, context: BuiltinExecutionContext): Promise { + switch (type) { + case "condition": return executeCondition(context); + case "switch": return executeSwitch(context); + case "foreach": return executeForeach(context); + case "merge": return executeMerge(context); + case "delay": return executeDelay(context); + case "approval": return this.executeApproval(context); + case "email_draft": return executeEmailDraft(context); + case "email_send": return this.executeEmailSend(context); + case "execute_subflow": return this.executeSubflow(context); + case "webhook_trigger": return { output: { ...context.flowInput } }; + default: throw new ValidationError(`Unsupported built-in node type: ${type}.`); + } + } + + private executeApproval(context: BuiltinExecutionContext): BuiltinExecutionResult { + if (!this.deps.approvalService) throw new ValidationError("Approval service is not configured."); + const approval = this.deps.approvalService.requireApproval({ projectId: context.projectId, flowId: context.flowId, + runId: context.runId, nodeId: context.nodeId, logicalItem: readString(context.config.logicalItem) ?? "default", + request: { summary: readString(context.config.summary) ?? "Approval required", payload: context.upstream } }); + return { output: { approved: true, approvalId: approval.id, ...context.upstream }, selectedPorts: ["approved"] }; + } + + private async executeEmailSend(context: BuiltinExecutionContext): Promise { + if (!this.deps.approvalService || !this.deps.outboxService) throw new ValidationError("Email side-effect services are not configured."); + const draft = buildEmail(context); + const logicalItem = readString(context.config.logicalItem) ?? "default"; + this.deps.approvalService.requireApproval({ projectId: context.projectId, flowId: context.flowId, runId: context.runId, + nodeId: context.nodeId, logicalItem, request: { effectType: "email", draft } }); + const sent = await this.deps.outboxService.dispatch({ projectId: context.projectId, flowId: context.flowId, + publicationId: context.publicationId, runId: context.runId, nodeId: context.nodeId, logicalItem, + effectType: "email", payload: draft }); + if (sent.status === "attention_required") throw new Error(sent.lastError ?? "Email provider outcome is unknown."); + if (sent.status !== "sent") throw new Error(sent.lastError ?? "Email send failed."); + return { output: { sent: true, outboxId: sent.id, providerMessageId: sent.providerMessageId } }; + } + + private async executeSubflow(context: BuiltinExecutionContext): Promise { + const flowId = readString(context.config.flowId); + if (!flowId) throw new ValidationError("Execute Subflow requires flowId."); + if (flowId === context.flowId) throw new ValidationError("A node flow cannot directly execute itself."); + if (context.subflowDepth >= MAX_SUBFLOW_DEPTH) throw new ValidationError(`Subflow depth exceeds ${MAX_SUBFLOW_DEPTH}.`); + if (!this.deps.executeSubflow) throw new ValidationError("Subflow execution is not configured."); + return { output: await this.deps.executeSubflow({ projectId: context.projectId, flowId, + input: readObject(context.config.input) ?? context.upstream, depth: context.subflowDepth + 1, signal: context.signal }) }; + } +} + +function executeCondition(context: BuiltinExecutionContext): BuiltinExecutionResult { + const actual = readPath({ input: context.flowInput, upstream: context.upstream }, readString(context.config.path) ?? "upstream"); + const expected = context.config.value; + const operator = readString(context.config.operator) ?? "truthy"; + const matched = compare(actual, expected, operator); + return { output: { matched, value: toJson(actual) }, selectedPorts: [matched ? "true" : "false"] }; +} + +function executeSwitch(context: BuiltinExecutionContext): BuiltinExecutionResult { + const actual = readPath({ input: context.flowInput, upstream: context.upstream }, readString(context.config.path) ?? "upstream"); + const cases = Array.isArray(context.config.cases) ? context.config.cases : []; + if (cases.length > 100) throw new ValidationError("Switch cases are limited to 100."); + const selected = cases.find((entry) => { + const item = readObject(entry); return item && compare(actual, item.value, readString(item.operator) ?? "equals"); + }); + const selectedCase = selected ? readString(readObject(selected)?.port) ?? readString(readObject(selected)?.id) ?? "default" : "default"; + return { output: { value: toJson(actual), selectedCase }, selectedPorts: [selectedCase] }; +} + +function executeForeach(context: BuiltinExecutionContext): BuiltinExecutionResult { + const path = readString(context.config.path) ?? "upstream.items"; + const value = readPath({ input: context.flowInput, upstream: context.upstream }, path); + if (!Array.isArray(value)) throw new ValidationError("Foreach input must resolve to an array."); + const configured = Number(context.config.maxItems ?? MAX_FOREACH_ITEMS); + const maxItems = Math.max(1, Math.min(MAX_FOREACH_ITEMS, Number.isFinite(configured) ? Math.floor(configured) : MAX_FOREACH_ITEMS)); + if (value.length > maxItems) throw new ValidationError(`Foreach input exceeds the bounded item limit of ${maxItems}.`); + return { output: { items: value.map(toJson), count: value.length }, selectedPorts: value.length ? ["items"] : ["empty"] }; +} + +function executeMerge(context: BuiltinExecutionContext): BuiltinExecutionResult { + const strategy = readString(context.config.strategy) ?? "object"; + const values = Object.values(context.upstream); + if (strategy === "array") return { output: { items: values } }; + if (strategy === "first") return { output: readObject(values[0]) ?? { value: toJson(values[0]) } }; + if (strategy === "object") return { output: Object.assign({}, ...values.filter((value) => readObject(value))) as NodeFlowJsonObject }; + throw new ValidationError(`Unsupported merge strategy: ${strategy}.`); +} + +async function executeDelay(context: BuiltinExecutionContext): Promise { + const parsed = Number(context.config.delayMs ?? 0); + if (!Number.isFinite(parsed) || parsed < 0 || parsed > MAX_DELAY_MS) throw new ValidationError(`Delay must be between 0 and ${MAX_DELAY_MS}ms.`); + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, Math.floor(parsed)); + context.signal?.addEventListener("abort", () => { clearTimeout(timer); reject(context.signal?.reason ?? new Error("Delay cancelled.")); }, { once: true }); + }); + return { output: { ...context.upstream, delayedMs: Math.floor(parsed) } }; +} + +function executeEmailDraft(context: BuiltinExecutionContext): BuiltinExecutionResult { return { output: { draft: buildEmail(context), sent: false } }; } +function buildEmail(context: BuiltinExecutionContext): NodeFlowJsonObject { + const to = context.config.to; const subject = readString(context.config.subject); const body = readString(context.config.body); + if ((!readString(to) && !Array.isArray(to)) || !subject || !body) throw new ValidationError("Email requires to, subject, and body."); + return { to: to as NodeFlowJsonValue, subject, body, ...(readString(context.config.from) ? { from: readString(context.config.from)! } : {}) }; +} +function compare(actual: unknown, expected: unknown, operator: string): boolean { + switch (operator) { + case "truthy": return Boolean(actual); case "falsy": return !actual; + case "equals": return JSON.stringify(actual) === JSON.stringify(expected); + case "not_equals": return JSON.stringify(actual) !== JSON.stringify(expected); + case "contains": return typeof actual === "string" ? actual.includes(String(expected)) : Array.isArray(actual) && actual.some((entry) => JSON.stringify(entry) === JSON.stringify(expected)); + case "greater_than": return typeof actual === "number" && typeof expected === "number" && actual > expected; + case "less_than": return typeof actual === "number" && typeof expected === "number" && actual < expected; + default: throw new ValidationError(`Unsupported condition operator: ${operator}.`); + } +} +function readPath(value: unknown, path: string): unknown { return path.split(".").reduce((current, key) => current && typeof current === "object" ? (current as Record)[key] : undefined, value); } +function readString(value: unknown): string | null { return typeof value === "string" && value.trim() ? value.trim() : null; } +function readObject(value: unknown): NodeFlowJsonObject | null { return value && typeof value === "object" && !Array.isArray(value) ? value as NodeFlowJsonObject : null; } +function toJson(value: unknown): NodeFlowJsonValue { return value === undefined ? null : JSON.parse(JSON.stringify(value)) as NodeFlowJsonValue; } diff --git a/src/services/node-flows/builtins/index.ts b/src/services/node-flows/builtins/index.ts new file mode 100644 index 0000000000..76f84ed29b --- /dev/null +++ b/src/services/node-flows/builtins/index.ts @@ -0,0 +1 @@ +export * from "./builtin-executors.js"; diff --git a/src/services/node-flows/egress-policy-service.ts b/src/services/node-flows/egress-policy-service.ts new file mode 100644 index 0000000000..750d6b8ade --- /dev/null +++ b/src/services/node-flows/egress-policy-service.ts @@ -0,0 +1,224 @@ +import { lookup as dnsLookup } from "node:dns/promises"; +import { isIP } from "node:net"; +import { ValidationError } from "../../repositories/repository-utils.js"; + +const RESTRICTED_HEADERS = new Set([ + "authorization", "cookie", "host", "proxy-authorization", "proxy-connection", + "connection", "transfer-encoding", "upgrade", "x-forwarded-for", "x-real-ip", +]); +const DEFAULT_CONTENT_TYPES = ["application/json", "text/"]; +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); + +export interface EgressPolicy { + allowHttp?: boolean; + allowedHosts?: string[]; + allowedPorts?: number[]; + maxRedirects?: number; + maxResponseBytes?: number; + allowedContentTypes?: string[]; + timeoutMs?: number; + maxRetries?: number; + requestsPerMinute?: number; +} + +export interface EgressRequest { + url: string | URL; + method?: string; + headers?: Record; + credentialHeaders?: Record; + body?: BodyInit; + signal?: AbortSignal; + policy?: EgressPolicy; + rateLimitKey?: string; +} + +export interface EgressResponse { + url: string; + status: number; + ok: boolean; + headers: Record; + contentType: string; + body: Uint8Array; + text(): string; + json(): unknown; +} + +export interface EgressPolicyServiceOptions { + fetch?: typeof fetch; + lookup?: (hostname: string) => Promise>; + defaults?: EgressPolicy; +} + +export class EgressPolicyService { + private readonly fetchImpl: typeof fetch; + private readonly lookup: NonNullable; + private readonly defaults: EgressPolicy; + private readonly rateWindows = new Map(); + + constructor(options: EgressPolicyServiceOptions = {}) { + this.fetchImpl = options.fetch ?? fetch; + this.lookup = options.lookup ?? (async (hostname) => dnsLookup(hostname, { all: true, verbatim: true })); + this.defaults = options.defaults ?? {}; + } + + async request(input: EgressRequest): Promise { + const policy = { ...this.defaults, ...input.policy }; + const method = (input.method ?? "GET").toUpperCase(); + const headers = normalizeRequestHeaders(input.headers, false); + Object.assign(headers, normalizeRequestHeaders(input.credentialHeaders, true)); + this.enforceRateLimit(input.rateLimitKey ?? new URL(input.url).hostname, policy.requestsPerMinute ?? 60); + + const retries = Math.max(0, Math.min(5, Math.floor(policy.maxRetries ?? 0))); + let lastError: unknown; + for (let attempt = 0; attempt <= retries; attempt += 1) { + try { + const response = await this.requestWithRedirects(input.url, method, headers, input.body, input.signal, policy); + if (!RETRYABLE_STATUSES.has(response.status) || attempt === retries || !isRetrySafe(method, headers)) return response; + } catch (error) { + lastError = error; + if (attempt === retries || !isRetrySafe(method, headers) || error instanceof ValidationError) throw error; + } + await boundedBackoff(attempt, input.signal); + } + throw lastError instanceof Error ? lastError : new Error("Egress request failed."); + } + + async validateUrl(value: string | URL, policy: EgressPolicy = {}): Promise<{ url: URL; addresses: string[] }> { + let url: URL; + try { url = new URL(value); } catch { throw new ValidationError("Egress URL is invalid."); } + if (url.username || url.password) throw new ValidationError("Credentials in URLs are not allowed."); + if (url.protocol !== "https:" && !(url.protocol === "http:" && policy.allowHttp === true)) { + throw new ValidationError("Egress URL must use HTTPS unless HTTP is explicitly enabled."); + } + const hostname = url.hostname.toLowerCase().replace(/\.$/, "").replace(/^\[|\]$/g, ""); + if (!hostname || hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") { + throw new ValidationError("Private, loopback, and metadata hosts are not allowed."); + } + const allowedHosts = policy.allowedHosts?.map((entry) => entry.toLowerCase()); + if (allowedHosts?.length && !allowedHosts.some((entry) => hostname === entry || (entry.startsWith("*.") && hostname.endsWith(entry.slice(1))))) { + throw new ValidationError(`Egress host is not allowlisted: ${hostname}.`); + } + const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80; + const allowedPorts = policy.allowedPorts ?? (url.protocol === "https:" ? [443] : [80]); + if (!Number.isInteger(port) || !allowedPorts.includes(port)) throw new ValidationError(`Egress port is not allowlisted: ${port}.`); + + const addresses = isIP(hostname) + ? [hostname] + : (await this.lookup(hostname)).map((entry) => entry.address); + if (addresses.length === 0) throw new ValidationError("Egress host did not resolve to an address."); + if (addresses.some(isBlockedAddress)) throw new ValidationError("Private, loopback, link-local, and metadata addresses are not allowed."); + const rebound = isIP(hostname) ? addresses : (await this.lookup(hostname)).map((entry) => entry.address); + if (rebound.length === 0 || !sameAddressSet(addresses, rebound) || rebound.some(isBlockedAddress)) { + throw new ValidationError("DNS rebinding was detected for the egress host."); + } + return { url, addresses: [...new Set(addresses)].sort() }; + } + + private async requestWithRedirects( + initialUrl: string | URL, method: string, headers: Record, body: BodyInit | undefined, + parentSignal: AbortSignal | undefined, policy: EgressPolicy, + ): Promise { + let current = (await this.validateUrl(initialUrl, policy)).url; + const maxRedirects = Math.max(0, Math.min(10, Math.floor(policy.maxRedirects ?? 3))); + for (let redirects = 0; ; redirects += 1) { + const timeoutMs = Math.max(1, Math.min(120_000, Math.floor(policy.timeoutMs ?? 30_000))); + const controller = new AbortController(); + const abort = (): void => controller.abort(parentSignal?.reason); + parentSignal?.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(() => controller.abort(new Error(`Egress request timed out after ${timeoutMs}ms.`)), timeoutMs); + let response: Response; + try { + response = await this.fetchImpl(current, { method, headers, body, redirect: "manual", signal: controller.signal }); + if (REDIRECT_STATUSES.has(response.status)) { + if (redirects >= maxRedirects) throw new ValidationError("Egress redirect limit exceeded."); + const location = response.headers.get("location"); + if (!location) throw new ValidationError("Egress redirect response omitted Location."); + const next = new URL(location, current); + if (next.origin !== current.origin) { + delete headers.authorization; + delete headers.cookie; + } + current = (await this.validateUrl(next, policy)).url; + continue; + } + return await this.readResponse(response, current, policy); + } finally { + clearTimeout(timer); parentSignal?.removeEventListener("abort", abort); + } + } + } + + private async readResponse(response: Response, url: URL, policy: EgressPolicy): Promise { + const maxBytes = Math.max(1, Math.min(50 * 1024 * 1024, Math.floor(policy.maxResponseBytes ?? 2 * 1024 * 1024))); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > maxBytes) throw new ValidationError("Egress response exceeds the configured size limit."); + const contentType = (response.headers.get("content-type") ?? "application/octet-stream").split(";", 1)[0]!.trim().toLowerCase(); + const allowed = policy.allowedContentTypes ?? DEFAULT_CONTENT_TYPES; + if (response.status !== 204 && !allowed.some((entry) => contentType === entry || (entry.endsWith("/") && contentType.startsWith(entry)))) { + throw new ValidationError(`Egress response content type is not allowed: ${contentType}.`); + } + const chunks: Uint8Array[] = []; let total = 0; + if (response.body) { + const reader = response.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { await reader.cancel(); throw new ValidationError("Egress response exceeds the configured size limit."); } + chunks.push(value); + } + } + const bytes = new Uint8Array(total); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + return { + url: url.toString(), status: response.status, ok: response.ok, + headers: Object.fromEntries([...response.headers].map(([key, value]) => [key.toLowerCase(), value])), + contentType, body: bytes, + text: () => new TextDecoder().decode(bytes), + json: () => JSON.parse(new TextDecoder().decode(bytes)) as unknown, + }; + } + + private enforceRateLimit(key: string, limit: number): void { + const boundedLimit = Math.max(1, Math.min(10_000, Math.floor(limit))); + const cutoff = Date.now() - 60_000; + const recent = (this.rateWindows.get(key) ?? []).filter((time) => time > cutoff); + if (recent.length >= boundedLimit) throw new ValidationError("Egress rate limit exceeded."); + recent.push(Date.now()); this.rateWindows.set(key, recent); + } +} + +function normalizeRequestHeaders(input: Record | undefined, trusted: boolean): Record { + const result: Record = {}; + for (const [rawName, rawValue] of Object.entries(input ?? {})) { + const name = rawName.trim().toLowerCase(); + if (!/^[!#$%&'*+.^_`|~0-9a-z-]+$/.test(name) || /[\r\n]/.test(rawValue)) throw new ValidationError("Egress header is invalid."); + if (!trusted && RESTRICTED_HEADERS.has(name)) throw new ValidationError(`Egress header is restricted: ${name}.`); + if (name === "host" || name === "connection" || name.startsWith("proxy-")) throw new ValidationError(`Egress header is restricted: ${name}.`); + result[name] = String(rawValue).trim(); + } + return result; +} + +function isBlockedAddress(address: string): boolean { + const normalized = address.toLowerCase().split("%", 1)[0]!; + if (normalized === "::1" || normalized === "::" || normalized.startsWith("fe80:") || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("ff") || normalized.startsWith("2001:db8:")) return true; + const mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1]; + if (mapped) return isBlockedAddress(mapped); + if (isIP(normalized) === 4) { + const [a, b] = normalized.split(".").map(Number); + return a === 0 || a === 10 || a === 127 || a >= 224 || (a === 169 && b === 254) + || (a === 172 && b! >= 16 && b! <= 31) || (a === 192 && b === 168) + || (a === 100 && b! >= 64 && b! <= 127) || (a === 192 && b === 0) + || (a === 192 && b === 2) || (a === 198 && (b === 18 || b === 19 || b === 51)) + || (a === 203 && b === 0); + } + return isIP(normalized) !== 6; +} +const sameAddressSet = (left: string[], right: string[]): boolean => [...new Set(left)].sort().join("|") === [...new Set(right)].sort().join("|"); +const isRetrySafe = (method: string, headers: Record): boolean => ["GET", "HEAD", "OPTIONS"].includes(method) || Boolean(headers["idempotency-key"]); +async function boundedBackoff(attempt: number, signal?: AbortSignal): Promise { + const ms = Math.min(2_000, 100 * (2 ** attempt)); + await new Promise((resolve, reject) => { const timer = setTimeout(resolve, ms); signal?.addEventListener("abort", () => { clearTimeout(timer); reject(signal.reason); }, { once: true }); }); +} diff --git a/src/services/node-flows/oauth-broker.ts b/src/services/node-flows/oauth-broker.ts new file mode 100644 index 0000000000..18a3cd8085 --- /dev/null +++ b/src/services/node-flows/oauth-broker.ts @@ -0,0 +1,107 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; +import { ValidationError } from "../../repositories/repository-utils.js"; + +export interface OAuthTokenSet { + accessToken: string; refreshToken?: string; expiresAt: string; scopes: string[]; tokenType?: string; +} +export interface OAuthTokenClient { + exchangeCode(input: { code: string; redirectUri: string; codeVerifier: string }): Promise; + refresh(refreshToken: string): Promise; + revoke?(token: string): Promise; + health?(accessToken: string): Promise; +} +export interface OAuthConnectionStore { + get(connectionId: string): Promise | OAuthTokenSet | null; + set(connectionId: string, tokens: OAuthTokenSet): Promise | void; + delete(connectionId: string): Promise | void; +} +interface OAuthStatePayload { connectionId: string; origin: string; redirectUri: string; verifier: string; expiresAt: number; nonce: string } + +export class OAuthBroker { + private readonly consumedStates = new Set(); + constructor(private readonly input: { key: Buffer; store: OAuthConnectionStore; client: OAuthTokenClient; allowedCallbackOrigins: string[] }) { + if (input.key.byteLength !== 32) throw new ValidationError("OAuth state encryption key must contain 32 bytes."); + } + + beginAuthorization(input: { connectionId: string; authorizationUrl: string; redirectUri: string; scopes: string[] }): { authorizationUrl: string; state: string } { + const redirect = new URL(input.redirectUri); this.assertCallbackOrigin(redirect.origin); + const verifier = randomBytes(32).toString("base64url"); + const payload: OAuthStatePayload = { connectionId: input.connectionId, origin: redirect.origin, redirectUri: redirect.toString(), verifier, expiresAt: Date.now() + 10 * 60_000, nonce: randomBytes(16).toString("hex") }; + const state = this.encrypt(payload); + const url = new URL(input.authorizationUrl); + url.searchParams.set("response_type", "code"); url.searchParams.set("redirect_uri", redirect.toString()); + url.searchParams.set("scope", [...new Set(input.scopes)].sort().join(" ")); url.searchParams.set("state", state); + url.searchParams.set("code_challenge_method", "S256"); + url.searchParams.set("code_challenge", createHash("sha256").update(verifier).digest("base64url")); + return { authorizationUrl: url.toString(), state }; + } + + async completeAuthorization(input: { state: string; code: string; callbackOrigin: string }): Promise<{ connectionId: string; expiresAt: string; scopes: string[] }> { + const state = this.decrypt(input.state); + const stateDigest = createHash("sha256").update(input.state).digest("hex"); + if (this.consumedStates.has(stateDigest)) throw new ValidationError("OAuth authorization state has already been used."); + this.assertCallbackOrigin(input.callbackOrigin); + if (state.origin !== new URL(input.callbackOrigin).origin) throw new ValidationError("OAuth callback origin does not match authorization state."); + if (state.expiresAt <= Date.now()) throw new ValidationError("OAuth authorization state has expired."); + this.consumedStates.add(stateDigest); + const tokens = normalizeTokens(await this.input.client.exchangeCode({ code: input.code, redirectUri: state.redirectUri, codeVerifier: state.verifier })); + await this.input.store.set(state.connectionId, tokens); + return { connectionId: state.connectionId, expiresAt: tokens.expiresAt, scopes: tokens.scopes }; + } + + async getAccessToken(connectionId: string, requiredScopes: string[] = []): Promise { + let tokens = await this.input.store.get(connectionId); + if (!tokens) throw new ValidationError("OAuth connection must be reconnected."); + if (requiredScopes.some((scope) => !tokens!.scopes.includes(scope))) throw new ValidationError("OAuth connection does not grant the required scopes."); + if (Date.parse(tokens.expiresAt) <= Date.now() + 30_000) { + if (!tokens.refreshToken) throw new ValidationError("OAuth connection has expired and must be reconnected."); + const refreshed = normalizeTokens(await this.input.client.refresh(tokens.refreshToken)); + tokens = { ...refreshed, refreshToken: refreshed.refreshToken ?? tokens.refreshToken }; + await this.input.store.set(connectionId, tokens); + } + return tokens.accessToken; + } + + async revoke(connectionId: string): Promise { + const tokens = await this.input.store.get(connectionId); + if (tokens && this.input.client.revoke) await this.input.client.revoke(tokens.refreshToken ?? tokens.accessToken); + await this.input.store.delete(connectionId); + } + async reconnect(connectionId: string): Promise { await this.revoke(connectionId); } + async health(connectionId: string): Promise<{ healthy: boolean; expiresAt: string | null; scopes: string[] }> { + let tokens = await this.input.store.get(connectionId); + if (!tokens) return { healthy: false, expiresAt: null, scopes: [] }; + try { const accessToken = await this.getAccessToken(connectionId); tokens = await this.input.store.get(connectionId) ?? tokens; return { healthy: this.input.client.health ? await this.input.client.health(accessToken) : true, expiresAt: tokens.expiresAt, scopes: tokens.scopes }; } + catch { return { healthy: false, expiresAt: tokens.expiresAt, scopes: tokens.scopes }; } + } + + private assertCallbackOrigin(origin: string): void { + let normalized: string; try { normalized = new URL(origin).origin; } catch { throw new ValidationError("OAuth callback origin is invalid."); } + if (!this.input.allowedCallbackOrigins.some((allowed) => new URL(allowed).origin === normalized)) throw new ValidationError("OAuth callback origin is not allowlisted."); + } + private encrypt(payload: OAuthStatePayload): string { + const iv = randomBytes(12); const cipher = createCipheriv("aes-256-gcm", this.input.key, iv); + const ciphertext = Buffer.concat([cipher.update(JSON.stringify(payload), "utf8"), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString("base64url"); + } + private decrypt(value: string): OAuthStatePayload { + try { + const packed = Buffer.from(value, "base64url"); if (packed.length < 29) throw new Error("short state"); + const decipher = createDecipheriv("aes-256-gcm", this.input.key, packed.subarray(0, 12)); + decipher.setAuthTag(packed.subarray(12, 28)); + return JSON.parse(Buffer.concat([decipher.update(packed.subarray(28)), decipher.final()]).toString("utf8")) as OAuthStatePayload; + } catch { throw new ValidationError("OAuth authorization state is invalid."); } + } +} + +function normalizeTokens(tokens: OAuthTokenSet): OAuthTokenSet { + if (!tokens.accessToken || !Number.isFinite(Date.parse(tokens.expiresAt))) throw new ValidationError("OAuth provider returned an invalid token response."); + return { ...tokens, scopes: [...new Set(tokens.scopes)].sort(), tokenType: tokens.tokenType ?? "Bearer" }; +} + +export class InMemoryOAuthConnectionStore implements OAuthConnectionStore { + private readonly records = new Map(); + get(id: string): OAuthTokenSet | null { return this.records.get(id) ?? null; } + set(id: string, tokens: OAuthTokenSet): void { this.records.set(id, { ...tokens }); } + delete(id: string): void { this.records.delete(id); } +} diff --git a/src/services/node-flows/outbox-service.ts b/src/services/node-flows/outbox-service.ts new file mode 100644 index 0000000000..8c1bfef991 --- /dev/null +++ b/src/services/node-flows/outbox-service.ts @@ -0,0 +1,42 @@ +import { AutomationOutboxRepository, type AutomationOutboxRecord } from "../../repositories/automation-outbox-repository.js"; +import type { NodeFlowJsonObject } from "../../contracts/node-flow-types.js"; + +export interface SideEffectProviderResult { providerMessageId: string } +export interface SideEffectProvider { send(effectType: string, payload: NodeFlowJsonObject, idempotencyKey: string): Promise } + +export class OutboxService { + constructor(private readonly repository: AutomationOutboxRepository, private readonly provider: SideEffectProvider) { + this.repository.recoverSending(); + } + + async dispatch(input: { + projectId: string; flowId: string; publicationId: string; runId: string; nodeId: string; + logicalItem?: string; effectType: string; payload: NodeFlowJsonObject; + }): Promise { + const record = this.repository.enqueue({ ...input, logicalItem: input.logicalItem?.trim() || "default" }); + if (record.status === "sent" || record.status === "attention_required") return record; + const claimed = this.repository.claim(record.id); + if (!claimed) return this.repository.get(record.id)!; + try { + const result = await this.provider.send(claimed.effectType, claimed.payload, claimed.idempotencyKey); + return this.repository.markSent(claimed.id, result.providerMessageId); + } catch (error) { + const unknownOutcome = error instanceof UnknownSideEffectOutcomeError; + return this.repository.markFailed(claimed.id, error instanceof Error ? error.message : String(error), unknownOutcome); + } + } +} + +export class UnknownSideEffectOutcomeError extends Error { + constructor(message = "The provider may have accepted the side effect, so automatic replay is disabled.") { + super(message); this.name = "UnknownSideEffectOutcomeError"; + } +} + +export class MockSideEffectProvider implements SideEffectProvider { + readonly sends: Array<{ effectType: string; payload: NodeFlowJsonObject; idempotencyKey: string }> = []; + async send(effectType: string, payload: NodeFlowJsonObject, idempotencyKey: string): Promise { + this.sends.push({ effectType, payload, idempotencyKey }); + return { providerMessageId: `mock-${idempotencyKey.slice(0, 16)}` }; + } +} diff --git a/tests/backend/domain/node-flows/node-definition-registry.test.ts b/tests/backend/domain/node-flows/node-definition-registry.test.ts index fd4fdefe02..6c2d175aba 100644 --- a/tests/backend/domain/node-flows/node-definition-registry.test.ts +++ b/tests/backend/domain/node-flows/node-definition-registry.test.ts @@ -2,11 +2,16 @@ import { describe, expect, it } from "vitest"; import { listNodeDefinitions, resolveNodeDefinition } from "../../../../src/domain/node-flows/node-definition-registry.js"; describe("node definition registry", () => { - it("registers only the six implemented executable node types", () => { + it("registers the governed executable catalog", () => { expect(listNodeDefinitions().filter((definition) => definition.executable).map((definition) => definition.type)).toEqual([ - "input", "set_fields", "template", "provider_prompt", "http_request", "output", + "input", "set_fields", "template", "provider_prompt", "http_request", "condition", "switch", + "foreach", "merge", "delay", "approval", "email_draft", "email_send", "execute_subflow", + "webhook_trigger", "output", ]); - expect(resolveNodeDefinition("condition", 1)).toBeNull(); + expect(resolveNodeDefinition("condition", 1)).toMatchObject({ + executable: true, + ports: expect.arrayContaining([expect.objectContaining({ id: "true" }), expect.objectContaining({ id: "false" })]), + }); expect(resolveNodeDefinition("http_request", 1)).toMatchObject({ executionKind: "http", sideEffect: "external", diff --git a/tests/backend/repositories/automation-governance-repositories.test.ts b/tests/backend/repositories/automation-governance-repositories.test.ts new file mode 100644 index 0000000000..ae0a962d38 --- /dev/null +++ b/tests/backend/repositories/automation-governance-repositories.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; +import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; +import { NodeFlowRepository } from "../../../src/repositories/node-flow-repository.js"; +import { AutomationApprovalRepository } from "../../../src/repositories/automation-approval-repository.js"; +import { AutomationOutboxRepository } from "../../../src/repositories/automation-outbox-repository.js"; +import { MockSideEffectProvider, OutboxService } from "../../../src/services/node-flows/outbox-service.js"; + +const dirs: string[] = []; +afterEach(async () => Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })))); + +async function fixture() { + const dir = await mkdtemp(join(tmpdir(), "automation-governance-")); dirs.push(dir); + const dbPath = join(dir, "app.db"); const storage = new AppDbStorage(dbPath); + const project = new ProjectManagementRepository(storage).createProject({ name: "Governance", sourceType: "local", sourceRef: dir }); + const flows = new NodeFlowRepository(storage); + const flow = flows.createFlow(project.id, { title: "Flow", graph: { nodes: [{ id: "draft", type: "email_draft", title: "Draft" }], edges: [] } }); + const publication = flows.getPublication(flow.id)!; + const run = flows.createRun({ flowId: flow.id, projectId: project.id, version: flow.version, publicationId: publication.id, status: "running" }); + return { dir, dbPath, storage, project, flow, publication, run }; +} + +describe("automation governance repositories", () => { + it("persists approval decisions across repository restarts", async () => { + const { storage, project, flow, run } = await fixture(); + const first = new AutomationApprovalRepository(storage); + const requested = first.request({ projectId: project.id, flowId: flow.id, runId: run.id, nodeId: "approve", logicalItem: "one", request: { summary: "Send" } }); + first.decide(requested.id, { status: "approved", decidedBy: "operator" }); + expect(new AutomationApprovalRepository(storage).get(requested.id)).toMatchObject({ status: "approved", decidedBy: "operator" }); + storage.close(); + }); + + it("deduplicates outbox sends and marks restart-unknown outcomes for attention", async () => { + const { storage, project, flow, publication, run } = await fixture(); + const repository = new AutomationOutboxRepository(storage); const provider = new MockSideEffectProvider(); + const service = new OutboxService(repository, provider); + const input = { projectId: project.id, flowId: flow.id, publicationId: publication.id, runId: run.id, nodeId: "send", logicalItem: "one", effectType: "email", payload: { to: "a@example.test" } }; + const first = await service.dispatch(input); const second = await service.dispatch(input); + expect(second.id).toBe(first.id); expect(provider.sends).toHaveLength(1); expect(first.providerMessageId).toMatch(/^mock-/); + + const pending = repository.enqueue({ ...input, logicalItem: "two" }); + repository.claim(pending.id); + new OutboxService(repository, provider); + expect(repository.get(pending.id)).toMatchObject({ status: "attention_required" }); + storage.close(); + }); +}); diff --git a/tests/backend/server/node-flow-webhook-routes.test.ts b/tests/backend/server/node-flow-webhook-routes.test.ts new file mode 100644 index 0000000000..5062f4c026 --- /dev/null +++ b/tests/backend/server/node-flow-webhook-routes.test.ts @@ -0,0 +1,20 @@ +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import { registerNodeFlowWebhookRoutes } from "../../../src/server/node-flow-webhook-routes.js"; + +describe("node flow webhook ingress", () => { + it("rejects unauthenticated requests and dispatches authenticated payloads", async () => { + const authenticate = vi.fn((pathToken: string, secret: string) => pathToken === "path" && secret === "secret" + ? { projectId: "project-1", flowId: "flow-1" } : null); + const runFlow = vi.fn().mockResolvedValue({ run: { id: "run-1", status: "succeeded" } }); + const app = express(); app.use(express.json()); + registerNodeFlowWebhookRoutes(app, { automationWebhookTriggerRepository: { authenticate }, nodeFlowService: { runFlow } } as never); + + expect((await request(app).post("/api/webhooks/node-flows/path").send({ ok: true })).status).toBe(401); + const accepted = await request(app).post("/api/webhooks/node-flows/path") + .set("x-codeux-webhook-secret", "secret").send({ ok: true }); + expect(accepted.status).toBe(202); + expect(runFlow).toHaveBeenCalledWith("project-1", "flow-1", { ok: true }, expect.objectContaining({ triggerType: "webhook" })); + }); +}); diff --git a/tests/backend/services/node-flow-builtins.test.ts b/tests/backend/services/node-flow-builtins.test.ts new file mode 100644 index 0000000000..5134bcbb69 --- /dev/null +++ b/tests/backend/services/node-flow-builtins.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { BuiltinExecutors, MAX_FOREACH_ITEMS } from "../../../src/services/node-flows/builtins/builtin-executors.js"; + +const base = { projectId: "p", flowId: "f", publicationId: "pub", runId: "r", nodeId: "n", upstream: {}, flowInput: {}, subflowDepth: 0 }; + +describe("governed built-in executors", () => { + it("selects explicit condition and switch ports", async () => { + const executors = new BuiltinExecutors(); + await expect(executors.execute("condition", { ...base, flowInput: { enabled: true }, config: { path: "input.enabled" } })) + .resolves.toMatchObject({ selectedPorts: ["true"] }); + await expect(executors.execute("switch", { ...base, flowInput: { tier: "pro" }, config: { path: "input.tier", cases: [{ id: "paid", value: "pro" }] } })) + .resolves.toMatchObject({ selectedPorts: ["paid"] }); + }); + + it("bounds foreach and delay", async () => { + const executors = new BuiltinExecutors(); + await expect(executors.execute("foreach", { ...base, upstream: { items: Array(MAX_FOREACH_ITEMS + 1).fill(null) }, config: { path: "upstream.items" } })) + .rejects.toThrow(/bounded item limit/i); + await expect(executors.execute("delay", { ...base, config: { delayMs: 3_600_001 } })).rejects.toThrow(/Delay must be between/i); + }); + + it("supports deterministic merge strategies and subflow recursion guards", async () => { + const executors = new BuiltinExecutors({ executeSubflow: async () => ({ ok: true }) }); + await expect(executors.execute("merge", { ...base, upstream: { a: { one: 1 }, b: { two: 2 } }, config: { strategy: "object" } })) + .resolves.toMatchObject({ output: { one: 1, two: 2 } }); + await expect(executors.execute("execute_subflow", { ...base, config: { flowId: "f" } })).rejects.toThrow(/cannot directly execute itself/i); + }); +}); diff --git a/tests/backend/services/node-flow-egress-policy-service.test.ts b/tests/backend/services/node-flow-egress-policy-service.test.ts new file mode 100644 index 0000000000..4eee4f284a --- /dev/null +++ b/tests/backend/services/node-flow-egress-policy-service.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from "vitest"; +import { EgressPolicyService } from "../../../src/services/node-flows/egress-policy-service.js"; + +const publicLookup = async (): Promise> => [{ address: "8.8.8.8", family: 4 }]; + +describe("EgressPolicyService", () => { + it.each([ + "https://127.0.0.1/x", "https://10.0.0.1/x", "https://169.254.169.254/latest/meta-data", + "https://metadata.google.internal/x", "https://localhost/x", "https://[::1]/x", + ])("rejects private, loopback, and metadata target %s", async (url) => { + await expect(new EgressPolicyService().validateUrl(url)).rejects.toThrow(/private|loopback|metadata/i); + }); + + it("requires HTTPS, rejects URL credentials and raw credential headers", async () => { + const service = new EgressPolicyService({ lookup: publicLookup }); + await expect(service.validateUrl("http://api.example.test/x")).rejects.toThrow(/HTTPS/); + await expect(service.validateUrl("https://user:pass@api.example.test/x")).rejects.toThrow(/credentials/i); + await expect(service.request({ url: "https://api.example.test/x", headers: { Authorization: "Bearer raw" } })).rejects.toThrow(/restricted/i); + }); + + it("detects rebinding and revalidates redirects", async () => { + let lookups = 0; + const rebinding = new EgressPolicyService({ lookup: async () => [{ address: ++lookups === 1 ? "8.8.8.8" : "127.0.0.1", family: 4 }] }); + await expect(rebinding.validateUrl("https://api.example.test/x")).rejects.toThrow(/rebinding/i); + + const fetchMock = vi.fn().mockResolvedValueOnce(new Response(null, { status: 302, headers: { location: "http://127.0.0.1/secret" } })); + const redirects = new EgressPolicyService({ lookup: publicLookup, fetch: fetchMock }); + await expect(redirects.request({ url: "https://api.example.test/x" })).rejects.toThrow(/HTTPS|private/i); + }); + + it("bounds response size, retry count, and host allowlists", async () => { + const oversized = new EgressPolicyService({ lookup: publicLookup, fetch: vi.fn().mockResolvedValue(new Response("12345", { headers: { "content-type": "text/plain", "content-length": "5" } })) }); + await expect(oversized.request({ url: "https://api.example.test/x", policy: { maxResponseBytes: 4 } })).rejects.toThrow(/size limit/i); + await expect(oversized.validateUrl("https://other.example.test/x", { allowedHosts: ["api.example.test"] })).rejects.toThrow(/allowlisted/i); + + const fetchMock = vi.fn().mockImplementation(async () => new Response("retry", { status: 503, headers: { "content-type": "text/plain" } })); + const retries = new EgressPolicyService({ lookup: publicLookup, fetch: fetchMock }); + const response = await retries.request({ url: "https://api.example.test/x", policy: { maxRetries: 2 } }); + expect(response.status).toBe(503); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); +}); diff --git a/tests/backend/services/node-flow-oauth-broker.test.ts b/tests/backend/services/node-flow-oauth-broker.test.ts new file mode 100644 index 0000000000..35bf55c516 --- /dev/null +++ b/tests/backend/services/node-flow-oauth-broker.test.ts @@ -0,0 +1,32 @@ +import { randomBytes } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { InMemoryOAuthConnectionStore, OAuthBroker } from "../../../src/services/node-flows/oauth-broker.js"; + +describe("OAuthBroker", () => { + it("uses encrypted PKCE state, rotates refresh tokens, checks scopes, health, and revocation", async () => { + const store = new InMemoryOAuthConnectionStore(); + const client = { + exchangeCode: vi.fn().mockResolvedValue({ accessToken: "access-1", refreshToken: "refresh-1", expiresAt: new Date(Date.now() - 1).toISOString(), scopes: ["mail.send"] }), + refresh: vi.fn().mockResolvedValue({ accessToken: "access-2", refreshToken: "refresh-2", expiresAt: new Date(Date.now() + 60_000).toISOString(), scopes: ["mail.send"] }), + revoke: vi.fn().mockResolvedValue(undefined), health: vi.fn().mockResolvedValue(true), + }; + const broker = new OAuthBroker({ key: randomBytes(32), store, client, allowedCallbackOrigins: ["https://app.example.test"] }); + const started = broker.beginAuthorization({ connectionId: "mail", authorizationUrl: "https://oauth.example.test/authorize", redirectUri: "https://app.example.test/oauth/callback", scopes: ["mail.send"] }); + expect(started.state).not.toContain("mail"); + expect(new URL(started.authorizationUrl).searchParams.get("code_challenge_method")).toBe("S256"); + await broker.completeAuthorization({ state: started.state, code: "code", callbackOrigin: "https://app.example.test" }); + await expect(broker.getAccessToken("mail", ["mail.send"])).resolves.toBe("access-2"); + await expect(broker.getAccessToken("mail", ["mail.read"])).rejects.toThrow(/scopes/i); + await expect(broker.health("mail")).resolves.toMatchObject({ healthy: true, scopes: ["mail.send"] }); + await broker.revoke("mail"); + await expect(broker.health("mail")).resolves.toMatchObject({ healthy: false }); + expect(client.revoke).toHaveBeenCalled(); + }); + + it("rejects callback-origin mismatches and tampered state", async () => { + const broker = new OAuthBroker({ key: randomBytes(32), store: new InMemoryOAuthConnectionStore(), client: { exchangeCode: vi.fn(), refresh: vi.fn() }, allowedCallbackOrigins: ["https://app.example.test"] }); + const started = broker.beginAuthorization({ connectionId: "x", authorizationUrl: "https://oauth.example.test", redirectUri: "https://app.example.test/callback", scopes: [] }); + await expect(broker.completeAuthorization({ state: `${started.state}x`, code: "x", callbackOrigin: "https://app.example.test" })).rejects.toThrow(/state is invalid/i); + await expect(broker.completeAuthorization({ state: started.state, code: "x", callbackOrigin: "https://evil.example.test" })).rejects.toThrow(/not allowlisted/i); + }); +}); diff --git a/tests/backend/services/node-flow-runtime-service.test.ts b/tests/backend/services/node-flow-runtime-service.test.ts index 6a94053222..c08eeb0df2 100644 --- a/tests/backend/services/node-flow-runtime-service.test.ts +++ b/tests/backend/services/node-flow-runtime-service.test.ts @@ -1,6 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import * as fs from "fs/promises"; -import * as http from "http"; import * as os from "os"; import * as path from "path"; import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; @@ -13,10 +12,11 @@ import { NodeFlowRuntimeService } from "../../../src/services/node-flow-runtime- import type { ProviderExecutionService } from "../../../src/services/provider-execution-service.js"; import type { NodeFlowGraph } from "../../../src/contracts/node-flow-types.js"; import type { CredentialBroker } from "../../../src/services/credentials/credential-broker.js"; +import { EgressPolicyService } from "../../../src/services/node-flows/egress-policy-service.js"; const tempDirs: string[] = []; -async function createRuntime(providerExecutionService?: Partial,credentialBroker?:Partial): Promise<{ +async function createRuntime(providerExecutionService?: Partial,credentialBroker?:Partial, egressPolicyService?: EgressPolicyService): Promise<{ dir: string; projectRepository: ProjectManagementRepository; nodeFlowRepository: NodeFlowRepository; @@ -36,6 +36,7 @@ async function createRuntime(providerExecutionService?: Partial DEFAULT_DASHBOARD_SETTINGS, }); return { dir, projectRepository, nodeFlowRepository, executionRepository, runtime }; @@ -46,6 +47,30 @@ afterEach(async () => { }); describe("NodeFlowRuntimeService", () => { + it("persists unselected condition branches as skipped while the selected branch runs", async () => { + const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(); + const project = projectRepository.createProject({ name: "Branch Project", sourceType: "local", sourceRef: dir }); + const flow = nodeFlowRepository.createFlow(project.id, { title: "Branch", graph: { + nodes: [ + { id: "condition", type: "condition", title: "Condition", data: { path: "input.enabled" } }, + { id: "yes", type: "set_fields", title: "Yes", data: { fields: { branch: "yes" } } }, + { id: "no", type: "set_fields", title: "No", data: { fields: { branch: "no" } } }, + { id: "merge", type: "merge", title: "Merge", data: { strategy: "object" } }, + { id: "output", type: "output", title: "Output" }, + ], + edges: [ + { fromNodeId: "condition", fromHandle: "true", toNodeId: "yes" }, + { fromNodeId: "condition", fromHandle: "false", toNodeId: "no" }, + { fromNodeId: "yes", toNodeId: "merge" }, { fromNodeId: "no", toNodeId: "merge" }, + { fromNodeId: "merge", toNodeId: "output" }, + ], + } }); + const result = await runtime.runFlow(project.id, flow.id, { enabled: true }); + expect(result.run.status).toBe("succeeded"); + expect(result.nodeRuns.find((node) => node.nodeId === "no")?.status).toBe("skipped"); + expect(result.output).toMatchObject({ branch: "yes" }); + }); + it("executes an explicitly pinned publication while latest selection follows the newest publication", async () => { const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(); const project = projectRepository.createProject({ name: "Version Project", sourceType: "local", sourceRef: dir }); @@ -173,21 +198,12 @@ describe("NodeFlowRuntimeService", () => { expect(executionRepository.getExecutionInvocation(promptRun!.executionInvocationId!)?.type).toBe("node_flow_node"); }); - it("executes HTTP request nodes with query, body, timeout, and JSON response extraction", async () => { - const server = http.createServer((req, res) => { - if (req.url?.startsWith("/ok?name=Ada")) { - res.setHeader("content-type", "application/json"); - res.end(JSON.stringify({ data: { message: "hello" } })); - return; - } - res.statusCode = 404; - res.end("not found"); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - try { - const address = server.address(); - const port = typeof address === "object" && address ? address.port : 0; - const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(); + it("executes governed HTTP request nodes with query, body, timeout, and JSON response extraction", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ data: { message: "hello" } }), { + status: 200, headers: { "content-type": "application/json" }, + })); + const egressPolicyService = new EgressPolicyService({ fetch: fetchMock, lookup: async () => [{ address: "8.8.8.8", family: 4 }] }); + const { dir, projectRepository, nodeFlowRepository, runtime } = await createRuntime(undefined, undefined, egressPolicyService); const project = projectRepository.createProject({ name: "HTTP Project", sourceType: "local", sourceRef: dir }); const flow = nodeFlowRepository.createFlow(project.id, { title: "HTTP", @@ -199,7 +215,7 @@ describe("NodeFlowRuntimeService", () => { title: "HTTP", data: { method: "POST", - url: `http://127.0.0.1:${port}/ok`, + url: "https://api.example.test/ok", query: { name: "Ada" }, body: { ok: true }, timeout: 1000, @@ -216,9 +232,7 @@ describe("NodeFlowRuntimeService", () => { expect(result.run.status).toBe("succeeded"); expect(result.output).toMatchObject({ status: 200, extracted: "hello" }); expect(result.nodeRuns[0]?.executionInvocationId).toMatch(/^xi_/); - } finally { - await new Promise((resolve) => server.close(() => resolve())); - } + expect(fetchMock).toHaveBeenCalledWith(expect.objectContaining({ search: "?name=Ada" }), expect.objectContaining({ redirect: "manual" })); }); it("fails HTTP nodes clearly and skips downstream nodes without continueOnError", async () => { From 4b389d6f6627c322b2a44057a0abd50878268fe0 Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 05:49:21 +0000 Subject: [PATCH 05/25] feat(task T05): implement via codex --- .code-ux/nodes/README.md | 5 + docs-web/architecture/custom-nodes.md | 31 ++ docs-web/architecture/index.md | 1 + .../docs/architecture-custom-nodes.mdx | 31 ++ .../content/docs/architecture-overview.mdx | 1 + docs-web/content/docs/registry.ts | 9 + .../docs.architecture-custom-nodes.lazy.tsx | 11 + docs/SUMMARY.md | 1 + docs/architecture/custom-nodes.md | 63 ++++ docs/index.md | 2 + .../dependency-factory/dashboard-factory.ts | 17 +- src/contracts/custom-node-types.ts | 229 ++++++++++++ src/contracts/node-definition-types.ts | 2 +- .../node-flows/node-definition-registry.ts | 16 +- src/repositories/custom-node-repository.ts | 189 ++++++++++ src/repositories/db/app-db-migrations.ts | 50 +++ .../custom-nodes/custom-node-build-service.ts | 339 ++++++++++++++++++ .../custom-node-project-service.ts | 133 +++++++ .../custom-node-runtime-service.ts | 196 ++++++++++ src/services/node-flow-runtime-service.ts | 45 ++- .../custom-node-repository.test.ts | 50 +++ .../services/custom-node-services.test.ts | 150 ++++++++ 22 files changed, 1564 insertions(+), 7 deletions(-) create mode 100644 .code-ux/nodes/README.md create mode 100644 docs-web/architecture/custom-nodes.md create mode 100644 docs-web/content/docs/architecture-custom-nodes.mdx create mode 100644 docs-web/routes/docs.architecture-custom-nodes.lazy.tsx create mode 100644 docs/architecture/custom-nodes.md create mode 100644 src/contracts/custom-node-types.ts create mode 100644 src/repositories/custom-node-repository.ts create mode 100644 src/services/custom-nodes/custom-node-build-service.ts create mode 100644 src/services/custom-nodes/custom-node-project-service.ts create mode 100644 src/services/custom-nodes/custom-node-runtime-service.ts create mode 100644 tests/backend/repositories/custom-node-repository.test.ts create mode 100644 tests/backend/services/custom-node-services.test.ts diff --git a/.code-ux/nodes/README.md b/.code-ux/nodes/README.md new file mode 100644 index 0000000000..59a8616080 --- /dev/null +++ b/.code-ux/nodes/README.md @@ -0,0 +1,5 @@ +# Custom node packages + +Code UX generates each project-owned TypeScript node under `.code-ux/nodes//`. +Packages are drafts until container validation passes and an immutable artifact is explicitly published. +Generated source is never imported or executed by the Code UX server process. diff --git a/docs-web/architecture/custom-nodes.md b/docs-web/architecture/custom-nodes.md new file mode 100644 index 0000000000..904d7b70a2 --- /dev/null +++ b/docs-web/architecture/custom-nodes.md @@ -0,0 +1,31 @@ +# Custom Node Architecture and Security + +Custom nodes are project-owned TypeScript packages that pass explicit validation and publication gates before Code UX can execute them. Generated code is never imported or evaluated by the Code UX server. + +## Availability + +Custom execution is available only for an immutable published type/version and only while the `CODE_UX_CUSTOM_NODES_ENABLED=true` feature gate is enabled. Unpublished definitions are absent from the executable registry, and a disabled gate rejects a run before credential resolution or Docker startup. No dashboard or public management route bypasses these gates. + +## Generated package and validation + +Packages live at `.code-ux/nodes//` and include a manifest, exact package metadata, frozen pnpm lockfile, strict TypeScript configuration, typed local SDK, `src/index.ts`, an isolated runner, deterministic tests, fixtures, and a multi-stage Dockerfile. + +The SDK exposes immutable input/config, correlation and invocation ids, cancellation, a redacting logger, deterministic clock, bounded HTTP and credential slots, temporary storage, and artifacts. It does not expose the project path, host environment, raw filesystem, subprocesses, Docker, or raw networking. + +Validation fails closed on malformed schemas or identity, undeclared capabilities, excessive resources, symlinks, unpinned dependencies, lockfile drift, a modified trusted Docker recipe, prohibited APIs, vulnerability-audit failure, TypeScript or deterministic-test failure, fixture mismatch, output-schema failure, resource/network-policy failure, or secret-canary leakage. The exact recipe restores locked dependencies with lifecycle scripts disabled, then performs typecheck, build, and tests in a network-disabled stage. The vulnerability check is an injected governed hook; validation fails when it is not configured. + +A passed revision produces an immutable content-addressed envelope with source and build digests, immutable image id, dependency inventory, validation report, creator/invocation/correlation metadata, and declared capabilities. Publication registers only its typed definition and digest. Flow graphs never embed custom source. + +## Isolated execution + +Published images run as non-root with a read-only root, all capabilities dropped, `no-new-privileges`, optional seccomp/AppArmor profiles, bounded CPU/memory/PIDs/time/stdout, and a size-limited no-exec tmpfs. The runtime uses `--network none` and mounts neither the project nor the Docker socket nor persistent run state. + +Each invocation receives a fresh mode-`0600` stdin envelope. Credential values come only from the existing project-scoped `CredentialBroker`; they are never added to the environment, image, labels, cache keys, graph JSON, or persistent volume. The envelope and scratch state are deleted after the run. + +HTTP remains fail closed unless a transport backed by the existing `EgressPolicyService` is supplied. The default runner rejects HTTP rather than enabling bridge or host networking. Any future transport must preserve the existing host/port allowlists, DNS and redirect revalidation, response-size, timeout, retry, and rate bounds. + +The parent validates the output schema and recursively replaces resolved credential values in outputs, logs, traces, and diagnostics before persistence. Content-addressed image caching therefore retains immutable code only, not credentials or cross-run mutable state. + +## Persistence + +`custom_nodes` stores lifecycle state, `custom_node_artifacts` stores immutable artifact envelopes, and `custom_node_publications` binds a type/version to an artifact digest. The published versioned registry is the executable authority. diff --git a/docs-web/architecture/index.md b/docs-web/architecture/index.md index a82e90633e..4acab22faf 100644 --- a/docs-web/architecture/index.md +++ b/docs-web/architecture/index.md @@ -22,6 +22,7 @@ It is for contributors and integrators who need to reason about *how* Code UX ma | 12 | [Speech input](./speech-input.md) | Persisted transcription settings, privacy boundary, provider fallback contract | | 13 | [Security model](./security.md) | Authentication, authorisation, secrets, network surface | | 14 | [Worker clarification contract](./worker-clarification-contract.md) | Durable human-owned worker questions, idempotent replies, and continuation boundary | +| 15 | [Custom nodes](./custom-nodes.md) | Generated TypeScript packages, validation/publication gates, and hardened container execution | ## Runtime Notes diff --git a/docs-web/content/docs/architecture-custom-nodes.mdx b/docs-web/content/docs/architecture-custom-nodes.mdx new file mode 100644 index 0000000000..904d7b70a2 --- /dev/null +++ b/docs-web/content/docs/architecture-custom-nodes.mdx @@ -0,0 +1,31 @@ +# Custom Node Architecture and Security + +Custom nodes are project-owned TypeScript packages that pass explicit validation and publication gates before Code UX can execute them. Generated code is never imported or evaluated by the Code UX server. + +## Availability + +Custom execution is available only for an immutable published type/version and only while the `CODE_UX_CUSTOM_NODES_ENABLED=true` feature gate is enabled. Unpublished definitions are absent from the executable registry, and a disabled gate rejects a run before credential resolution or Docker startup. No dashboard or public management route bypasses these gates. + +## Generated package and validation + +Packages live at `.code-ux/nodes//` and include a manifest, exact package metadata, frozen pnpm lockfile, strict TypeScript configuration, typed local SDK, `src/index.ts`, an isolated runner, deterministic tests, fixtures, and a multi-stage Dockerfile. + +The SDK exposes immutable input/config, correlation and invocation ids, cancellation, a redacting logger, deterministic clock, bounded HTTP and credential slots, temporary storage, and artifacts. It does not expose the project path, host environment, raw filesystem, subprocesses, Docker, or raw networking. + +Validation fails closed on malformed schemas or identity, undeclared capabilities, excessive resources, symlinks, unpinned dependencies, lockfile drift, a modified trusted Docker recipe, prohibited APIs, vulnerability-audit failure, TypeScript or deterministic-test failure, fixture mismatch, output-schema failure, resource/network-policy failure, or secret-canary leakage. The exact recipe restores locked dependencies with lifecycle scripts disabled, then performs typecheck, build, and tests in a network-disabled stage. The vulnerability check is an injected governed hook; validation fails when it is not configured. + +A passed revision produces an immutable content-addressed envelope with source and build digests, immutable image id, dependency inventory, validation report, creator/invocation/correlation metadata, and declared capabilities. Publication registers only its typed definition and digest. Flow graphs never embed custom source. + +## Isolated execution + +Published images run as non-root with a read-only root, all capabilities dropped, `no-new-privileges`, optional seccomp/AppArmor profiles, bounded CPU/memory/PIDs/time/stdout, and a size-limited no-exec tmpfs. The runtime uses `--network none` and mounts neither the project nor the Docker socket nor persistent run state. + +Each invocation receives a fresh mode-`0600` stdin envelope. Credential values come only from the existing project-scoped `CredentialBroker`; they are never added to the environment, image, labels, cache keys, graph JSON, or persistent volume. The envelope and scratch state are deleted after the run. + +HTTP remains fail closed unless a transport backed by the existing `EgressPolicyService` is supplied. The default runner rejects HTTP rather than enabling bridge or host networking. Any future transport must preserve the existing host/port allowlists, DNS and redirect revalidation, response-size, timeout, retry, and rate bounds. + +The parent validates the output schema and recursively replaces resolved credential values in outputs, logs, traces, and diagnostics before persistence. Content-addressed image caching therefore retains immutable code only, not credentials or cross-run mutable state. + +## Persistence + +`custom_nodes` stores lifecycle state, `custom_node_artifacts` stores immutable artifact envelopes, and `custom_node_publications` binds a type/version to an artifact digest. The published versioned registry is the executable authority. diff --git a/docs-web/content/docs/architecture-overview.mdx b/docs-web/content/docs/architecture-overview.mdx index d4d9c67af9..821388f8af 100644 --- a/docs-web/content/docs/architecture-overview.mdx +++ b/docs-web/content/docs/architecture-overview.mdx @@ -22,6 +22,7 @@ It is for contributors and integrators who need to reason about *how* Code UX ma | 12 | [Speech input](/docs/architecture-speech-input) | Persisted transcription settings, privacy boundary, provider fallback contract | | 13 | [Security model](/docs/architecture-security) | Authentication, authorisation, secrets, network surface | | 14 | [Worker clarification contract](/docs/architecture-worker-clarification-contract) | Durable human-owned worker questions, idempotent replies, and continuation boundary | +| 15 | [Custom nodes](/docs/architecture-custom-nodes) | Generated TypeScript packages, validation/publication gates, and hardened container execution | ## Runtime Notes diff --git a/docs-web/content/docs/registry.ts b/docs-web/content/docs/registry.ts index 42d840a534..23b762da50 100644 --- a/docs-web/content/docs/registry.ts +++ b/docs-web/content/docs/registry.ts @@ -107,6 +107,7 @@ export type DocsSlug = | 'settings-google-drive-mount' | 'user-dashboard-custom-dashboards' | 'architecture-custom-dashboard-foundation' + | 'architecture-custom-nodes' | 'architecture-managed-container-runtime' | 'architecture-node-flow-builtins-and-security' | 'architecture-node-flow-durable-execution' @@ -853,6 +854,13 @@ export const docsRegistry: Record = { title: "Custom Dashboard Foundation", description: "Custom dashboards are a persisted domain model for project-scoped dashboard generation. The foundation stores manifests, generated file bundles, data-source node graphs, validation history, and publication state, and...", }, + 'architecture-custom-nodes': { + id: 'architecture-custom-nodes', + path: '/docs/architecture-custom-nodes', + section: 'Architecture', + title: "Custom Node Architecture and Security", + description: "Custom nodes are project-owned TypeScript packages that pass explicit validation and publication gates before Code UX can execute them. Generated code is never imported or evaluated by the Code UX server.", + }, 'architecture-managed-container-runtime': { id: 'architecture-managed-container-runtime', path: '/docs/architecture-managed-container-runtime', @@ -1016,6 +1024,7 @@ export const orderedDocs: DocsRegistryEntry[] = [ docsRegistry['settings-google-drive-mount'], docsRegistry['user-dashboard-custom-dashboards'], docsRegistry['architecture-custom-dashboard-foundation'], + docsRegistry['architecture-custom-nodes'], docsRegistry['architecture-managed-container-runtime'], docsRegistry['architecture-node-flow-builtins-and-security'], docsRegistry['architecture-node-flow-durable-execution'], diff --git a/docs-web/routes/docs.architecture-custom-nodes.lazy.tsx b/docs-web/routes/docs.architecture-custom-nodes.lazy.tsx new file mode 100644 index 0000000000..42fab00e9a --- /dev/null +++ b/docs-web/routes/docs.architecture-custom-nodes.lazy.tsx @@ -0,0 +1,11 @@ +import { createLazyFileRoute } from '@tanstack/react-router' +import ArchitectureCustomNodesContent from '../content/docs/architecture-custom-nodes.mdx' +import { DocsPage } from '../components/docs/DocsPage' + +export const Route = createLazyFileRoute('/docs/architecture-custom-nodes')({ + component: () => ( + + + + ) +}) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 820eada147..1d7a3c0e6f 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -41,6 +41,7 @@ - [Node Flows](./architecture/node-flows.md) - [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) - [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) +- [Custom Node Architecture and Security](./architecture/custom-nodes.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/docs/architecture/custom-nodes.md b/docs/architecture/custom-nodes.md new file mode 100644 index 0000000000..78a6ea816e --- /dev/null +++ b/docs/architecture/custom-nodes.md @@ -0,0 +1,63 @@ +# Custom Node Architecture and Security + +Custom nodes are project-owned TypeScript packages that move through an explicit draft, validation, publication, and container-execution boundary. Generated code is never imported, evaluated, or executed in the Code UX process. + +## Availability and gates + +Custom execution is available only when both gates pass: + +1. A source revision has completed validation, produced an immutable artifact, and been explicitly published. +2. `CODE_UX_CUSTOM_NODES_ENABLED=true` (or an equivalent explicitly enabled runtime dependency) is active. + +An unpublished type/version is absent from the executable node-definition registry. A disabled feature gate rejects execution before credentials are resolved or Docker starts. There is no dashboard or public management route that bypasses these service gates. + +## Package contract + +`CustomNodeProjectService` generates `.code-ux/nodes//` with `node.json`, exact package metadata, a frozen pnpm lockfile, strict TypeScript configuration, `src/index.ts`, a local typed SDK, an isolated stdio runner, deterministic tests, fixtures, and a multi-stage Dockerfile. + +The handler receives only `NodeExecutionContext`: immutable JSON input/config, correlation and invocation ids, an abort signal, a redacting logger, a deterministic clock, bounded HTTP and credential slots, tmpfs-backed temporary storage, and artifact writing. It does not receive a project path, process environment, host filesystem handle, subprocess API, Docker handle, or raw network client. + +## Validation and publication + +`CustomNodeBuildService` changes a draft to `validating` and runs these fail-closed checks: + +- manifest identity, value schemas, credential slots, capabilities, and resource limits +- package size, file count, symlink, exact dependency, and frozen-lockfile checks +- prohibited API scans for filesystem, environment, subprocess, Docker, raw network, worker, and alternate-runtime access +- least-privilege comparison between declared capabilities and SDK usage +- a required vulnerability-audit hook +- an exact trusted Docker recipe whose locked, script-disabled dependency restore is separate from network-disabled TypeScript, build, and deterministic-test stages +- an isolated fixture execution with CPU, memory, PID, time, output, scratch, and network bounds +- output-schema, deterministic expected-output, and secret-canary checks + +Any failed check records a `failed` report and no artifact. A passed build records dependency inventory, source revision, deterministic build digest, Docker image id, validation report, creator/invocation/correlation metadata, manifest, and declared capabilities. The artifact envelope is content-addressed and immutable. Publication stores only its digest and registers a typed `custom.*@version` definition; flow graph JSON contains the definition reference and configuration, never source. + +The audit hook is intentionally injected so custom nodes consume the governed dependency policy instead of creating a second vulnerability policy. With no hook, validation fails. + +## Runtime boundary + +`CustomNodeRuntimeService` resolves only a published artifact and launches its immutable image with: + +- `--network none`, never host networking +- a non-root uid, read-only root filesystem, all Linux capabilities dropped, and `no-new-privileges` +- optional configured seccomp and AppArmor profiles +- CPU, memory/swap, PID, timeout, stdout, and tmpfs size limits from the validated manifest +- no project mount, Docker socket, host environment, persistent volume, or cross-run writable state +- a fresh stdin credential/input envelope and a fresh tmpfs scratch directory per invocation +- Docker logging disabled; bounded stdout/stderr are captured by the parent + +Credential values are resolved through the existing `CredentialBroker` using the project, credential id, workspace, and required capability. They are never placed in an image layer, image label, cache key, graph, or process environment. The temporary stdin file is mode `0600`, removed after the invocation, and its in-memory values are cleared. + +The SDK's HTTP authority is fail closed unless a transport backed by the existing `EgressPolicyService` is supplied. The default network-none runner rejects HTTP calls; it never falls back to Docker bridge networking or a second allowlist. A future broker transport must retain the T02 host/port, DNS-rebinding, redirect, response-size, timeout, retry, and rate policies before enabling `network.http` publication. + +Outputs are schema-checked before persistence. Resolved credential canaries are recursively redacted from JSON output and replaced in logs and diagnostics. Run directories are unique and deleted after each invocation, so image caching retains only immutable code and cannot retain plaintext credentials or mutable run state. + +## Persistence + +SQLite separates mutable lifecycle state from immutable execution authority: + +- `custom_nodes` stores the current draft identity, lifecycle state, report, and selected artifact digest. +- `custom_node_artifacts` stores immutable content-addressed artifact envelopes. +- `custom_node_publications` binds one custom type/version to one artifact digest. + +This mirrors node-flow publication: the versioned registry remains the executable authority, while source stays in the project package directory. diff --git a/docs/index.md b/docs/index.md index d459e6830d..7c8f643b42 100644 --- a/docs/index.md +++ b/docs/index.md @@ -68,6 +68,7 @@ Use this page as the main entrypoint. 30. [Node Flows](./architecture/node-flows.md) 31. [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) 32. [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) +33. [Custom Node Architecture and Security](./architecture/custom-nodes.md) 31. [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) 32. [Memory Claims and Evidence](./architecture/memory-claims.md) 33. [Speech Input Architecture](./architecture/speech-input.md) @@ -162,6 +163,7 @@ Use this page as the main entrypoint. - [Node Flows](./architecture/node-flows.md) - [Node Flow Durable Execution](./architecture/node-flow-durable-execution.md) - [Node Flow Built-ins and External-Effect Security](./architecture/node-flow-builtins-and-security.md) +- [Custom Node Architecture and Security](./architecture/custom-nodes.md) - [Custom Dashboard Foundation](./architecture/custom-dashboard-foundation.md) - [Memory Claims and Evidence](./architecture/memory-claims.md) - [Speech Input Architecture](./architecture/speech-input.md) diff --git a/src/app/dependency-factory/dashboard-factory.ts b/src/app/dependency-factory/dashboard-factory.ts index 494ba30a63..32367a3a6f 100644 --- a/src/app/dependency-factory/dashboard-factory.ts +++ b/src/app/dependency-factory/dashboard-factory.ts @@ -39,6 +39,10 @@ import { EgressPolicyService } from "../../services/node-flows/egress-policy-ser import { AutomationApprovalRepository } from "../../repositories/automation-approval-repository.js"; import { AutomationOutboxRepository } from "../../repositories/automation-outbox-repository.js"; import { AutomationWebhookTriggerRepository } from "../../repositories/automation-webhook-trigger-repository.js"; +import { CustomNodeRepository } from "../../repositories/custom-node-repository.js"; +import { CustomNodeRuntimeService } from "../../services/custom-nodes/custom-node-runtime-service.js"; +import { customNodeDefinitionFromArtifact } from "../../contracts/custom-node-types.js"; +import { registerCustomNodeDefinition } from "../../domain/node-flows/node-definition-registry.js"; export interface DashboardDependencies { credentialBroker: CoreDependencies["credentialBroker"]; @@ -237,6 +241,16 @@ export function createDashboardDependencies( const webhookTriggerRepository = coreDeps.automationWebhookTriggerRepository ?? new AutomationWebhookTriggerRepository(coreDeps.appDbStorage); const approvalService = new ApprovalService(approvalRepository); + const egressPolicyService = new EgressPolicyService(); + const customNodeRepository = new CustomNodeRepository(coreDeps.appDbStorage); + for (const { artifact } of customNodeRepository.listPublications()) { + registerCustomNodeDefinition(customNodeDefinitionFromArtifact(artifact)); + } + const customNodeRuntimeService = new CustomNodeRuntimeService({ + repository: customNodeRepository, + credentialBroker: coreDeps.credentialBroker, + egressPolicyService, + }); const nodeFlowRuntimeService = new NodeFlowRuntimeService({ nodeFlowRepository: coreDeps.nodeFlowRepository, executionRepository, @@ -244,7 +258,8 @@ export function createDashboardDependencies( settingsRepository, providerExecutionService, credentialBroker: coreDeps.credentialBroker, - egressPolicyService: new EgressPolicyService(), + egressPolicyService, + customNodeRuntimeService, approvalService, outboxService: new OutboxService(outboxRepository, new MockSideEffectProvider()), getDashboardSettings: (projectId) => resolveDashboardSettings({ projectId }), diff --git a/src/contracts/custom-node-types.ts b/src/contracts/custom-node-types.ts new file mode 100644 index 0000000000..0181a20880 --- /dev/null +++ b/src/contracts/custom-node-types.ts @@ -0,0 +1,229 @@ +import type { NodeDefinitionManifest } from "./node-definition-types.js"; +import type { NodeFlowJsonObject, NodeFlowJsonValue, NodeFlowValueSchema } from "./node-flow-types.js"; + +export const CUSTOM_NODE_SCHEMA_VERSION = 1 as const; +export const CUSTOM_NODE_FEATURE_FLAG = "CODE_UX_CUSTOM_NODES_ENABLED" as const; + +export type CustomNodeLifecycleStatus = "draft" | "validating" | "passed" | "failed" | "published"; +export type CustomNodeCapability = + | "network.http" + | "credentials.read" + | "temporary-storage.write" + | "artifacts.write" + | "clock.read"; + +export interface CustomNodeCredentialSlot { + slot: string; + label: string; + required: boolean; + allowedKinds: string[]; + requiredCapability: string; +} + +export interface CustomNodeResourceLimits { + cpu: number; + memoryMb: number; + pids: number; + timeoutMs: number; + maxOutputBytes: number; + scratchMb: number; +} + +export interface CustomNodeHttpPolicy { + allowedHosts: string[]; + allowedPorts?: number[]; + maxRequests: number; + timeoutMs: number; + maxResponseBytes: number; +} + +export interface CustomNodeManifest { + schemaVersion: typeof CUSTOM_NODE_SCHEMA_VERSION; + id: string; + nodeType: string; + version: number; + name: string; + description: string; + entrypoint: "dist/index.js"; + inputSchema: NodeFlowValueSchema; + outputSchema: NodeFlowValueSchema; + configurationSchema: NodeFlowValueSchema; + capabilities: CustomNodeCapability[]; + credentials: CustomNodeCredentialSlot[]; + resources: CustomNodeResourceLimits; + http?: CustomNodeHttpPolicy; +} + +export interface CustomNodeLogger { + debug(message: string, fields?: NodeFlowJsonObject): void; + info(message: string, fields?: NodeFlowJsonObject): void; + warn(message: string, fields?: NodeFlowJsonObject): void; + error(message: string, fields?: NodeFlowJsonObject): void; +} + +export interface CustomNodeHttpRequest { + url: string; + method?: "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE"; + headers?: Record; + body?: string; +} + +export interface CustomNodeHttpResponse { + status: number; + headers: Record; + body: string; +} + +export interface CustomNodeArtifactWriter { + write(name: string, content: string | Uint8Array, mediaType: string): Promise<{ name: string; digest: string; size: number }>; +} + +/** The only authority exposed to generated code. Implementations are supplied by the isolated runner. */ +export interface NodeExecutionContext { + input: Readonly; + config: Readonly; + correlationId: string; + invocationId: string; + signal: AbortSignal; + logger: CustomNodeLogger; + clock: { now(): string }; + http: { request(request: CustomNodeHttpRequest): Promise }; + credentials: { get(slot: string): Promise }; + temporaryStorage: { + read(path: string): Promise; + write(path: string, content: string | Uint8Array): Promise; + }; + artifacts: CustomNodeArtifactWriter; +} + +export type CustomNodeHandler = (context: NodeExecutionContext) => Promise; + +export interface CustomNodeValidationIssue { + check: string; + code: string; + message: string; +} + +export interface CustomNodeValidationCheck { + name: string; + passed: boolean; + durationMs: number; + details?: string; +} + +export interface CustomNodeValidationReport { + valid: boolean; + checks: CustomNodeValidationCheck[]; + issues: CustomNodeValidationIssue[]; + validatedAt: string; +} + +export interface CustomNodeDependency { + name: string; + version: string; + integrity?: string; +} + +export interface CustomNodeArtifact { + digest: string; + nodeId: string; + projectId: string; + version: number; + sourceRevision: string; + buildDigest: string; + runtimeImageDigest: string; + dependencies: CustomNodeDependency[]; + validationReport: CustomNodeValidationReport; + createdBy: string; + invocationId: string; + correlationId: string; + capabilities: CustomNodeCapability[]; + manifest: CustomNodeManifest; + createdAt: string; +} + +export interface CustomNodeRecord { + id: string; + projectId: string; + status: CustomNodeLifecycleStatus; + sourceRevision: string; + manifest: CustomNodeManifest; + validationReport: CustomNodeValidationReport | null; + artifactDigest: string | null; + createdBy: string; + createdAt: string; + updatedAt: string; +} + +export interface CustomNodePublication { + id: string; + nodeId: string; + projectId: string; + nodeType: string; + version: number; + artifactDigest: string; + publishedBy: string; + publishedAt: string; +} + +export interface CustomNodeExecutionRequest { + projectId: string; + nodeType: string; + version: number; + input: NodeFlowJsonObject; + config: NodeFlowJsonObject; + credentialBindings: Record; + workspaceId: string; + invocationId: string; + correlationId: string; + signal?: AbortSignal; +} + +export interface CustomNodeExecutionResult { + output: NodeFlowJsonObject; + artifactDigest: string; + logs: string; + diagnostics: string; +} + +export interface CreateCustomNodeDraftInput { + manifest: CustomNodeManifest; + sourceRevision: string; + createdBy: string; +} + +export function customNodeDefinitionFromArtifact(artifact: CustomNodeArtifact): NodeDefinitionManifest { + const { manifest } = artifact; + return { + type: manifest.nodeType, + version: manifest.version, + executable: true, + executionKind: "custom", + configurationSchema: manifest.configurationSchema, + ui: { + label: manifest.name, + description: manifest.description, + category: "custom", + widgetSchema: { fields: [] }, + }, + ports: [ + { id: "input", direction: "input", schema: manifest.inputSchema, cardinality: "many" }, + { id: "output", direction: "output", schema: manifest.outputSchema, cardinality: "one" }, + ], + credentials: manifest.credentials.map((slot) => ({ + slot: slot.slot, + label: slot.label, + required: slot.required, + allowedKinds: [...slot.allowedKinds], + })), + capabilities: [...manifest.capabilities], + sideEffect: manifest.capabilities.includes("network.http") ? "external" : "none", + defaultPolicy: { retry: { maxAttempts: 1, backoffMs: 0 }, timeout: { timeoutMs: manifest.resources.timeoutMs } }, + documentation: "docs/architecture/custom-nodes.md", + deprecation: { deprecated: false }, + }; +} + +export function isNodeFlowJsonObject(value: NodeFlowJsonValue): value is NodeFlowJsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/contracts/node-definition-types.ts b/src/contracts/node-definition-types.ts index 6e49a1fca6..9638fa2d0a 100644 --- a/src/contracts/node-definition-types.ts +++ b/src/contracts/node-definition-types.ts @@ -6,7 +6,7 @@ import type { NodeWidgetSchema, } from "./node-flow-types.js"; -export type NodeDefinitionExecutionKind = "local" | "provider" | "http" | "unavailable"; +export type NodeDefinitionExecutionKind = "local" | "provider" | "http" | "custom" | "unavailable"; export interface NodeDefinitionCredentialRequirement { slot: string; diff --git a/src/domain/node-flows/node-definition-registry.ts b/src/domain/node-flows/node-definition-registry.ts index f32a23f8d5..960b1f3702 100644 --- a/src/domain/node-flows/node-definition-registry.ts +++ b/src/domain/node-flows/node-definition-registry.ts @@ -131,7 +131,7 @@ const manifests: NodeDefinitionManifest[] = [ const keyFor = (type: string, version: number): string => `${type}@${version}`; const registry = new Map(manifests.map((manifest) => [keyFor(manifest.type, manifest.version), manifest])); -export const listNodeDefinitions = (): readonly NodeDefinitionManifest[] => manifests; +export const listNodeDefinitions = (): readonly NodeDefinitionManifest[] => [...manifests]; export const resolveNodeDefinition = (type: string, version: number): NodeDefinitionManifest | null => ( registry.get(keyFor(type, version)) ?? null @@ -140,3 +140,17 @@ export const resolveNodeDefinition = (type: string, version: number): NodeDefini export const resolveLatestNodeDefinition = (type: string): NodeDefinitionManifest | null => ( manifests.filter((manifest) => manifest.type === type).sort((left, right) => right.version - left.version)[0] ?? null ); + +export const registerCustomNodeDefinition = (manifest: NodeDefinitionManifest): void => { + if (manifest.executionKind !== "custom" || !manifest.type.startsWith("custom.") || manifest.executable !== true) { + throw new Error("Only executable custom node definitions can be registered dynamically."); + } + const key = keyFor(manifest.type, manifest.version); + const existing = registry.get(key); + if (existing) { + if (JSON.stringify(existing) !== JSON.stringify(manifest)) throw new Error(`Node definition is immutable once registered: ${key}.`); + return; + } + registry.set(key, manifest); + manifests.push(manifest); +}; diff --git a/src/repositories/custom-node-repository.ts b/src/repositories/custom-node-repository.ts new file mode 100644 index 0000000000..f4d41d6ce5 --- /dev/null +++ b/src/repositories/custom-node-repository.ts @@ -0,0 +1,189 @@ +import { randomUUID } from "node:crypto"; +import type { + CreateCustomNodeDraftInput, + CustomNodeArtifact, + CustomNodeLifecycleStatus, + CustomNodeManifest, + CustomNodePublication, + CustomNodeRecord, + CustomNodeValidationReport, +} from "../contracts/custom-node-types.js"; +import { AppDbStorage } from "./app-db-storage.js"; +import type { DatabaseAdapter } from "./db/database-adapter.js"; +import { EntityNotFoundError, ValidationError } from "./repository-utils.js"; + +interface CustomNodeRow { + id: string; + project_id: string; + status: string; + source_revision: string; + manifest_json: string; + validation_report_json: string | null; + artifact_digest: string | null; + created_by: string; + created_at: string; + updated_at: string; +} + +interface CustomNodeArtifactRow { artifact_json: string } +interface CustomNodePublicationRow { + id: string; node_id: string; project_id: string; node_type: string; version: number | string; + artifact_digest: string; published_by: string; published_at: string; +} + +const LIFECYCLE_STATUSES: readonly CustomNodeLifecycleStatus[] = ["draft", "validating", "passed", "failed", "published"]; + +export class CustomNodeRepository { + private readonly db: DatabaseAdapter; + + constructor(storage: AppDbStorage = new AppDbStorage()) { + this.db = storage.getDatabase(); + } + + createDraft(projectId: string, input: CreateCustomNodeDraftInput): CustomNodeRecord { + this.requireProject(projectId); + if (!input.sourceRevision.trim()) throw new ValidationError("Custom node source revision is required."); + if (!input.createdBy.trim()) throw new ValidationError("Custom node creator is required."); + const now = new Date().toISOString(); + this.db.prepare(` + INSERT INTO custom_nodes ( + id, project_id, status, source_revision, manifest_json, validation_report_json, + artifact_digest, created_by, created_at, updated_at + ) VALUES (?, ?, 'draft', ?, ?, NULL, NULL, ?, ?, ?) + `).run(input.manifest.id, projectId, input.sourceRevision.trim(), JSON.stringify(input.manifest), input.createdBy.trim(), now, now); + return this.requireNode(input.manifest.id); + } + + getNode(nodeId: string): CustomNodeRecord | null { + const row = this.db.prepare("SELECT * FROM custom_nodes WHERE id = ?").get(nodeId) as CustomNodeRow | undefined; + return row ? this.mapNode(row) : null; + } + + listProjectNodes(projectId: string): CustomNodeRecord[] { + this.requireProject(projectId); + return (this.db.prepare("SELECT * FROM custom_nodes WHERE project_id = ? ORDER BY updated_at DESC").all(projectId) as unknown as CustomNodeRow[]) + .map((row) => this.mapNode(row)); + } + + beginValidation(nodeId: string): CustomNodeRecord { + const node = this.requireNode(nodeId); + if (node.status === "published") throw new ValidationError("Published custom node revisions are immutable."); + this.updateLifecycle(nodeId, "validating", null, null); + return this.requireNode(nodeId); + } + + completeValidation(nodeId: string, report: CustomNodeValidationReport, artifact: CustomNodeArtifact | null): CustomNodeRecord { + const node = this.requireNode(nodeId); + if (node.status !== "validating") throw new ValidationError("Custom node must be validating before validation can complete."); + if (report.valid !== Boolean(artifact)) throw new ValidationError("Passed validation requires an artifact and failed validation must not publish one."); + if (artifact && (artifact.nodeId !== node.id || artifact.projectId !== node.projectId || artifact.sourceRevision !== node.sourceRevision)) { + throw new ValidationError("Custom node artifact does not match the validated draft."); + } + this.db.transaction(() => { + if (artifact) this.insertArtifact(artifact); + this.updateLifecycle(nodeId, report.valid ? "passed" : "failed", report, artifact?.digest ?? null); + }); + return this.requireNode(nodeId); + } + + publish(nodeId: string, publishedBy: string): CustomNodePublication { + const node = this.requireNode(nodeId); + if (node.status !== "passed" || !node.artifactDigest || node.validationReport?.valid !== true) { + throw new ValidationError("Only a passed custom node artifact can be published."); + } + if (!publishedBy.trim()) throw new ValidationError("Custom node publisher is required."); + const artifact = this.getArtifact(node.artifactDigest); + if (!artifact) throw new EntityNotFoundError(`Custom node artifact not found: ${node.artifactDigest}`); + const now = new Date().toISOString(); + const publication: CustomNodePublication = { + id: randomUUID(), nodeId: node.id, projectId: node.projectId, nodeType: node.manifest.nodeType, + version: node.manifest.version, artifactDigest: artifact.digest, publishedBy: publishedBy.trim(), publishedAt: now, + }; + this.db.transaction(() => { + this.db.prepare(` + INSERT INTO custom_node_publications ( + id, node_id, project_id, node_type, version, artifact_digest, published_by, published_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run(publication.id, publication.nodeId, publication.projectId, publication.nodeType, publication.version, + publication.artifactDigest, publication.publishedBy, publication.publishedAt); + this.updateLifecycle(node.id, "published", node.validationReport, node.artifactDigest); + }); + return publication; + } + + getArtifact(digest: string): CustomNodeArtifact | null { + const row = this.db.prepare("SELECT artifact_json FROM custom_node_artifacts WHERE digest = ?").get(digest) as CustomNodeArtifactRow | undefined; + return row ? JSON.parse(row.artifact_json) as CustomNodeArtifact : null; + } + + resolvePublished(nodeType: string, version: number): { publication: CustomNodePublication; artifact: CustomNodeArtifact } | null { + const row = this.db.prepare("SELECT * FROM custom_node_publications WHERE node_type = ? AND version = ?") + .get(nodeType, version) as CustomNodePublicationRow | undefined; + if (!row) return null; + const publication = this.mapPublication(row); + const artifact = this.getArtifact(publication.artifactDigest); + return artifact ? { publication, artifact } : null; + } + + listPublications(): Array<{ publication: CustomNodePublication; artifact: CustomNodeArtifact }> { + const rows = this.db.prepare("SELECT * FROM custom_node_publications ORDER BY published_at ASC").all() as unknown as CustomNodePublicationRow[]; + return rows.flatMap((row) => { + const publication = this.mapPublication(row); + const artifact = this.getArtifact(publication.artifactDigest); + return artifact ? [{ publication, artifact }] : []; + }); + } + + private insertArtifact(artifact: CustomNodeArtifact): void { + const serialized = JSON.stringify(artifact); + const existing = this.db.prepare("SELECT artifact_json FROM custom_node_artifacts WHERE digest = ?").get(artifact.digest) as CustomNodeArtifactRow | undefined; + if (existing) { + if (existing.artifact_json !== serialized) throw new ValidationError("Content-addressed custom node artifact digest collision."); + return; + } + this.db.prepare(` + INSERT INTO custom_node_artifacts (digest, node_id, project_id, version, artifact_json, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(artifact.digest, artifact.nodeId, artifact.projectId, artifact.version, serialized, artifact.createdAt); + } + + private updateLifecycle( + nodeId: string, + status: CustomNodeLifecycleStatus, + report: CustomNodeValidationReport | null, + artifactDigest: string | null, + ): void { + this.db.prepare(` + UPDATE custom_nodes + SET status = ?, validation_report_json = ?, artifact_digest = ?, updated_at = ? + WHERE id = ? + `).run(status, report ? JSON.stringify(report) : null, artifactDigest, new Date().toISOString(), nodeId); + } + + private requireProject(projectId: string): void { + if (!this.db.prepare("SELECT id FROM projects WHERE id = ?").get(projectId)) throw new EntityNotFoundError(`Project not found: ${projectId}`); + } + + private requireNode(nodeId: string): CustomNodeRecord { + const node = this.getNode(nodeId); + if (!node) throw new EntityNotFoundError(`Custom node not found: ${nodeId}`); + return node; + } + + private mapNode(row: CustomNodeRow): CustomNodeRecord { + if (!LIFECYCLE_STATUSES.includes(row.status as CustomNodeLifecycleStatus)) throw new ValidationError(`Invalid custom node status: ${row.status}`); + return { + id: row.id, projectId: row.project_id, status: row.status as CustomNodeLifecycleStatus, + sourceRevision: row.source_revision, manifest: JSON.parse(row.manifest_json) as CustomNodeManifest, + validationReport: row.validation_report_json ? JSON.parse(row.validation_report_json) as CustomNodeValidationReport : null, + artifactDigest: row.artifact_digest, createdBy: row.created_by, createdAt: row.created_at, updatedAt: row.updated_at, + }; + } + + private mapPublication(row: CustomNodePublicationRow): CustomNodePublication { + return { + id: row.id, nodeId: row.node_id, projectId: row.project_id, nodeType: row.node_type, + version: Number(row.version), artifactDigest: row.artifact_digest, publishedBy: row.published_by, publishedAt: row.published_at, + }; + } +} diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index 3206e63557..a83425e514 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -596,6 +596,55 @@ export function ensureAutomationCredentialTables(db: DatabaseAdapter): void { ensureIndex(db, "idx_automation_credential_rotations_credential", "automation_credential_rotations", "credential_id, rotated_at DESC"); } +export function ensureCustomNodeTables(db: DatabaseAdapter): void { + db.exec(` + CREATE TABLE IF NOT EXISTS custom_nodes ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'draft', + source_revision TEXT NOT NULL, + manifest_json TEXT NOT NULL, + validation_report_json TEXT, + artifact_digest TEXT, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS custom_node_artifacts ( + digest TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL, + artifact_json TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (node_id) REFERENCES custom_nodes(id) ON DELETE CASCADE, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE + ) + `); + db.exec(` + CREATE TABLE IF NOT EXISTS custom_node_publications ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + project_id TEXT NOT NULL, + node_type TEXT NOT NULL, + version INTEGER NOT NULL, + artifact_digest TEXT NOT NULL, + published_by TEXT NOT NULL, + published_at TEXT NOT NULL, + FOREIGN KEY (node_id) REFERENCES custom_nodes(id) ON DELETE CASCADE, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + FOREIGN KEY (artifact_digest) REFERENCES custom_node_artifacts(digest), + UNIQUE (node_type, version) + ) + `); + ensureIndex(db, "idx_custom_nodes_project_status", "custom_nodes", "project_id, status, updated_at DESC"); + ensureIndex(db, "idx_custom_node_artifacts_node", "custom_node_artifacts", "node_id, version DESC"); + ensureIndex(db, "idx_custom_node_publications_project", "custom_node_publications", "project_id, published_at DESC"); +} + export function migrateSprintLinkedIssuesExternalSources(db: DatabaseAdapter): void { ensureColumn(db, "sprint_linked_issues", "project_key", "TEXT"); ensureColumn(db, "sprint_linked_issues", "external_id", "TEXT"); @@ -833,6 +882,7 @@ export function runMigrations(db: DatabaseAdapter): void { migratePersistedNodeFlowGraphs(db); ensureCustomDashboardTables(db); ensureAutomationCredentialTables(db); + ensureCustomNodeTables(db); ensureColumn(db, "projects", "initialization_mode", "TEXT NOT NULL DEFAULT 'existing'"); ensureColumn(db, "provider_invocations", "tool_call_count", "INTEGER NOT NULL DEFAULT 0"); diff --git a/src/services/custom-nodes/custom-node-build-service.ts b/src/services/custom-nodes/custom-node-build-service.ts new file mode 100644 index 0000000000..7225c87ee3 --- /dev/null +++ b/src/services/custom-nodes/custom-node-build-service.ts @@ -0,0 +1,339 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { + CustomNodeArtifact, + CustomNodeCapability, + CustomNodeDependency, + CustomNodeManifest, + CustomNodeValidationCheck, + CustomNodeValidationIssue, + CustomNodeValidationReport, +} from "../../contracts/custom-node-types.js"; +import { CUSTOM_NODE_SCHEMA_VERSION } from "../../contracts/custom-node-types.js"; +import type { CustomNodeRepository } from "../../repositories/custom-node-repository.js"; +import { EntityNotFoundError, ValidationError } from "../../repositories/repository-utils.js"; +import { customNodeDefinitionFromArtifact } from "../../contracts/custom-node-types.js"; +import { registerCustomNodeDefinition } from "../../domain/node-flows/node-definition-registry.js"; +import { runCommandStrict, type CommandResult } from "../cli-process-runner.js"; +import { createCustomNodeDockerfile, CustomNodeProjectService } from "./custom-node-project-service.js"; +import { buildCustomNodeDockerRunArgs, redactSecrets, validateValueAgainstSchema } from "./custom-node-runtime-service.js"; + +const ALLOWED_CAPABILITIES = new Set([ + "network.http", "credentials.read", "temporary-storage.write", "artifacts.write", "clock.read", +]); +const SOURCE_LIMIT_BYTES = 2 * 1024 * 1024; +const FILE_LIMIT = 256; +const PROHIBITED_SOURCE_PATTERNS: Array<{ code: string; pattern: RegExp; message: string }> = [ + { code: "host_filesystem", pattern: /(?:from\s*|import\s*\(|require\s*\()\s*["'](?:node:)?fs(?:\/promises)?["']|process\.cwd|import\.meta\.url/, message: "Host filesystem APIs are prohibited." }, + { code: "host_environment", pattern: /process\.env|process\.argv|process\.execPath/, message: "Host process environment APIs are prohibited." }, + { code: "subprocess", pattern: /["'](?:node:)?child_process["']|\bspawn\s*\(|\bexec(?:File)?\s*\(|\bfork\s*\(/, message: "Subprocess APIs are prohibited." }, + { code: "docker", pattern: /\/var\/run\/docker\.sock|DOCKER_HOST|\bdocker\s+(?:run|exec|build)/i, message: "Docker access is prohibited." }, + { code: "raw_network", pattern: /\bfetch\s*\(|["'](?:node:)?(?:http|https|net|tls|dns)(?:\/promises)?["']|\bWebSocket\b/, message: "Raw network APIs are prohibited; use context.http." }, + { code: "native_escape", pattern: /["'](?:node:)?(?:worker_threads|cluster)["']|\bDeno\.|\bBun\./, message: "Native worker and alternate runtime APIs are prohibited." }, +]; + +export interface CustomNodeAuditResult { passed: boolean; details: string } +export interface ValidateCustomNodeInput { + projectRoot: string; + nodeId: string; + creator: string; + invocationId: string; + correlationId: string; + signal?: AbortSignal; +} + +export interface CustomNodeBuildServiceDeps { + repository: CustomNodeRepository; + projectService?: CustomNodeProjectService; + commandRunner?: (command: string, args: string[], cwd: string, options?: { signal?: AbortSignal; timeout?: number; maxStdoutChars?: number; stdinFile?: string }) => Promise; + vulnerabilityAudit?: (dependencies: readonly CustomNodeDependency[], signal?: AbortSignal) => Promise; + runtimeImage?: string; +} + +export class CustomNodeBuildService { + private readonly projectService: CustomNodeProjectService; + + constructor(private readonly deps: CustomNodeBuildServiceDeps) { + this.projectService = deps.projectService ?? new CustomNodeProjectService(); + } + + publish(nodeId: string, publishedBy: string): CustomNodeArtifact { + const publication = this.deps.repository.publish(nodeId, publishedBy); + const artifact = this.deps.repository.getArtifact(publication.artifactDigest); + if (!artifact) throw new EntityNotFoundError(`Custom node artifact not found: ${publication.artifactDigest}`); + registerCustomNodeDefinition(customNodeDefinitionFromArtifact(artifact)); + return artifact; + } + + registerPublishedDefinitions(): number { + const published = this.deps.repository.listPublications(); + for (const { artifact } of published) registerCustomNodeDefinition(customNodeDefinitionFromArtifact(artifact)); + return published.length; + } + + async validateAndBuild(input: ValidateCustomNodeInput): Promise<{ report: CustomNodeValidationReport; artifact: CustomNodeArtifact | null }> { + const node = this.deps.repository.getNode(input.nodeId); + if (!node) throw new EntityNotFoundError(`Custom node not found: ${input.nodeId}`); + const root = this.projectService.resolveNodeRoot(input.projectRoot, input.nodeId); + this.deps.repository.beginValidation(node.id); + const started = Date.now(); + const checks: CustomNodeValidationCheck[] = []; + const issues: CustomNodeValidationIssue[] = []; + try { + const bundle = await readSourceBundle(root); + const manifest = parseManifest(bundle.get("node.json")); + runCheck("manifest-schema", checks, issues, () => validateManifest(manifest, node.manifest)); + runCheck("prohibited-api-scan", checks, issues, () => scanProhibitedApis(bundle)); + runCheck("capability-comparison", checks, issues, () => compareCapabilities(manifest, bundle)); + const packageJson = parsePackageJson(bundle.get("package.json")); + const dependencies = dependencyInventory(packageJson); + runCheck("lockfile-verification", checks, issues, () => verifyLockfile(bundle.get("pnpm-lock.yaml"), dependencies)); + runCheck("trusted-build-recipe", checks, issues, () => { + if (bundle.get("Dockerfile") !== createCustomNodeDockerfile()) throw new ValidationError("Custom node Dockerfile must match the trusted generated build recipe."); + }); + runCheck("resource-policy", checks, issues, () => validateResourcePolicy(manifest)); + + if (issues.length === 0) { + const auditStarted = Date.now(); + const audit = this.deps.vulnerabilityAudit + ? await this.deps.vulnerabilityAudit(dependencies, input.signal) + : { passed: false, details: "The governed vulnerability-audit hook is not configured." }; + checks.push({ name: "vulnerability-audit", passed: audit.passed, durationMs: Date.now() - auditStarted, details: audit.details }); + if (!audit.passed) issues.push({ check: "vulnerability-audit", code: "vulnerability_audit_failed", message: audit.details }); + } + + let runtimeImageDigest = ""; + if (issues.length === 0) { + const sourceDigest = digestBundle(bundle); + const tag = `code-ux-custom-node:${sourceDigest.slice("sha256:".length, 28)}`; + const buildStarted = Date.now(); + try { + await this.run("docker", ["build", "--pull=false", "--label", `code-ux.custom-node-source=${sourceDigest}`, "-t", tag, "."], root, input.signal, 10 * 60_000); + const inspect = await this.run("docker", ["image", "inspect", "--format", "{{.Id}}", tag], root, input.signal, 30_000); + runtimeImageDigest = inspect.stdout.trim(); + if (!/^sha256:[a-f0-9]{64}$/i.test(runtimeImageDigest)) throw new Error("Docker returned a non-content-addressed image id."); + checks.push({ name: "typescript-and-tests", passed: true, durationMs: Date.now() - buildStarted, details: "Frozen install, typecheck, build, and deterministic tests completed in the image build." }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + checks.push({ name: "typescript-and-tests", passed: false, durationMs: Date.now() - buildStarted, details: message }); + issues.push({ check: "typescript-and-tests", code: "container_build_failed", message }); + } + + if (runtimeImageDigest && issues.length === 0) { + await this.validateFixture({ root, manifest, runtimeImageDigest, sourceDigest, input, checks, issues }); + } + + if (issues.length === 0) { + const report = reportFor(checks, issues); + const artifactSeed = { + nodeId: node.id, projectId: node.projectId, version: manifest.version, + sourceRevision: node.sourceRevision, buildDigest: sourceDigest, runtimeImageDigest, + dependencies, validationReport: report, createdBy: input.creator, + invocationId: input.invocationId, correlationId: input.correlationId, + capabilities: manifest.capabilities, manifest, + }; + const createdAt = new Date().toISOString(); + const digest = `sha256:${createHash("sha256").update(canonicalJson(artifactSeed)).digest("hex")}`; + const artifact: CustomNodeArtifact = { digest, ...artifactSeed, createdAt }; + this.deps.repository.completeValidation(node.id, report, artifact); + return { report, artifact }; + } + } + } catch (error) { + issues.push({ check: "validation", code: "validation_error", message: error instanceof Error ? error.message : String(error) }); + } + const report = reportFor(checks, issues); + if (checks.length === 0) checks.push({ name: "validation", passed: false, durationMs: Date.now() - started, details: issues[0]?.message }); + this.deps.repository.completeValidation(node.id, report, null); + return { report, artifact: null }; + } + + private async validateFixture(args: { + root: string; manifest: CustomNodeManifest; runtimeImageDigest: string; sourceDigest: string; + input: ValidateCustomNodeInput; checks: CustomNodeValidationCheck[]; issues: CustomNodeValidationIssue[]; + }): Promise { + const started = Date.now(); + const runDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-custom-node-validation-")); + const canary = `CODEUX_SECRET_CANARY_${args.input.correlationId}`; + try { + const fixture = JSON.parse(await fs.readFile(path.join(args.root, "fixtures", "basic.json"), "utf8")) as { input: unknown; config: unknown; output: unknown }; + if (!isObject(fixture.input) || !isObject(fixture.config) || !isObject(fixture.output)) throw new ValidationError("Fixture input, config, and output must be objects."); + const inputIssues = validateValueAgainstSchema(fixture.input, args.manifest.inputSchema, "fixture.input"); + if (inputIssues.length) throw new ValidationError(inputIssues[0]!); + const envelope = { input: fixture.input, config: fixture.config, correlationId: args.input.correlationId, invocationId: args.input.invocationId, credentials: { __canary: canary }, now: "2026-01-01T00:00:00.000Z" }; + const inputPath = path.join(runDirectory, "input.json"); + await fs.writeFile(inputPath, JSON.stringify(envelope), { mode: 0o600 }); + if (process.platform !== "win32") await fs.chmod(inputPath, 0o600); + const artifact = { + digest: args.sourceDigest, runtimeImageDigest: args.runtimeImageDigest, manifest: args.manifest, + } as CustomNodeArtifact; + const plan = buildCustomNodeDockerRunArgs({ artifact, containerName: `code-ux-custom-node-validation-${args.input.nodeId}` }); + const result = await this.run("docker", plan, args.root, args.input.signal, args.manifest.resources.timeoutMs, inputPath); + const outputText = result.stdout; + if (Buffer.byteLength(outputText) > args.manifest.resources.maxOutputBytes) throw new ValidationError("Fixture output exceeded the declared limit."); + const output = JSON.parse(outputText) as unknown; + const outputIssues = validateValueAgainstSchema(output, args.manifest.outputSchema, "fixture.output"); + if (outputIssues.length) throw new ValidationError(outputIssues[0]!); + if (canonicalJson(output) !== canonicalJson(fixture.output)) throw new ValidationError("Fixture output did not match the deterministic expected output."); + const observable = redactSecrets(`${result.stdout}\n${result.stderr}\n${outputText}`, [canary]); + if (observable.includes(canary)) throw new ValidationError("Secret canary survived output redaction."); + args.checks.push({ name: "fixture-resource-network-secret", passed: true, durationMs: Date.now() - started, details: "Fixture ran with network-none, bounded resources, an ephemeral scratch/run directory, and secret-canary redaction." }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + args.checks.push({ name: "fixture-resource-network-secret", passed: false, durationMs: Date.now() - started, details: message }); + args.issues.push({ check: "fixture-resource-network-secret", code: "fixture_failed", message }); + } finally { + await fs.rm(runDirectory, { recursive: true, force: true }); + } + } + + private async run(command: string, args: string[], cwd: string, signal: AbortSignal | undefined, timeout: number, stdinFile?: string): Promise { + const runner = this.deps.commandRunner ?? ((cmd, argv, dir, options) => runCommandStrict(cmd, argv, dir, process.env, options)); + return runner(command, args, cwd, { signal, timeout, maxStdoutChars: 1024 * 1024, stdinFile }); + } +} + +async function readSourceBundle(root: string): Promise> { + const bundle = new Map(); + const visit = async (directory: string): Promise => { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + const relative = path.relative(root, target).split(path.sep).join("/"); + if (entry.isSymbolicLink()) throw new ValidationError(`Symbolic links are prohibited in custom node packages: ${relative}`); + if (entry.isDirectory()) { + if (!new Set(["node_modules", "dist", ".git"]).has(entry.name)) await visit(target); + continue; + } + if (!entry.isFile()) continue; + if (bundle.size >= FILE_LIMIT) throw new ValidationError("Custom node package contains too many files."); + const stat = await fs.stat(target); + if (stat.size > SOURCE_LIMIT_BYTES) throw new ValidationError(`Custom node file is too large: ${relative}`); + bundle.set(relative, await fs.readFile(target, "utf8")); + } + }; + await visit(root); + for (const required of ["node.json", "package.json", "pnpm-lock.yaml", "tsconfig.json", "Dockerfile", "src/index.ts", "src/sdk.ts", "src/runner.ts", "tests/index.test.ts", "fixtures/basic.json"]) { + if (!bundle.has(required)) throw new ValidationError(`Custom node package is missing ${required}.`); + } + return bundle; +} + +function parseManifest(value: string | undefined): CustomNodeManifest { + if (!value) throw new ValidationError("Custom node manifest is missing."); + try { return JSON.parse(value) as CustomNodeManifest; } catch { throw new ValidationError("Custom node manifest is invalid JSON."); } +} + +function parsePackageJson(value: string | undefined): Record { + if (!value) throw new ValidationError("Custom node package metadata is missing."); + try { const parsed = JSON.parse(value) as unknown; if (isObject(parsed)) return parsed; } catch { /* normalized below */ } + throw new ValidationError("Custom node package metadata is invalid JSON."); +} + +function validateManifest(manifest: CustomNodeManifest, persisted: CustomNodeManifest): void { + if (manifest.schemaVersion !== CUSTOM_NODE_SCHEMA_VERSION) throw new ValidationError("Unsupported custom node manifest schema version."); + if (!/^[a-z][a-z0-9-]{2,63}$/.test(manifest.id) || manifest.nodeType !== `custom.${manifest.id}`) throw new ValidationError("Custom node id or type is invalid."); + if (manifest.id !== persisted.id || manifest.nodeType !== persisted.nodeType || manifest.version !== persisted.version) throw new ValidationError("Filesystem manifest does not match the persisted draft identity."); + if (!Number.isInteger(manifest.version) || manifest.version < 1 || manifest.entrypoint !== "dist/index.js") throw new ValidationError("Custom node version or entrypoint is invalid."); + for (const schema of [manifest.inputSchema, manifest.outputSchema, manifest.configurationSchema]) validateSchema(schema); + const slots = new Set(); + for (const slot of manifest.credentials) { + if (!/^[a-z][a-z0-9_-]{0,63}$/.test(slot.slot) || slots.has(slot.slot) || !slot.requiredCapability.trim()) throw new ValidationError("Custom node credential slots must be unique and valid."); + slots.add(slot.slot); + } + if (new Set(manifest.capabilities).size !== manifest.capabilities.length || manifest.capabilities.some((capability) => !ALLOWED_CAPABILITIES.has(capability))) throw new ValidationError("Custom node declares an unknown or duplicate capability."); + if (manifest.http && !manifest.capabilities.includes("network.http")) throw new ValidationError("HTTP policy requires network.http capability."); + if (manifest.capabilities.includes("network.http") && !manifest.http) throw new ValidationError("network.http capability requires a bounded HTTP policy."); + if (manifest.credentials.length && !manifest.capabilities.includes("credentials.read")) throw new ValidationError("Credential slots require credentials.read capability."); +} + +function validateSchema(schema: unknown): void { + if (!isObject(schema) || !["any", "object", "array", "string", "number", "boolean", "null"].includes(String(schema.type))) throw new ValidationError("Custom node value schema is invalid."); + if (schema.type === "object" && schema.properties !== undefined) { + if (!isObject(schema.properties)) throw new ValidationError("Custom node schema properties must be an object."); + for (const child of Object.values(schema.properties)) validateSchema(child); + } + if (schema.type === "array" && schema.items !== undefined) validateSchema(schema.items); +} + +function scanProhibitedApis(bundle: ReadonlyMap): void { + for (const [file, source] of bundle) { + if (!file.startsWith("src/") || file === "src/sdk.ts" || file === "src/runner.ts") continue; + for (const rule of PROHIBITED_SOURCE_PATTERNS) if (rule.pattern.test(source)) throw new ValidationError(`${rule.message} (${file}, ${rule.code})`); + } +} + +function compareCapabilities(manifest: CustomNodeManifest, bundle: ReadonlyMap): void { + const source = [...bundle].filter(([file]) => file.startsWith("src/") && !["src/sdk.ts", "src/runner.ts"].includes(file)).map(([, content]) => content).join("\n"); + const inferred = new Set(); + if (/context\.http\b/.test(source)) inferred.add("network.http"); + if (/context\.credentials\b/.test(source)) inferred.add("credentials.read"); + if (/context\.temporaryStorage\b/.test(source)) inferred.add("temporary-storage.write"); + if (/context\.artifacts\b/.test(source)) inferred.add("artifacts.write"); + if (/context\.clock\b/.test(source)) inferred.add("clock.read"); + const declared = new Set(manifest.capabilities); + const missing = [...inferred].filter((capability) => !declared.has(capability)); + const unused = [...declared].filter((capability) => !inferred.has(capability) && capability !== "credentials.read"); + if (missing.length || unused.length) throw new ValidationError(`Declared capabilities must exactly match SDK usage (missing: ${missing.join(", ") || "none"}; unused: ${unused.join(", ") || "none"}).`); +} + +function validateResourcePolicy(manifest: CustomNodeManifest): void { + const { resources } = manifest; + if (!(resources.cpu >= 0.1 && resources.cpu <= 4)) throw new ValidationError("Custom node CPU limit must be between 0.1 and 4."); + if (!Number.isInteger(resources.memoryMb) || resources.memoryMb < 32 || resources.memoryMb > 2048) throw new ValidationError("Custom node memory limit must be between 32 and 2048 MiB."); + if (!Number.isInteger(resources.pids) || resources.pids < 16 || resources.pids > 256) throw new ValidationError("Custom node PID limit must be between 16 and 256."); + if (!Number.isInteger(resources.timeoutMs) || resources.timeoutMs < 100 || resources.timeoutMs > 120_000) throw new ValidationError("Custom node timeout must be between 100 and 120000ms."); + if (!Number.isInteger(resources.maxOutputBytes) || resources.maxOutputBytes < 1024 || resources.maxOutputBytes > 2 * 1024 * 1024) throw new ValidationError("Custom node output limit is invalid."); + if (!Number.isInteger(resources.scratchMb) || resources.scratchMb < 1 || resources.scratchMb > 256) throw new ValidationError("Custom node scratch limit is invalid."); + if (manifest.http) { + if (!manifest.http.allowedHosts.length || manifest.http.allowedHosts.some((host) => !/^[a-z0-9.-]+$/i.test(host))) throw new ValidationError("Custom node HTTP hosts must be explicit DNS names."); + if (manifest.http.maxRequests < 1 || manifest.http.maxRequests > 100 || manifest.http.timeoutMs > resources.timeoutMs || manifest.http.maxResponseBytes > resources.maxOutputBytes) throw new ValidationError("Custom node HTTP policy exceeds resource bounds."); + } +} + +function dependencyInventory(packageJson: Record): CustomNodeDependency[] { + const result: CustomNodeDependency[] = []; + for (const field of ["dependencies", "devDependencies"] as const) { + const dependencies = packageJson[field]; + if (!isObject(dependencies)) continue; + for (const [name, version] of Object.entries(dependencies)) { + if (typeof version !== "string" || !/^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$/.test(version)) throw new ValidationError(`Dependency ${name} must use an exact version.`); + result.push({ name, version }); + } + } + return result.sort((left, right) => left.name.localeCompare(right.name)); +} + +function verifyLockfile(lockfile: string | undefined, dependencies: readonly CustomNodeDependency[]): void { + if (!lockfile?.startsWith("lockfileVersion: '9.0'")) throw new ValidationError("A pnpm v9 frozen lockfile is required."); + for (const dependency of dependencies) { + if (!lockfile.includes(`specifier: ${dependency.version}`) || !lockfile.includes(`version: ${dependency.version}`)) throw new ValidationError(`Lockfile does not pin ${dependency.name}@${dependency.version}.`); + } +} + +function digestBundle(bundle: ReadonlyMap): string { + const hash = createHash("sha256"); + for (const [file, content] of [...bundle].sort(([left], [right]) => left.localeCompare(right))) hash.update(file).update("\0").update(content).update("\0"); + return `sha256:${hash.digest("hex")}`; +} + +function runCheck(name: string, checks: CustomNodeValidationCheck[], issues: CustomNodeValidationIssue[], action: () => void): void { + const started = Date.now(); + try { action(); checks.push({ name, passed: true, durationMs: Date.now() - started }); } + catch (error) { const message = error instanceof Error ? error.message : String(error); checks.push({ name, passed: false, durationMs: Date.now() - started, details: message }); issues.push({ check: name, code: `${name.replace(/-/g, "_")}_failed`, message }); } +} + +function reportFor(checks: CustomNodeValidationCheck[], issues: CustomNodeValidationIssue[]): CustomNodeValidationReport { + return { valid: issues.length === 0, checks, issues, validatedAt: new Date().toISOString() }; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (isObject(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/services/custom-nodes/custom-node-project-service.ts b/src/services/custom-nodes/custom-node-project-service.ts new file mode 100644 index 0000000000..fb33698a2a --- /dev/null +++ b/src/services/custom-nodes/custom-node-project-service.ts @@ -0,0 +1,133 @@ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import type { CustomNodeManifest } from "../../contracts/custom-node-types.js"; +import { CUSTOM_NODE_SCHEMA_VERSION } from "../../contracts/custom-node-types.js"; +import { ValidationError } from "../../repositories/repository-utils.js"; + +const NODE_ID_PATTERN = /^[a-z][a-z0-9-]{2,63}$/; + +export interface GenerateCustomNodeProjectInput { + projectRoot: string; + nodeId: string; + name: string; + description?: string; + overwrite?: boolean; +} + +export interface GeneratedCustomNodeProject { + root: string; + manifest: CustomNodeManifest; + files: string[]; +} + +export class CustomNodeProjectService { + async generate(input: GenerateCustomNodeProjectInput): Promise { + const nodeId = input.nodeId.trim(); + if (!NODE_ID_PATTERN.test(nodeId)) { + throw new ValidationError("Custom node id must be 3-64 lowercase letters, numbers, or hyphens and start with a letter."); + } + const projectRoot = path.resolve(input.projectRoot); + const root = path.resolve(projectRoot, ".code-ux", "nodes", nodeId); + this.requireContained(projectRoot, root); + if (!input.overwrite) { + await fs.access(root).then( + () => { throw new ValidationError(`Custom node project already exists: ${nodeId}`); }, + () => undefined, + ); + } else { + await fs.rm(root, { recursive: true, force: true }); + } + + const manifest = defaultManifest(nodeId, input.name, input.description ?? ""); + const files = projectFiles(manifest); + await fs.mkdir(root, { recursive: true, mode: 0o700 }); + for (const [relativePath, content] of Object.entries(files)) { + const target = path.resolve(root, relativePath); + this.requireContained(root, target); + await fs.mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + await fs.writeFile(target, content, { encoding: "utf8", mode: 0o600 }); + } + return { root, manifest, files: Object.keys(files).sort() }; + } + + async readManifest(projectRoot: string, nodeId: string): Promise { + const root = this.resolveNodeRoot(projectRoot, nodeId); + let parsed: unknown; + try { + parsed = JSON.parse(await fs.readFile(path.join(root, "node.json"), "utf8")); + } catch (error) { + throw new ValidationError(`Unable to read custom node manifest: ${error instanceof Error ? error.message : String(error)}`); + } + return parsed as CustomNodeManifest; + } + + resolveNodeRoot(projectRoot: string, nodeId: string): string { + if (!NODE_ID_PATTERN.test(nodeId)) throw new ValidationError("Invalid custom node id."); + const base = path.resolve(projectRoot); + const root = path.resolve(base, ".code-ux", "nodes", nodeId); + this.requireContained(base, root); + return root; + } + + private requireContained(basePath: string, targetPath: string): void { + const relative = path.relative(basePath, targetPath); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new ValidationError("Custom node path escapes the project root."); + } +} + +function defaultManifest(nodeId: string, name: string, description: string): CustomNodeManifest { + return { + schemaVersion: CUSTOM_NODE_SCHEMA_VERSION, + id: nodeId, + nodeType: `custom.${nodeId}`, + version: 1, + name: name.trim() || nodeId, + description: description.trim(), + entrypoint: "dist/index.js", + inputSchema: { type: "object" }, + outputSchema: { type: "object" }, + configurationSchema: { type: "object" }, + capabilities: ["clock.read"], + credentials: [], + resources: { cpu: 0.5, memoryMb: 128, pids: 64, timeoutMs: 30_000, maxOutputBytes: 262_144, scratchMb: 32 }, + }; +} + +function projectFiles(manifest: CustomNodeManifest): Record { + const packageName = `@codeux-custom/${manifest.id}`; + return { + "node.json": `${JSON.stringify(manifest, null, 2)}\n`, + "package.json": `${JSON.stringify({ + name: packageName, version: `${manifest.version}.0.0`, private: true, type: "module", packageManager: "pnpm@10.33.0", + scripts: { typecheck: "tsc --noEmit", build: "tsc", test: "node --test dist/tests/*.test.js" }, + devDependencies: { "@types/node": "25.6.0", typescript: "5.9.3" }, + }, null, 2)}\n`, + "pnpm-lock.yaml": lockfile(packageName), + "tsconfig.json": `${JSON.stringify({ compilerOptions: { + target: "ES2022", module: "NodeNext", moduleResolution: "NodeNext", strict: true, + outDir: "dist", rootDir: ".", declaration: true, noUncheckedIndexedAccess: true, + }, include: ["src/**/*.ts", "tests/**/*.ts"] }, null, 2)}\n`, + "src/sdk.ts": sdkSource(), + "src/index.ts": `import type { CustomNodeHandler } from "./sdk.js";\n\nexport const run: CustomNodeHandler = async (context) => {\n context.logger.info("custom node started", { correlationId: context.correlationId });\n return { ...context.input, executedAt: context.clock.now() };\n};\n`, + "src/runner.ts": runnerSource(), + "tests/index.test.ts": `import assert from "node:assert/strict";\nimport test from "node:test";\nimport { run } from "../src/index.js";\n\ntest("returns deterministic fixture output", async () => {\n const context = { input: { value: 1 }, config: {}, correlationId: "fixture", invocationId: "fixture", signal: new AbortController().signal, logger: { debug() {}, info() {}, warn() {}, error() {} }, clock: { now: () => "2026-01-01T00:00:00.000Z" }, http: { request: async () => { throw new Error("not declared"); } }, credentials: { get: async () => { throw new Error("not declared"); } }, temporaryStorage: { read: async () => null, write: async () => undefined }, artifacts: { write: async (name: string, content: string | Uint8Array) => ({ name, digest: "fixture", size: typeof content === "string" ? content.length : content.byteLength }) } };\n assert.deepEqual(await run(context), { value: 1, executedAt: "2026-01-01T00:00:00.000Z" });\n});\n`, + "fixtures/basic.json": `${JSON.stringify({ input: { value: 1 }, config: {}, output: { value: 1, executedAt: "2026-01-01T00:00:00.000Z" } }, null, 2)}\n`, + "Dockerfile": createCustomNodeDockerfile(), + }; +} + +function lockfile(packageName: string): string { + return `lockfileVersion: '9.0'\n\nsettings:\n autoInstallPeers: false\n excludeLinksFromLockfile: false\n\nimporters:\n .:\n devDependencies:\n '@types/node':\n specifier: 25.6.0\n version: 25.6.0\n typescript:\n specifier: 5.9.3\n version: 5.9.3\n\npackages:\n '@types/node@25.6.0':\n resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}\n typescript@5.9.3:\n resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}\n engines: {node: '>=14.17'}\n hasBin: true\n undici-types@7.19.2:\n resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==}\n\nsnapshots:\n '@types/node@25.6.0':\n dependencies:\n undici-types: 7.19.2\n typescript@5.9.3: {}\n undici-types@7.19.2: {}\n# ${packageName}\n`; +} + +function sdkSource(): string { + return `export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };\nexport type JsonObject = { [key: string]: Json };\nexport interface NodeExecutionContext {\n readonly input: Readonly; readonly config: Readonly; readonly correlationId: string; readonly invocationId: string; readonly signal: AbortSignal;\n readonly logger: { debug(message: string, fields?: JsonObject): void; info(message: string, fields?: JsonObject): void; warn(message: string, fields?: JsonObject): void; error(message: string, fields?: JsonObject): void };\n readonly clock: { now(): string };\n readonly http: { request(request: { url: string; method?: string; headers?: Record; body?: string }): Promise<{ status: number; headers: Record; body: string }> };\n readonly credentials: { get(slot: string): Promise };\n readonly temporaryStorage: { read(path: string): Promise; write(path: string, content: string | Uint8Array): Promise };\n readonly artifacts: { write(name: string, content: string | Uint8Array, mediaType: string): Promise<{ name: string; digest: string; size: number }> };\n}\nexport type CustomNodeHandler = (context: NodeExecutionContext) => Promise;\n`; +} + +function runnerSource(): string { + return `import * as fs from "node:fs/promises";\nimport * as crypto from "node:crypto";\nimport * as path from "node:path";\nimport { run } from "./index.js";\nimport type { Json, JsonObject, NodeExecutionContext } from "./sdk.js";\nconst scratch = "/tmp/codeux"; const chunks: Buffer[] = []; for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));\nconst envelope = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { input: JsonObject; config: JsonObject; correlationId: string; invocationId: string; credentials: Record; now: string };\nconst secrets = Object.values(envelope.credentials); const redactText = (value: string): string => { let text = value; for (const secret of secrets) if (secret) text = text.split(secret).join("[REDACTED]"); return text; }; const redactJson = (value: Json): Json => typeof value === "string" ? redactText(value) : Array.isArray(value) ? value.map(redactJson) : value && typeof value === "object" ? Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactJson(entry)])) : value;\nconst log = (level: string, message: string, fields?: JsonObject): void => process.stderr.write(JSON.stringify({ level, message: redactText(message), fields: fields ? redactJson(fields) : undefined, correlationId: envelope.correlationId }) + "\\n");\nconst safePath = (value: string): string => { const target = path.resolve(scratch, value); if (target !== scratch && !target.startsWith(scratch + path.sep)) throw new Error("temporary storage path escaped scratch"); return target; };\nconst context: NodeExecutionContext = { input: Object.freeze(envelope.input), config: Object.freeze(envelope.config), correlationId: envelope.correlationId, invocationId: envelope.invocationId, signal: new AbortController().signal, logger: { debug: (m,f) => log("debug",m,f), info: (m,f) => log("info",m,f), warn: (m,f) => log("warn",m,f), error: (m,f) => log("error",m,f) }, clock: { now: () => envelope.now }, http: { request: async () => { throw new Error("HTTP requires the Code UX egress broker and is unavailable in a network-none runner"); } }, credentials: { get: async (slot) => { const value = envelope.credentials[slot]; if (!value) throw new Error("credential slot is unavailable"); return value; } }, temporaryStorage: { read: async (file) => fs.readFile(safePath(file)).catch(() => null), write: async (file, content) => { const target = safePath(file); await fs.mkdir(path.dirname(target), { recursive: true }); await fs.writeFile(target, content); } }, artifacts: { write: async (name, content) => { const target = safePath("artifacts/" + name); await fs.mkdir(path.dirname(target), { recursive: true }); await fs.writeFile(target, content); const bytes = typeof content === "string" ? Buffer.from(content) : content; return { name, digest: "sha256:" + crypto.createHash("sha256").update(bytes).digest("hex"), size: bytes.byteLength }; } } };\ntry { const output = await run(context); process.stdout.write(JSON.stringify(output)); } catch (error) { log("error", error instanceof Error ? error.message : String(error)); process.exitCode = 1; }\n`; +} + +export function createCustomNodeDockerfile(): string { + return `# syntax=docker/dockerfile:1.7\nFROM node:22-bookworm-slim AS build\nWORKDIR /build\nCOPY package.json pnpm-lock.yaml ./\nRUN corepack enable && pnpm install --frozen-lockfile --ignore-scripts\nCOPY tsconfig.json ./\nCOPY src ./src\nCOPY tests ./tests\nRUN --network=none pnpm run typecheck && pnpm run build && pnpm run test\n\nFROM gcr.io/distroless/nodejs22-debian12:nonroot\nWORKDIR /app\nCOPY --from=build --chown=65532:65532 /build/dist ./dist\nUSER 65532:65532\nENTRYPOINT ["/nodejs/bin/node", "/app/dist/src/runner.js"]\n`; +} diff --git a/src/services/custom-nodes/custom-node-runtime-service.ts b/src/services/custom-nodes/custom-node-runtime-service.ts new file mode 100644 index 0000000000..f184828106 --- /dev/null +++ b/src/services/custom-nodes/custom-node-runtime-service.ts @@ -0,0 +1,196 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { + CustomNodeArtifact, + CustomNodeExecutionRequest, + CustomNodeExecutionResult, + CustomNodeManifest, +} from "../../contracts/custom-node-types.js"; +import type { NodeFlowJsonObject, NodeFlowJsonValue, NodeFlowValueSchema } from "../../contracts/node-flow-types.js"; +import type { CustomNodeRepository } from "../../repositories/custom-node-repository.js"; +import { EntityNotFoundError, ValidationError } from "../../repositories/repository-utils.js"; +import { DOCKER_DROP_ALL_CAPS_ARGS, DOCKER_NETWORK_NONE_ARGS, DOCKER_NO_NEW_PRIVILEGES_ARGS } from "../cli-docker-utils.js"; +import { runCommandStrict, type CommandResult } from "../cli-process-runner.js"; +import type { CredentialBroker } from "../credentials/credential-broker.js"; +import type { EgressPolicyService } from "../node-flows/egress-policy-service.js"; + +export const CUSTOM_NODE_CONTAINER_SCRATCH = "/tmp/codeux"; + +export interface CustomNodeDockerPlanInput { + artifact: CustomNodeArtifact; + containerName: string; + seccompProfile?: string; + appArmorProfile?: string; +} + +export function buildCustomNodeDockerRunArgs(input: CustomNodeDockerPlanInput): string[] { + const limits = input.artifact.manifest.resources; + const args = [ + "run", "--rm", "-i", "--name", input.containerName, + ...DOCKER_NETWORK_NONE_ARGS, + ...DOCKER_NO_NEW_PRIVILEGES_ARGS, + ...DOCKER_DROP_ALL_CAPS_ARGS, + "--read-only", + "--user", "65532:65532", + "--pids-limit", String(limits.pids), + "--memory", `${limits.memoryMb}m`, + "--memory-swap", `${limits.memoryMb}m`, + "--cpus", String(limits.cpu), + "--tmpfs", `${CUSTOM_NODE_CONTAINER_SCRATCH}:rw,nosuid,nodev,noexec,size=${limits.scratchMb}m,mode=700,uid=65532,gid=65532`, + "--log-driver", "none", + "--label", "code-ux.managed=true", + "--label", "code-ux.custom-node=true", + "--label", `code-ux.custom-node-digest=${input.artifact.digest}`, + ]; + if (input.seccompProfile) args.push("--security-opt", `seccomp=${input.seccompProfile}`); + if (input.appArmorProfile) args.push("--security-opt", `apparmor=${input.appArmorProfile}`); + args.push(input.artifact.runtimeImageDigest); + return args; +} + +export interface CustomNodeRuntimeServiceDeps { + repository: CustomNodeRepository; + credentialBroker: CredentialBroker; + egressPolicyService: EgressPolicyService; + featureEnabled?: boolean; + commandRunner?: (command: string, args: string[], cwd: string, options: { signal?: AbortSignal; timeout: number; maxStdoutChars: number; stdinFile?: string }) => Promise; + seccompProfile?: string; + appArmorProfile?: string; +} + +export class CustomNodeRuntimeService { + constructor(private readonly deps: CustomNodeRuntimeServiceDeps) {} + + async execute(request: CustomNodeExecutionRequest): Promise { + if (!(this.deps.featureEnabled ?? process.env.CODE_UX_CUSTOM_NODES_ENABLED === "true")) { + throw new ValidationError("Custom node execution is disabled by the custom-node feature gate."); + } + const resolved = this.deps.repository.resolvePublished(request.nodeType, request.version); + if (!resolved || resolved.publication.projectId !== request.projectId) { + throw new EntityNotFoundError(`Published custom node not found: ${request.nodeType}@${request.version}`); + } + const { artifact } = resolved; + await this.validateDeclaredEgress(artifact.manifest); + const credentials = await this.resolveCredentials(artifact.manifest, request); + const secrets = Object.values(credentials); + const runDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-custom-node-run-")); + try { + if (process.platform !== "win32") await fs.chmod(runDirectory, 0o700); + const envelope = { + input: request.input, + config: request.config, + correlationId: request.correlationId, + invocationId: request.invocationId, + credentials, + now: new Date().toISOString(), + }; + const inputPath = path.join(runDirectory, "input.json"); + await fs.writeFile(inputPath, JSON.stringify(envelope), { mode: 0o600 }); + if (process.platform !== "win32") await fs.chmod(inputPath, 0o600); + const dockerArgs = buildCustomNodeDockerRunArgs({ + artifact, + containerName: containerName(request.invocationId), + seccompProfile: this.deps.seccompProfile, + appArmorProfile: this.deps.appArmorProfile, + }); + const runner = this.deps.commandRunner ?? (async (command, args, cwd, options) => runCommandStrict(command, args, cwd, process.env, options)); + const result = await runner("docker", dockerArgs, process.cwd(), { + signal: request.signal, + timeout: artifact.manifest.resources.timeoutMs, + maxStdoutChars: artifact.manifest.resources.maxOutputBytes, + stdinFile: inputPath, + }); + const rawOutput = result.stdout; + if (Buffer.byteLength(rawOutput) > artifact.manifest.resources.maxOutputBytes) throw new ValidationError("Custom node output exceeded its declared limit."); + let output: unknown; + try { output = JSON.parse(rawOutput); } catch { throw new ValidationError("Custom node did not produce valid JSON output."); } + const schemaIssues = validateValueAgainstSchema(output, artifact.manifest.outputSchema, "output"); + if (schemaIssues.length > 0) throw new ValidationError(`Custom node output schema validation failed: ${schemaIssues[0]}`); + const safeOutput = redactJson(output as NodeFlowJsonValue, secrets); + if (!isObject(safeOutput)) throw new ValidationError("Custom node output must be a JSON object."); + return { + output: safeOutput, + artifactDigest: artifact.digest, + logs: redactSecrets(result.stderr, secrets), + diagnostics: JSON.stringify(safeOutput), + }; + } finally { + for (const key of Object.keys(credentials)) credentials[key] = ""; + await fs.rm(runDirectory, { recursive: true, force: true }); + } + } + + private async resolveCredentials(manifest: CustomNodeManifest, request: CustomNodeExecutionRequest): Promise> { + const resolved: Record = {}; + for (const slot of manifest.credentials) { + const bindingKey = request.credentialBindings[slot.slot]; + if (!bindingKey) { + if (slot.required) throw new ValidationError(`Required custom node credential slot is not bound: ${slot.slot}`); + continue; + } + const credential = await this.deps.credentialBroker.resolveCredentialId({ + projectId: request.projectId, + bindingKey, + credentialId: bindingKey, + capability: slot.requiredCapability, + workspaceId: request.workspaceId, + }); + resolved[slot.slot] = credential.value; + } + return resolved; + } + + private async validateDeclaredEgress(manifest: CustomNodeManifest): Promise { + if (!manifest.capabilities.includes("network.http")) return; + if (!manifest.http) throw new ValidationError("Published custom node is missing its bounded HTTP policy."); + for (const host of manifest.http.allowedHosts) { + await this.deps.egressPolicyService.validateUrl(`https://${host}`, { + allowedHosts: manifest.http.allowedHosts, + allowedPorts: manifest.http.allowedPorts ?? [443], + timeoutMs: manifest.http.timeoutMs, + maxResponseBytes: manifest.http.maxResponseBytes, + requestsPerMinute: manifest.http.maxRequests, + }); + } + } +} + +function containerName(invocationId: string): string { + return `code-ux-custom-node-${invocationId.toLowerCase().replace(/[^a-z0-9_.-]/g, "-").slice(0, 48)}`; +} + +export function validateValueAgainstSchema(value: unknown, schema: NodeFlowValueSchema, field: string): string[] { + if (schema.type === "any") return []; + if (schema.type === "null") return value === null ? [] : [`${field} must be null`]; + if (schema.type === "array") { + if (!Array.isArray(value)) return [`${field} must be an array`]; + return schema.items ? value.flatMap((item, index) => validateValueAgainstSchema(item, schema.items!, `${field}[${index}]`)) : []; + } + if (schema.type === "object") { + if (!isObject(value)) return [`${field} must be an object`]; + const issues = (schema.required ?? []).filter((key) => !(key in value)).map((key) => `${field}.${key} is required`); + for (const [key, child] of Object.entries(schema.properties ?? {})) { + if (key in value) issues.push(...validateValueAgainstSchema(value[key], child, `${field}.${key}`)); + } + return issues; + } + return typeof value === schema.type ? [] : [`${field} must be ${schema.type}`]; +} + +export function redactSecrets(value: string, secrets: readonly string[]): string { + let redacted = value; + for (const secret of secrets) if (secret) redacted = redacted.split(secret).join("[REDACTED]"); + return redacted; +} + +function redactJson(value: NodeFlowJsonValue, secrets: readonly string[]): NodeFlowJsonValue { + if (typeof value === "string") return redactSecrets(value, secrets); + if (Array.isArray(value)) return value.map((entry) => redactJson(entry, secrets)); + if (isObject(value)) return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactJson(entry as NodeFlowJsonValue, secrets)])); + return value; +} + +function isObject(value: unknown): value is NodeFlowJsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index a3f662e047..72f7667521 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -20,6 +20,7 @@ import { BuiltinExecutors } from "./node-flows/builtins/builtin-executors.js"; import type { ApprovalService } from "./node-flows/approval-service.js"; import { ApprovalRequiredError } from "./node-flows/approval-service.js"; import type { OutboxService } from "./node-flows/outbox-service.js"; +import type { CustomNodeRuntimeService } from "./custom-nodes/custom-node-runtime-service.js"; import type { NodeFlowFailureClassification } from "../contracts/node-flow-execution-policy-types.js"; import { buildProviderInvocationWorkspaceOptions } from "../infrastructure/providers/cli/invocation-workspace-preparer.js"; import type { @@ -55,6 +56,7 @@ interface NodeFlowRuntimeDeps { egressPolicyService?: EgressPolicyService; approvalService?: ApprovalService; outboxService?: OutboxService; + customNodeRuntimeService?: CustomNodeRuntimeService; } interface RuntimeContext { @@ -362,7 +364,9 @@ export class NodeFlowRuntimeService { node: NodeFlowNode, nodeRun: NodeFlowNodeRunRecord, ): Promise { - if (EXTERNALLY_OBSERVABLE_NODE_TYPES.has(node.type)) { + const reference = node.definition ?? { type: node.type, version: 1 }; + const definition = resolveNodeDefinition(reference.type, reference.version); + if (EXTERNALLY_OBSERVABLE_NODE_TYPES.has(node.type) || definition?.executionKind === "custom") { const invocation = this.deps.executionRepository.createExecutionInvocation({ projectId: context.projectId, skipValidation: true, @@ -375,9 +379,11 @@ export class NodeFlowRuntimeService { this.deps.nodeFlowRepository.updateNodeAttempt(context.currentAttemptId, { invocationId: invocation.id }); } try { - const result = node.type === "provider_prompt" - ? await this.executeProviderPromptNode(context, node, invocation.id) - : await this.executeHttpRequestNode(context, node, invocation.id); + const result = definition?.executionKind === "custom" + ? await this.executeCustomNode(context, node, invocation.id, reference.type, reference.version) + : node.type === "provider_prompt" + ? await this.executeProviderPromptNode(context, node, invocation.id) + : await this.executeHttpRequestNode(context, node, invocation.id); this.deps.executionRepository.updateExecutionInvocation(invocation.id, { status: "completed", finishedAt: new Date().toISOString(), @@ -419,6 +425,37 @@ export class NodeFlowRuntimeService { } } + private async executeCustomNode( + context: RuntimeContext, + node: NodeFlowNode, + invocationId: string, + nodeType: string, + version: number, + ): Promise { + if (!this.deps.customNodeRuntimeService) throw new ValidationError("Custom node runtime service is not configured."); + const result = await this.deps.customNodeRuntimeService.execute({ + projectId: context.projectId, + nodeType, + version, + input: this.buildNodeInput(context, node.id), + config: evaluateTemplates(readNodeConfig(node), context) as NodeFlowJsonObject, + credentialBindings: Object.fromEntries((node.credentialBindings ?? []).map((binding) => [binding.slot, binding.credentialId])), + workspaceId: context.runId, + invocationId, + correlationId: context.runId, + signal: context.options.signal, + }); + if (context.currentAttemptId) { + this.deps.nodeFlowRepository.updateNodeAttempt(context.currentAttemptId, { artifactDigest: result.artifactDigest }); + } + this.deps.executionRepository.appendExecutionInvocationMessage(invocationId, { + role: "assistant", + contentMarkdown: "Custom node container execution completed.", + metadata: { flowId: context.flowId, runId: context.runId, nodeId: node.id, artifactDigest: result.artifactDigest }, + }); + return { output: result.output }; + } + private executeSetFieldsNode(context: RuntimeContext, node: NodeFlowNode): NodeFlowJsonObject { const config = readNodeConfig(node); const base = readBoolean(config.replace) ? {} : this.firstUpstreamObject(context, node.id); diff --git a/tests/backend/repositories/custom-node-repository.test.ts b/tests/backend/repositories/custom-node-repository.test.ts new file mode 100644 index 0000000000..a23ab73b44 --- /dev/null +++ b/tests/backend/repositories/custom-node-repository.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import type { CustomNodeArtifact, CustomNodeManifest, CustomNodeValidationReport } from "../../../src/contracts/custom-node-types.js"; +import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; +import { CustomNodeRepository } from "../../../src/repositories/custom-node-repository.js"; +import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; + +const manifest = (id = "safe-node"): CustomNodeManifest => ({ + schemaVersion: 1, id, nodeType: `custom.${id}`, version: 1, name: "Safe node", description: "", + entrypoint: "dist/index.js", inputSchema: { type: "object" }, outputSchema: { type: "object" }, + configurationSchema: { type: "object" }, capabilities: ["clock.read"], credentials: [], + resources: { cpu: 0.5, memoryMb: 128, pids: 64, timeoutMs: 30_000, maxOutputBytes: 262_144, scratchMb: 32 }, +}); + +describe("CustomNodeRepository", () => { + it("enforces draft, validation, immutable artifact, and publication lifecycle", () => { + const storage = new AppDbStorage(":memory:"); + const project = new ProjectManagementRepository(storage).createProject({ name: "Custom Node Test", sourceType: "local", sourceRef: "/tmp/custom-node-test" }); + const repository = new CustomNodeRepository(storage); + const nodeManifest = manifest(); + const draft = repository.createDraft(project.id, { manifest: nodeManifest, sourceRevision: "revision-1", createdBy: "test" }); + expect(draft.status).toBe("draft"); + repository.beginValidation(draft.id); + const report: CustomNodeValidationReport = { valid: true, checks: [{ name: "all", passed: true, durationMs: 1 }], issues: [], validatedAt: new Date().toISOString() }; + const artifact: CustomNodeArtifact = { + digest: `sha256:${"a".repeat(64)}`, nodeId: draft.id, projectId: project.id, version: 1, + sourceRevision: "revision-1", buildDigest: `sha256:${"b".repeat(64)}`, + runtimeImageDigest: `sha256:${"c".repeat(64)}`, dependencies: [], validationReport: report, + createdBy: "test", invocationId: "invocation", correlationId: "correlation", + capabilities: ["clock.read"], manifest: nodeManifest, createdAt: new Date().toISOString(), + }; + expect(repository.completeValidation(draft.id, report, artifact).status).toBe("passed"); + const publication = repository.publish(draft.id, "publisher"); + expect(publication).toMatchObject({ nodeType: "custom.safe-node", version: 1, artifactDigest: artifact.digest }); + expect(repository.resolvePublished("custom.safe-node", 1)?.artifact).toEqual(artifact); + expect(repository.getNode(draft.id)?.status).toBe("published"); + expect(() => repository.beginValidation(draft.id)).toThrow(/immutable/i); + storage.close(); + }); + + it("rejects publication after failed validation", () => { + const storage = new AppDbStorage(":memory:"); + const project = new ProjectManagementRepository(storage).createProject({ name: "Failed Node Test", sourceType: "local", sourceRef: "/tmp/failed-node-test" }); + const repository = new CustomNodeRepository(storage); + repository.createDraft(project.id, { manifest: manifest("failed-node"), sourceRevision: "revision-1", createdBy: "test" }); + repository.beginValidation("failed-node"); + repository.completeValidation("failed-node", { valid: false, checks: [], issues: [{ check: "scan", code: "failed", message: "failed" }], validatedAt: new Date().toISOString() }, null); + expect(() => repository.publish("failed-node", "publisher")).toThrow(/passed/i); + storage.close(); + }); +}); diff --git a/tests/backend/services/custom-node-services.test.ts b/tests/backend/services/custom-node-services.test.ts new file mode 100644 index 0000000000..3443ed9751 --- /dev/null +++ b/tests/backend/services/custom-node-services.test.ts @@ -0,0 +1,150 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { CustomNodeArtifact, CustomNodeManifest, CustomNodeValidationReport } from "../../../src/contracts/custom-node-types.js"; +import { resolveNodeDefinition } from "../../../src/domain/node-flows/node-definition-registry.js"; +import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; +import { CustomNodeRepository } from "../../../src/repositories/custom-node-repository.js"; +import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; +import { CustomNodeBuildService } from "../../../src/services/custom-nodes/custom-node-build-service.js"; +import { CustomNodeProjectService } from "../../../src/services/custom-nodes/custom-node-project-service.js"; +import { buildCustomNodeDockerRunArgs, CustomNodeRuntimeService } from "../../../src/services/custom-nodes/custom-node-runtime-service.js"; +import type { CommandResult } from "../../../src/services/cli-process-runner.js"; + +const temporaryDirectories: string[] = []; +const result = (stdout = "", stderr = ""): CommandResult => ({ ok: true, code: 0, stdout, stderr }); + +async function fixture(nodeId: string): Promise<{ root: string; storage: AppDbStorage; repository: CustomNodeRepository; projectId: string; projectService: CustomNodeProjectService }> { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-custom-node-test-")); + temporaryDirectories.push(root); + const storage = new AppDbStorage(path.join(root, "app.db")); + const project = new ProjectManagementRepository(storage).createProject({ name: "Custom Node Service Test", sourceType: "local", sourceRef: root }); + const repository = new CustomNodeRepository(storage); + const projectService = new CustomNodeProjectService(); + const generated = await projectService.generate({ projectRoot: root, nodeId, name: "Safe node" }); + repository.createDraft(project.id, { manifest: generated.manifest, sourceRevision: "revision-1", createdBy: "test" }); + return { root, storage, repository, projectId: project.id, projectService }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); +}); + +describe("custom node project and build services", () => { + it("generates the SDK package and validates/builds/publishes a deterministic fixture through Docker", async () => { + const setup = await fixture("fixture-node"); + const calls: string[][] = []; + const commandRunner = vi.fn(async (_command: string, args: string[]): Promise => { + calls.push(args); + if (args[0] === "image") return result(`sha256:${"d".repeat(64)}\n`); + if (args[0] === "run") return result(JSON.stringify({ value: 1, executedAt: "2026-01-01T00:00:00.000Z" })); + return result(); + }); + const service = new CustomNodeBuildService({ repository: setup.repository, projectService: setup.projectService, commandRunner, vulnerabilityAudit: async () => ({ passed: true, details: "fixture audit passed" }) }); + const validation = await service.validateAndBuild({ projectRoot: setup.root, nodeId: "fixture-node", creator: "test", invocationId: "invocation", correlationId: "correlation" }); + + expect(validation.report.valid).toBe(true); + expect(validation.artifact).toMatchObject({ sourceRevision: "revision-1", runtimeImageDigest: `sha256:${"d".repeat(64)}`, capabilities: ["clock.read"] }); + expect(calls.some((args) => args[0] === "build" && args.includes("--pull=false"))).toBe(true); + await expect(fs.readFile(path.join(setup.root, ".code-ux", "nodes", "fixture-node", "Dockerfile"), "utf8")).resolves.toContain("RUN --network=none pnpm run typecheck"); + expect(calls.some((args) => args[0] === "run" && args.includes("--read-only"))).toBe(true); + const artifact = service.publish("fixture-node", "publisher"); + expect(resolveNodeDefinition("custom.fixture-node", 1)?.executionKind).toBe("custom"); + expect(artifact.digest).toBe(validation.artifact?.digest); + await expect(fs.readFile(path.join(setup.root, ".code-ux", "nodes", "fixture-node", "src", "sdk.ts"), "utf8")).resolves.toContain("NodeExecutionContext"); + setup.storage.close(); + }); + + it.each([ + ["host filesystem", 'import fs from "node:fs";'], + ["host environment", "const value = process.env.SECRET;"], + ["subprocess", 'import { spawn } from "node:child_process";'], + ["Docker", 'const socket = "/var/run/docker.sock";'], + ["unrestricted network", 'const response = await fetch("https://example.test");'], + ])("fails closed when generated code requests %s access", async (_name, attack) => { + const nodeId = `blocked-${temporaryDirectories.length + 10}`; + const setup = await fixture(nodeId); + await fs.writeFile(path.join(setup.root, ".code-ux", "nodes", nodeId, "src", "index.ts"), `${attack}\nexport const run = async () => ({});\n`); + const commandRunner = vi.fn(async (): Promise => result()); + const validation = await new CustomNodeBuildService({ repository: setup.repository, projectService: setup.projectService, commandRunner }) + .validateAndBuild({ projectRoot: setup.root, nodeId, creator: "test", invocationId: "invocation", correlationId: "correlation" }); + expect(validation.report.valid).toBe(false); + expect(validation.report.issues.some((issue) => issue.check === "prohibited-api-scan")).toBe(true); + expect(commandRunner).not.toHaveBeenCalled(); + setup.storage.close(); + }); + + it("rejects resource exhaustion declarations before Docker execution", async () => { + const setup = await fixture("resource-node"); + const manifestPath = path.join(setup.root, ".code-ux", "nodes", "resource-node", "node.json"); + const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")) as CustomNodeManifest; + manifest.resources.memoryMb = 100_000; + await fs.writeFile(manifestPath, JSON.stringify(manifest)); + const commandRunner = vi.fn(async (): Promise => result()); + const validation = await new CustomNodeBuildService({ repository: setup.repository, commandRunner }) + .validateAndBuild({ projectRoot: setup.root, nodeId: "resource-node", creator: "test", invocationId: "invocation", correlationId: "correlation" }); + expect(validation.report.issues.some((issue) => issue.check === "resource-policy")).toBe(true); + expect(commandRunner).not.toHaveBeenCalled(); + setup.storage.close(); + }); +}); + +describe("custom node Docker plan and runtime security", () => { + it("applies a non-root read-only network-none container with bounded resources and no project or Docker mounts", () => { + const artifact = artifactFixture("plan-node", "/immutable-image"); + const args = buildCustomNodeDockerRunArgs({ artifact, containerName: "custom-run", seccompProfile: "/profiles/custom.json", appArmorProfile: "code-ux-custom-node" }); + expect(args).toEqual(expect.arrayContaining(["--network", "none", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", "--read-only", "--user", "65532:65532", "--pids-limit", "64", "--memory", "128m", "--cpus", "0.5"])); + expect(args).toContain("seccomp=/profiles/custom.json"); + expect(args).toContain("apparmor=code-ux-custom-node"); + expect(args.join(" ")).not.toMatch(/docker\.sock|--network host|\/workspace|\/project/); + expect(args.filter((arg) => arg.startsWith("type="))).toEqual([]); + }); + + it("uses the credential broker, isolates each run, and redacts canaries from outputs, logs, and diagnostics", async () => { + const setup = await fixture("runtime-node"); + const artifact = artifactFixture("runtime-node", `sha256:${"e".repeat(64)}`, setup.projectId); + artifact.manifest.capabilities = ["credentials.read"]; + artifact.manifest.credentials = [{ slot: "api", label: "API", required: true, allowedKinds: ["http"], requiredCapability: "read" }]; + setup.repository.beginValidation("runtime-node"); + setup.repository.completeValidation("runtime-node", artifact.validationReport, artifact); + setup.repository.publish("runtime-node", "publisher"); + const secret = "CODEUX_SECRET_CANARY_DO_NOT_LEAK"; + const runDirectories: string[] = []; + const commandRunner = vi.fn(async (_command: string, _args: string[], _cwd: string, options: { stdinFile?: string }): Promise => { + if (!options.stdinFile) throw new Error("missing credential bundle"); + runDirectories.push(path.dirname(options.stdinFile)); + return result(JSON.stringify({ value: secret }), `log ${secret}`); + }); + const broker = { resolveCredentialId: vi.fn(async () => ({ credentialId: "credential", value: secret, version: 1 })) }; + const runtime = new CustomNodeRuntimeService({ repository: setup.repository, credentialBroker: broker as never, egressPolicyService: {} as never, featureEnabled: true, commandRunner }); + const execute = () => runtime.execute({ projectId: setup.projectId, nodeType: "custom.runtime-node", version: 1, input: {}, config: {}, credentialBindings: { api: "credential" }, workspaceId: "run", invocationId: `invocation-${runDirectories.length}`, correlationId: "correlation" }); + const first = await execute(); + const second = await execute(); + expect(first).toMatchObject({ output: { value: "[REDACTED]" }, logs: "log [REDACTED]", diagnostics: "{\"value\":\"[REDACTED]\"}" }); + expect(second.output).toEqual({ value: "[REDACTED]" }); + expect(runDirectories[0]).not.toBe(runDirectories[1]); + await expect(fs.access(runDirectories[0]!)).rejects.toThrow(); + await expect(fs.access(runDirectories[1]!)).rejects.toThrow(); + expect(broker.resolveCredentialId).toHaveBeenCalledTimes(2); + setup.storage.close(); + }); + + it("keeps execution behind the feature and publication gates", async () => { + const setup = await fixture("gated-node"); + const runtime = new CustomNodeRuntimeService({ repository: setup.repository, credentialBroker: {} as never, egressPolicyService: {} as never, featureEnabled: false }); + await expect(runtime.execute({ projectId: setup.projectId, nodeType: "custom.gated-node", version: 1, input: {}, config: {}, credentialBindings: {}, workspaceId: "run", invocationId: "invocation", correlationId: "correlation" })).rejects.toThrow(/feature gate/i); + setup.storage.close(); + }); +}); + +function artifactFixture(nodeId: string, runtimeImageDigest: string, projectId = "project"): CustomNodeArtifact { + const generatedManifest: CustomNodeManifest = { + schemaVersion: 1, id: nodeId, nodeType: `custom.${nodeId}`, version: 1, name: "Test", description: "", + entrypoint: "dist/index.js", inputSchema: { type: "object" }, outputSchema: { type: "object" }, configurationSchema: { type: "object" }, + capabilities: [], credentials: [], resources: { cpu: 0.5, memoryMb: 128, pids: 64, timeoutMs: 30_000, maxOutputBytes: 262_144, scratchMb: 32 }, + }; + const report: CustomNodeValidationReport = { valid: true, checks: [], issues: [], validatedAt: new Date().toISOString() }; + return { digest: `sha256:${"a".repeat(64)}`, nodeId, projectId, version: 1, sourceRevision: "revision-1", buildDigest: `sha256:${"b".repeat(64)}`, runtimeImageDigest, dependencies: [], validationReport: report, createdBy: "test", invocationId: "invocation", correlationId: "correlation", capabilities: [], manifest: generatedManifest, createdAt: new Date().toISOString() }; +} From aaa85a0bea92d77af3db2eb6617d4f3d57ca5c85 Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 06:11:55 +0000 Subject: [PATCH 06/25] feat(task T06): implement via codex --- .../docs/developer-management-actions.mdx | 15 + docs-web/content/docs/developer-mcp-tools.mdx | 13 +- docs-web/developer/management-actions.md | 15 + docs-web/developer/mcp-tools.md | 13 +- docs/mcp/runtime-and-dispatch.md | 9 +- docs/mcp/tools-and-contracts.md | 8 +- src/api/mcp/tool-registry.ts | 3 +- src/app/dependency-factory/core-factory.ts | 2 +- .../dependency-factory/dashboard-factory.ts | 15 +- src/contracts/internal-management-types.ts | 25 +- src/contracts/mcp-tool-definitions.ts | 37 ++- src/contracts/node-flow-types.ts | 59 ++++ src/mcp/management-tool-handler.ts | 27 +- src/mcp/management/node-flow-actions.ts | 163 ++++++++++- src/repositories/custom-node-repository.ts | 10 + src/repositories/node-flow-repository.ts | 12 +- src/server/code-ux-server.ts | 4 + src/server/mcp-request-router.ts | 1 + src/server/node-flow-routes.ts | 102 +++++++ src/services/agent-mcp-access.ts | 12 + .../custom-node-project-service.ts | 6 + src/services/node-flow-agent-skill-service.ts | 57 ++++ src/services/node-flow-runtime-service.ts | 6 +- src/services/node-flow-service.ts | 272 ++++++++++++++++++ .../mcp/management-node-flow-actions.test.ts | 47 +++ tests/backend/mcp/mcp-management.test.ts | 28 +- tests/backend/server/node-flow-routes.test.ts | 10 + .../backend/services/agent-mcp-access.test.ts | 10 + .../node-flow-agent-skill-service.test.ts | 33 +++ .../services/node-flow-service.test.ts | 35 +++ 30 files changed, 1006 insertions(+), 43 deletions(-) create mode 100644 src/services/node-flow-agent-skill-service.ts create mode 100644 tests/backend/services/node-flow-agent-skill-service.test.ts diff --git a/docs-web/content/docs/developer-management-actions.mdx b/docs-web/content/docs/developer-management-actions.mdx index aec827ebfd..8bf6fe17e6 100644 --- a/docs-web/content/docs/developer-management-actions.mdx +++ b/docs-web/content/docs/developer-management-actions.mdx @@ -113,6 +113,21 @@ Task create/update fields include `title`, `name`, `promptMarkdown`, `descriptio --- +## `node_flows` + +| Action group | Approval | Behavior | +| --- | --- | --- | +| Catalog | – | `catalog` and `get_node_definition` return executable manifests and schemas without complete flow graphs. | +| Drafts | – | `create_draft`, `patch_draft`, and `validate_draft` return validation, policy, credential, capability, and side-effect summaries. `patch_draft` requires `draftRevision`. | +| Custom nodes | – | `create_custom_node`, `update_custom_node`, and `validate_custom_node` reuse the governed project/build services. | +| Credentials | – | `request_credential` and `inspect_bindings` expose metadata and permission findings only. | +| Review | – / ✅ | `dry_run` is side-effect free; `publish` and `rollback` require exact-payload approval; `compare_versions` returns structural summaries. | +| Operations | – | `run`, `cancel`, `retry`, `inspect_run`, and `list_runs` enforce project ownership and publication selection. | + +Compatibility aliases remain for legacy list/get/create/update/delete/validate/run/attach/detach calls. Legacy create/update auto-publish for backward compatibility; governed draft actions do not. Attachments grant the agent only `run_attached_flow`, never the full management surface. + +--- + ## `scheduler` | Action | Destructive | Required payload | Description | diff --git a/docs-web/content/docs/developer-mcp-tools.mdx b/docs-web/content/docs/developer-mcp-tools.mdx index cde7ce28d4..785ad691b6 100644 --- a/docs-web/content/docs/developer-mcp-tools.mdx +++ b/docs-web/content/docs/developer-mcp-tools.mdx @@ -54,7 +54,8 @@ action-specific fields, and an optional `approval` object for destructive action | `manage_scheduler` | orchestration | Create and run scheduled sprints, quicksprints, messages, and node flows. | | `scheduler_code_ux` | orchestration | Agent-owned wakeups with restricted list/schedule/cancel actions. | | `manage_agents` | agents & memory | Manage agent presets and sync them to project markdown. | -| `manage_node_flows` | agents & memory | Manage reusable node workflows, run them, and attach them as agent skills. | +| `manage_node_flows` | agents & memory | Govern draft automation graphs, credentials, publication, versions, and runs. | +| `run_attached_flow` | agents & memory | Run one published flow attached to the authenticated agent without exposing its graph or credentials. | | `manage_memory` | agents & memory | Inspect, search, promote, and re-embed short/long-term memory. | | `add_long_term_memory` | agents & memory | Store one canonical durable project memory and return rich confirmation-widget data. | | `manage_skills` | agents & memory | Manage persistent skill storages, skill markdown, and agent storage attachments. | @@ -126,7 +127,7 @@ Clarification states are `pending`, `replied`, `expired`, and `cancelled`. Repea | `manage_scheduler` | `list`, `create`, `update`, `delete`, `run_due`, `schedule_sprint`, `schedule_quicksprint`, `schedule_chat`, `schedule_node_flow` | | `scheduler_code_ux` | `list`, `schedule_wakeup`, `cancel` | | `manage_agents` | `list`, `get`, `create`, `update`, `delete`, `sync` | -| `manage_node_flows` | `list`, `get`, `create`, `update`, `delete`, `validate`, `run`, `list_runs`, `get_run`, `attach_to_agent`, `detach_from_agent` | +| `manage_node_flows` | `catalog`, `get_node_definition`, `create_draft`, `patch_draft`, `validate_draft`, custom-node and credential actions, `dry_run`, `publish`, `compare_versions`, `rollback`, `run`, `cancel`, `retry`, `inspect_run`, plus compatibility aliases | | `manage_memory` | `list`, `get`, `count`, `create`, `update`, `delete`, `search`, `promote`, `get_map`, `model_status`, `start_reembed` | | `manage_skills` | `authoring_prompt`, `list_storages`, `get_storage`, `create_storage`, `update_storage`, `delete_storage`, `reset_storage`, `list_agent_storages`, `attach_storage`, `detach_storage`, `list_skills`, `get_skill`, `create_skill`, `update_skill`, `delete_skill`, `import_markdown`, `export_markdown` | | `manage_settings` | `get_system`, `get_project_override`, `resolve_project_effective`, `get_sprint_override`, `resolve_sprint_effective`, `replace_system_settings`, `patch_system_setting`, `replace_project_settings`, `patch_project_setting`, `reset_project_settings`, `replace_sprint_settings`, `patch_sprint_setting`, `reset_sprint_settings`, `export_settings_bundle`, `apply_settings_bundle` | @@ -191,13 +192,9 @@ memory remediation, or global scheduler destructive controls. ## Node flows -`manage_node_flows` exposes project node workflows through MCP. It supports graph validation, CRUD, -runtime execution, run inspection, and flow-backed agent skill attachments. +`manage_node_flows` exposes governed project automation authoring through MCP. Draft patches require an optimistic `draftRevision`; conflicts return expected/actual revisions without writes. Validation and dry-run responses contain policy findings, required credentials, requested capabilities, side-effect diffs, and redacted summaries. Publication and rollback require exact-payload approval. Legacy CRUD/run/attach aliases remain compatible. -Create and update calls validate the structured graph before repository writes. `run` delegates to the -node-flow runtime through `NodeFlowService.runFlow`, and `delete` requires the normal approval -handshake. Responses mask secret-shaped graph data, inputs, and outputs before returning them to MCP -clients. +Attached flows appear to the owning authenticated agent only as name, description, input schema, flow id, and the `run_attached_flow` operation. Calls enforce project ownership, attachment, publication, and credential policy and record agent/conversation provenance without exposing complete graphs or credential material. Agents should build Code UX-adapted node flows rather than cloning n8n workflows one-to-one. Graphs should include dynamic widget schemas for editable graph inputs and node fields; callers can provide diff --git a/docs-web/developer/management-actions.md b/docs-web/developer/management-actions.md index 2b6a6ff6b9..4ee6316e49 100644 --- a/docs-web/developer/management-actions.md +++ b/docs-web/developer/management-actions.md @@ -113,6 +113,21 @@ Task create/update fields include `title`, `name`, `promptMarkdown`, `descriptio --- +## `node_flows` + +| Action group | Approval | Behavior | +| --- | --- | --- | +| Catalog | – | `catalog` and `get_node_definition` return executable manifests and schemas without complete flow graphs. | +| Drafts | – | `create_draft`, `patch_draft`, and `validate_draft` return validation, policy, credential, capability, and side-effect summaries. `patch_draft` requires `draftRevision`. | +| Custom nodes | – | `create_custom_node`, `update_custom_node`, and `validate_custom_node` reuse the governed project/build services. | +| Credentials | – | `request_credential` and `inspect_bindings` expose metadata and permission findings only. | +| Review | – / ✅ | `dry_run` is side-effect free; `publish` and `rollback` require exact-payload approval; `compare_versions` returns structural summaries. | +| Operations | – | `run`, `cancel`, `retry`, `inspect_run`, and `list_runs` enforce project ownership and publication selection. | + +Compatibility aliases remain for legacy list/get/create/update/delete/validate/run/attach/detach calls. Legacy create/update auto-publish for backward compatibility; governed draft actions do not. Attachments grant the agent only `run_attached_flow`, never the full management surface. + +--- + ## `scheduler` | Action | Destructive | Required payload | Description | diff --git a/docs-web/developer/mcp-tools.md b/docs-web/developer/mcp-tools.md index bd15393af1..7633d6ac16 100644 --- a/docs-web/developer/mcp-tools.md +++ b/docs-web/developer/mcp-tools.md @@ -54,7 +54,8 @@ action-specific fields, and an optional `approval` object for destructive action | `manage_scheduler` | orchestration | Create and run scheduled sprints, quicksprints, messages, and node flows. | | `scheduler_code_ux` | orchestration | Agent-owned wakeups with restricted list/schedule/cancel actions. | | `manage_agents` | agents & memory | Manage agent presets and sync them to project markdown. | -| `manage_node_flows` | agents & memory | Manage reusable node workflows, run them, and attach them as agent skills. | +| `manage_node_flows` | agents & memory | Govern draft automation graphs, credentials, publication, versions, and runs. | +| `run_attached_flow` | agents & memory | Run one published flow attached to the authenticated agent without exposing its graph or credentials. | | `manage_memory` | agents & memory | Inspect, search, promote, and re-embed short/long-term memory. | | `add_long_term_memory` | agents & memory | Store one canonical durable project memory and return rich confirmation-widget data. | | `manage_skills` | agents & memory | Manage persistent skill storages, skill markdown, and agent storage attachments. | @@ -126,7 +127,7 @@ Clarification states are `pending`, `replied`, `expired`, and `cancelled`. Repea | `manage_scheduler` | `list`, `create`, `update`, `delete`, `run_due`, `schedule_sprint`, `schedule_quicksprint`, `schedule_chat`, `schedule_node_flow` | | `scheduler_code_ux` | `list`, `schedule_wakeup`, `cancel` | | `manage_agents` | `list`, `get`, `create`, `update`, `delete`, `sync` | -| `manage_node_flows` | `list`, `get`, `create`, `update`, `delete`, `validate`, `run`, `list_runs`, `get_run`, `attach_to_agent`, `detach_from_agent` | +| `manage_node_flows` | `catalog`, `get_node_definition`, `create_draft`, `patch_draft`, `validate_draft`, custom-node and credential actions, `dry_run`, `publish`, `compare_versions`, `rollback`, `run`, `cancel`, `retry`, `inspect_run`, plus compatibility aliases | | `manage_memory` | `list`, `get`, `count`, `create`, `update`, `delete`, `search`, `promote`, `get_map`, `model_status`, `start_reembed` | | `manage_skills` | `authoring_prompt`, `list_storages`, `get_storage`, `create_storage`, `update_storage`, `delete_storage`, `reset_storage`, `list_agent_storages`, `attach_storage`, `detach_storage`, `list_skills`, `get_skill`, `create_skill`, `update_skill`, `delete_skill`, `import_markdown`, `export_markdown` | | `manage_settings` | `get_system`, `get_project_override`, `resolve_project_effective`, `get_sprint_override`, `resolve_sprint_effective`, `replace_system_settings`, `patch_system_setting`, `replace_project_settings`, `patch_project_setting`, `reset_project_settings`, `replace_sprint_settings`, `patch_sprint_setting`, `reset_sprint_settings`, `export_settings_bundle`, `apply_settings_bundle` | @@ -191,13 +192,9 @@ memory remediation, or global scheduler destructive controls. ## Node flows -`manage_node_flows` exposes project node workflows through MCP. It supports graph validation, CRUD, -runtime execution, run inspection, and flow-backed agent skill attachments. +`manage_node_flows` exposes governed project automation authoring through MCP. Draft patches require an optimistic `draftRevision`; conflicts return expected/actual revisions without writes. Validation and dry-run responses contain policy findings, required credentials, requested capabilities, side-effect diffs, and redacted summaries. Publication and rollback require exact-payload approval. Legacy CRUD/run/attach aliases remain compatible. -Create and update calls validate the structured graph before repository writes. `run` delegates to the -node-flow runtime through `NodeFlowService.runFlow`, and `delete` requires the normal approval -handshake. Responses mask secret-shaped graph data, inputs, and outputs before returning them to MCP -clients. +Attached flows appear to the owning authenticated agent only as name, description, input schema, flow id, and the `run_attached_flow` operation. Calls enforce project ownership, attachment, publication, and credential policy and record agent/conversation provenance without exposing complete graphs or credential material. Agents should build Code UX-adapted node flows rather than cloning n8n workflows one-to-one. Graphs should include dynamic widget schemas for editable graph inputs and node fields; callers can provide diff --git a/docs/mcp/runtime-and-dispatch.md b/docs/mcp/runtime-and-dispatch.md index 012f15c35f..9bfd562754 100644 --- a/docs/mcp/runtime-and-dispatch.md +++ b/docs/mcp/runtime-and-dispatch.md @@ -143,14 +143,17 @@ Runtime behavior: ## Node Flow Tools -`manage_node_flows` uses `NodeFlowService` as the MCP backend boundary. The action layer parses MCP payloads, applies optional widget schemas into the graph, masks secret-shaped response fields, and delegates graph validation, persistence, run inspection, runtime execution, and agent skill attachments to the service. +`manage_node_flows` uses `NodeFlowService` as the MCP backend boundary. The thin action layer parses MCP payloads and delegates catalog lookup, optimistic drafts, validation/policy review, credential metadata, publication/version operations, run controls, custom-node authoring, and attachments. Governed responses use graph summaries unless a legacy project-manager `get` explicitly requests the stored flow. Runtime behavior: -- `create` and `update` validate graph specs before repository writes. +- `create_draft` and `patch_draft` append immutable versions without auto-publication; stale `draftRevision` values return structured conflicts without writes. Legacy `create` and `update` retain auto-publication compatibility. +- `dry_run` never invokes the runtime; it reports validation, policy, credential, capability, and side-effect findings with redacted simulated output. +- `publish` and `rollback` use the stateful exact-payload approval handshake. Runtime execution always resolves a publication. - `run` calls the configured node-flow runtime through `NodeFlowService.runFlow`. +- `cancel`, `retry`, and `inspect_run` operate on project-owned durable run records. - `delete` uses the same stateful approval handshake as other destructive management actions. -- `attach_to_agent` and `detach_from_agent` manage flow-backed skill attachments for agent presets; the agent still needs explicit MCP access if it should call `manage_node_flows` itself. +- Attachments automatically expose only `run_attached_flow` to the owning agent. That operation verifies project, attachment, publication, and credential policy and records initiating agent/conversation metadata; it does not grant `manage_node_flows` or expose graphs/secrets. ## Custom MCP Defaults diff --git a/docs/mcp/tools-and-contracts.md b/docs/mcp/tools-and-contracts.md index 82ad2efa05..f802016e77 100644 --- a/docs/mcp/tools-and-contracts.md +++ b/docs/mcp/tools-and-contracts.md @@ -425,9 +425,11 @@ The dedicated management tools (`manage_sprints`, `manage_tasks`, `manage_quicks ## Node Flow Tools -`manage_node_flows` exposes project node workflows through the project-manager MCP surface. It supports `list`, `get`, `create`, `update`, `delete`, `validate`, `run`, `list_runs`, `get_run`, `attach_to_agent`, and `detach_from_agent`. +`manage_node_flows` is the project-manager automation-authoring surface. Governed actions are `catalog`, `get_node_definition`, `create_draft`, `patch_draft`, `validate_draft`, `create_custom_node`, `update_custom_node`, `validate_custom_node`, `request_credential`, `inspect_bindings`, `dry_run`, `publish`, `compare_versions`, `rollback`, `run`, `cancel`, `retry`, `inspect_run`, and `list_runs`. Compatibility aliases remain for `list`, `get`, `create`, `update`, `delete`, `validate`, `get_run`, `attach_to_agent`/`attach`, and `detach_from_agent`/`detach`. -Node-flow management always delegates graph validation and persistence to `NodeFlowService`; `run` delegates execution through the configured node-flow runtime service. Create and update calls reject malformed graph specs before repository writes. `delete` uses the standard stateful approval handshake. +New drafts are not executable until published. `patch_draft` requires the last observed positive integer `draftRevision`; stale revisions return a `draft_revision_conflict` containing expected and actual revisions without writing. Draft review responses are summaries rather than full graphs: they include validation issues, policy findings, credential requirements, requested capabilities, side-effect diffs, node/edge counts, and the active published version. `publish`, `rollback`, and `delete` use the exact-payload, one-use approval handshake. + +`dry_run` performs validation and policy simulation without executing nodes or side effects. It returns `executed: false`, redacted result metadata, and blockers such as missing or denied credential bindings. Credential actions return metadata only; decrypted credential values never cross the service or MCP response boundary. Custom-node validation reuses the governed project generator/build pipeline and returns checks, issues, capabilities, and credential slots rather than source bundles. The graph payload is the shared `NodeFlowGraph` contract: @@ -459,6 +461,8 @@ Attach a flow as an agent skill: } ``` +An attachment also gives that authenticated agent the narrow `run_attached_flow` capability. Its catalog entry contains only `flowId`, name, description, input schema, and `operation: "run_attached_flow"`; it never includes the graph or credentials. Execution verifies project ownership, the attachment, current publication, and credential policy, then records `initiatingAgentId`, the originating conversation id when present, and `triggerType: "attached_flow"` in run audit metadata. + Run a flow: ```json diff --git a/src/api/mcp/tool-registry.ts b/src/api/mcp/tool-registry.ts index 5d8d06b28d..b44bed9293 100644 --- a/src/api/mcp/tool-registry.ts +++ b/src/api/mcp/tool-registry.ts @@ -1,5 +1,5 @@ import type { ToolName as ContractToolName } from "../../contracts/mcp-tool-definitions.js"; -import type { ManageCodeUxArgs, ManageProjectsArgs, ManageSprintsArgs, ManageTasksArgs, ManageQuicksprintsArgs, ManageSchedulerArgs, SchedulerArgs, ManageAgentsArgs, ManageNodeFlowsArgs, ManageMemoryArgs, AddLongTermMemoryArgs, ManageSkillsArgs, ManageSettingsArgs, ManagePreviewArgs, ManageCustomDashboardsArgs, ManageChatProvidersArgs, ManageTelemetryArgs, SearchKnowledgeArgs, SearchSkillsArgs, RequestClarificationArgs, ReplyToClarificationArgs } from "../../contracts/internal-management-types.js"; +import type { ManageCodeUxArgs, ManageProjectsArgs, ManageSprintsArgs, ManageTasksArgs, ManageQuicksprintsArgs, ManageSchedulerArgs, SchedulerArgs, ManageAgentsArgs, ManageNodeFlowsArgs, ManageMemoryArgs, AddLongTermMemoryArgs, ManageSkillsArgs, ManageSettingsArgs, ManagePreviewArgs, ManageCustomDashboardsArgs, ManageChatProvidersArgs, ManageTelemetryArgs, SearchKnowledgeArgs, SearchSkillsArgs, RequestClarificationArgs, ReplyToClarificationArgs, RunAttachedFlowArgs } from "../../contracts/internal-management-types.js"; import type { PullWorkerTaskDispatchArgs, RegisterExternalWorkerEndpointArgs, UpdateWorkerTaskDispatchArgs } from "../../services/worker-task-dispatch-service.js"; export interface McpToolArgsByName { @@ -14,6 +14,7 @@ export interface McpToolArgsByName { scheduler_code_ux: SchedulerArgs; manage_agents: ManageAgentsArgs; manage_node_flows: ManageNodeFlowsArgs; + run_attached_flow: RunAttachedFlowArgs; manage_memory: ManageMemoryArgs; add_long_term_memory: AddLongTermMemoryArgs; manage_skills: ManageSkillsArgs; diff --git a/src/app/dependency-factory/core-factory.ts b/src/app/dependency-factory/core-factory.ts index 4e56765e25..6d97a2bd16 100644 --- a/src/app/dependency-factory/core-factory.ts +++ b/src/app/dependency-factory/core-factory.ts @@ -310,7 +310,7 @@ export function createCoreDependencies( const schedulerRepository = new SchedulerRepository(appDbStorage, dashboardRealtimeService); const skillRepository = new SkillRepository(appDbStorage); const nodeFlowRepository = new NodeFlowRepository(appDbStorage, dashboardRealtimeService); - const nodeFlowService = new NodeFlowService(nodeFlowRepository); + const nodeFlowService = new NodeFlowService(nodeFlowRepository, undefined, credentialBroker); const embeddingService = new EmbeddingService(); const embeddingModelManager = new EmbeddingModelManager( embeddingService, diff --git a/src/app/dependency-factory/dashboard-factory.ts b/src/app/dependency-factory/dashboard-factory.ts index 32367a3a6f..629e0a49ff 100644 --- a/src/app/dependency-factory/dashboard-factory.ts +++ b/src/app/dependency-factory/dashboard-factory.ts @@ -41,6 +41,8 @@ import { AutomationOutboxRepository } from "../../repositories/automation-outbox import { AutomationWebhookTriggerRepository } from "../../repositories/automation-webhook-trigger-repository.js"; import { CustomNodeRepository } from "../../repositories/custom-node-repository.js"; import { CustomNodeRuntimeService } from "../../services/custom-nodes/custom-node-runtime-service.js"; +import { CustomNodeProjectService } from "../../services/custom-nodes/custom-node-project-service.js"; +import { CustomNodeBuildService } from "../../services/custom-nodes/custom-node-build-service.js"; import { customNodeDefinitionFromArtifact } from "../../contracts/custom-node-types.js"; import { registerCustomNodeDefinition } from "../../domain/node-flows/node-definition-registry.js"; @@ -243,6 +245,8 @@ export function createDashboardDependencies( const approvalService = new ApprovalService(approvalRepository); const egressPolicyService = new EgressPolicyService(); const customNodeRepository = new CustomNodeRepository(coreDeps.appDbStorage); + const customNodeProjectService = new CustomNodeProjectService(); + const customNodeBuildService = new CustomNodeBuildService({ repository: customNodeRepository, projectService: customNodeProjectService }); for (const { artifact } of customNodeRepository.listPublications()) { registerCustomNodeDefinition(customNodeDefinitionFromArtifact(artifact)); } @@ -267,7 +271,16 @@ export function createDashboardDependencies( if (coreDeps.nodeFlowRepository) { new NodeFlowRecoveryService(coreDeps.nodeFlowRepository).recover(); } - const nodeFlowService = new NodeFlowService(coreDeps.nodeFlowRepository, nodeFlowRuntimeService); + const nodeFlowService = new NodeFlowService(coreDeps.nodeFlowRepository, nodeFlowRuntimeService, coreDeps.credentialBroker, { + repository: customNodeRepository, + projectService: customNodeProjectService, + buildService: customNodeBuildService, + resolveProjectRoot: (projectId) => { + const project = coreDeps.projectManagementRepository.getProject(projectId); + if (!project) throw new Error(`Project not found: ${projectId}`); + return project.baseDir; + }, + }); const activityCacheService = new ActivityCacheService( { diff --git a/src/contracts/internal-management-types.ts b/src/contracts/internal-management-types.ts index e9bea58c43..7303c30b09 100644 --- a/src/contracts/internal-management-types.ts +++ b/src/contracts/internal-management-types.ts @@ -220,13 +220,30 @@ export interface ManageAgentsArgs { } export interface ManageNodeFlowsArgs { - action: "list" | "get" | "create" | "update" | "delete" | "validate" | "run" | "list_runs" | "get_run" | "attach_to_agent" | "detach_from_agent"; + action: + | "catalog" | "get_node_definition" | "create_draft" | "patch_draft" | "validate_draft" + | "create_custom_node" | "update_custom_node" | "validate_custom_node" + | "request_credential" | "inspect_bindings" | "dry_run" | "publish" | "compare_versions" + | "rollback" | "run" | "cancel" | "retry" | "inspect_run" | "list_runs" + | "list" | "get" | "create" | "update" | "delete" | "validate" | "get_run" + | "attach_to_agent" | "detach_from_agent" | "attach" | "detach"; projectId?: string; flowId?: string; runId?: string; + nodeType?: string; + nodeVersion?: number; + nodeId?: string; + slot?: string; + draftRevision?: number; + fromVersion?: number; + toVersion?: number; + version?: number; + publishedBy?: string; name?: string; description?: string; graph?: NodeFlowGraph; + patch?: Record; + operations?: import("./node-flow-types.js").NodeFlowGraphPatchOperation[]; widgets?: NodeWidgetSchema | Record; input?: NodeFlowJsonObject; agentPresetId?: string; @@ -234,6 +251,12 @@ export interface ManageNodeFlowsArgs { approval?: ManagementApproval; } +export interface RunAttachedFlowArgs { + projectId: string; + flowId: string; + input?: NodeFlowJsonObject; +} + export interface ManageMemoryArgs { action: "search" | "list" | "get" | "create" | "update" | "delete" | "promote" | "start_reembed" | "get_map" | "count" | "model_status" | "create_claim" | "list_claims" | "get_claim" | "update_claim" | "add_claim_evidence" | "deprecate_claim"; projectId?: string; diff --git a/src/contracts/mcp-tool-definitions.ts b/src/contracts/mcp-tool-definitions.ts index d5be21c953..b5f8b2695b 100644 --- a/src/contracts/mcp-tool-definitions.ts +++ b/src/contracts/mcp-tool-definitions.ts @@ -390,17 +390,33 @@ export const TOOL_DEFINITIONS = [ name: "manage_node_flows", runtimeRoles: ["project_manager"], category: "agents_memory", - description: "Manage Code UX node flows as reusable agent skills. Supports list, get, create, update, delete, validate, run, list_runs, get_run, attach_to_agent, and detach_from_agent. Build Code UX-adapted flows with typed graph nodes and dynamic widget schemas for editable fields; do not clone n8n workflows one-to-one. Deleting flows requires approval confirmation.", + description: "Govern node-flow authoring, validation, credential bindings, publication, version review, and operational runs. Legacy list/get/create/update/delete/validate/run/attach aliases remain available; publish, rollback, and delete require approval.", inputSchema: { type: "object", properties: { - action: { type: "string", enum: ["list", "get", "create", "update", "delete", "validate", "run", "list_runs", "get_run", "attach_to_agent", "detach_from_agent"], description: "The node-flow action to perform." }, + action: { type: "string", enum: ["catalog", "get_node_definition", "create_draft", "patch_draft", "validate_draft", "create_custom_node", "update_custom_node", "validate_custom_node", "request_credential", "inspect_bindings", "dry_run", "publish", "compare_versions", "rollback", "run", "cancel", "retry", "inspect_run", "list_runs", "list", "get", "create", "update", "delete", "validate", "get_run", "attach_to_agent", "detach_from_agent", "attach", "detach"], description: "The governed node-flow action or compatibility alias to perform." }, projectId: { type: "string", description: "Required for list, create, run, and validation of new graph specs." }, flowId: { type: "string", description: "Required for get, update, delete, run, list_runs, attach_to_agent, and detach_from_agent." }, runId: { type: "string", description: "Required for get_run." }, + nodeType: { type: "string", description: "Node-definition type for get_node_definition." }, + nodeVersion: { type: "integer", minimum: 1, description: "Optional node-definition version." }, + nodeId: { type: "string", description: "Node id for custom-node and credential actions." }, + slot: { type: "string", description: "Credential slot for request_credential." }, + draftRevision: { type: "integer", minimum: 1, description: "Required optimistic revision for patch, publish, and rollback." }, + fromVersion: { type: "integer", minimum: 1 }, + toVersion: { type: "integer", minimum: 1 }, + version: { type: "integer", minimum: 1, description: "Historical version used by rollback." }, + publishedBy: { type: "string", description: "Publication audit actor; defaults to the MCP project manager identity." }, + sourceRevision: { type: "string", description: "Immutable custom-node source revision." }, + actor: { type: "string", description: "Custom-node audit actor." }, + invocationId: { type: "string", description: "Custom-node validation invocation id." }, + correlationId: { type: "string", description: "Custom-node validation correlation id." }, + manifest: { type: "object", additionalProperties: true, description: "Complete governed custom-node manifest for update_custom_node." }, name: { type: "string", description: "Required for create. Optional title for update." }, description: { type: "string", description: "Optional flow or attached skill description." }, graph: { type: "object", additionalProperties: true, description: "Structured node-flow graph. Required for create and standalone validate. Optional for update and validate existing flow." }, + patch: { type: "object", additionalProperties: true, description: "Optimistic draft patch containing graph, operations, name, or description." }, + operations: { type: "array", items: { type: "object", additionalProperties: true }, description: "Typed graph patch operations." }, widgets: { type: "object", additionalProperties: true, description: "Optional dynamic widget schemas for editable graph inputs or node fields. Use a fields array for graph inputSchema or node-id keys for node widgetSchema entries." }, input: { type: "object", additionalProperties: true, description: "Optional JSON object passed to run." }, agentPresetId: { type: "string", description: "Required for attach_to_agent and detach_from_agent." }, @@ -415,6 +431,23 @@ export const TOOL_DEFINITIONS = [ required: ["action"], }, }, + { + name: "run_attached_flow", + runtimeRoles: ["project_manager"], + audiences: ["project_manager", "worker"], + category: "agents_memory", + description: "Run one published node flow explicitly attached to the authenticated agent. The server derives agent and conversation identity and never exposes the flow graph or credential material.", + inputSchema: { + type: "object", + additionalProperties: false, + properties: { + projectId: { type: "string", minLength: 1 }, + flowId: { type: "string", minLength: 1 }, + input: { type: "object", additionalProperties: true }, + }, + required: ["projectId", "flowId"], + }, + }, { name: "manage_memory", runtimeRoles: ["project_manager"], diff --git a/src/contracts/node-flow-types.ts b/src/contracts/node-flow-types.ts index d675f327d9..bce9c7ce8c 100644 --- a/src/contracts/node-flow-types.ts +++ b/src/contracts/node-flow-types.ts @@ -184,6 +184,65 @@ export interface UpdateNodeFlowInput { graph?: NodeFlowGraph; } +export type NodeFlowGraphPatchOperation = + | { op: "upsert_node"; node: NodeFlowNode } + | { op: "remove_node"; nodeId: string } + | { op: "upsert_edge"; edge: NodeFlowEdge } + | { op: "remove_edge"; edgeId?: string; fromNodeId?: string; toNodeId?: string } + | { op: "set_input_schema"; inputSchema: NodeWidgetSchema | null } + | { op: "set_metadata"; metadata: NodeFlowJsonObject | null }; + +export interface PatchNodeFlowDraftInput { + projectId: string; + draftRevision: number; + graph?: NodeFlowGraph; + operations?: NodeFlowGraphPatchOperation[]; + title?: string; + description?: string; +} + +export interface NodeFlowConcurrencyConflict { + code: "draft_revision_conflict"; + flowId: string; + expectedDraftRevision: number; + actualDraftRevision: number; + message: string; +} + +export interface NodeFlowPolicyFinding { + severity: "info" | "warning" | "error"; + code: string; + nodeId?: string; + message: string; +} + +export interface NodeFlowRequiredCredential { + nodeId: string; + slot: string; + allowedKinds: string[]; + requiredCapabilities: string[]; + required: boolean; + credentialId: string | null; + status: "bound" | "missing" | "denied"; +} + +export interface NodeFlowDraftReview { + flowId: string; + projectId: string; + name: string; + description: string; + draftRevision: number; + nodeCount: number; + edgeCount: number; + valid: boolean; + validationIssues: NodeFlowValidationIssue[]; + policyFindings: NodeFlowPolicyFinding[]; + requiredCredentials: NodeFlowRequiredCredential[]; + requestedCapabilities: string[]; + sideEffectDiffs: Array<{ nodeId: string; sideEffect: NodeFlowSideEffect; description: string }>; + publishedVersion: number | null; +} + export interface NodeFlowValidationIssue { field: string; code: string; diff --git a/src/mcp/management-tool-handler.ts b/src/mcp/management-tool-handler.ts index ffed6a7444..36e3870e22 100644 --- a/src/mcp/management-tool-handler.ts +++ b/src/mcp/management-tool-handler.ts @@ -20,10 +20,11 @@ import type { SearchKnowledgeArgs, SearchSkillsArgs, RequestClarificationArgs, - ReplyToClarificationArgs + ReplyToClarificationArgs, + RunAttachedFlowArgs, } from "../contracts/internal-management-types.js"; import type { KnowledgeService } from "../services/knowledge-service.js"; -import { getCurrentMcpAgentId } from "../server/mcp-agent-context.js"; +import { getCurrentMcpAgentId, getCurrentMcpThreadId } from "../server/mcp-agent-context.js"; import type { SprintPreviewService } from "../services/sprint-preview-service.js"; import type { CustomDashboardRepository } from "../repositories/custom-dashboard-repository.js"; import type { CustomDashboardValidationService } from "../services/custom-dashboard-validation-service.js"; @@ -46,6 +47,7 @@ import type { } from "../services/worker-task-dispatch-service.js"; import type { SkillService } from "../services/skill-service.js"; import type { NodeFlowService } from "../services/node-flow-service.js"; +import { NodeFlowAgentSkillService } from "../services/node-flow-agent-skill-service.js"; import type { WorkerClarificationService } from "../services/worker-clarification-service.js"; import type { WorkerClarificationContinuationService } from "../services/worker-clarification-continuation-service.js"; @@ -116,6 +118,7 @@ export class ManagementToolHandler { private readonly settingsActions: SettingsActions; private readonly agentActions: AgentActions; private readonly nodeFlowActions: NodeFlowActions; + private readonly nodeFlowAgentSkillService: NodeFlowAgentSkillService; private readonly memoryActions: MemoryActions; private readonly skillActions: SkillActions; private readonly previewActions: PreviewActions; @@ -126,6 +129,7 @@ export class ManagementToolHandler { this.settingsActions = new SettingsActions(deps.settingsRepository); this.agentActions = new AgentActions(deps.agentPresetSyncService); this.nodeFlowActions = new NodeFlowActions(deps.nodeFlowService); + this.nodeFlowAgentSkillService = new NodeFlowAgentSkillService(deps.nodeFlowService); this.memoryActions = new MemoryActions(deps.memoryService, deps.memoryPromotionService, deps.embeddingModelManager); this.skillActions = new SkillActions(deps.skillService); this.previewActions = new PreviewActions(deps.sprintPreviewService); @@ -239,6 +243,8 @@ export class ManagementToolHandler { || args.action.startsWith("replace_") || args.action === "remove_session" || args.action === "archive" + || args.action === "publish" + || args.action === "rollback" || args.action === "deprecate_claim"; } @@ -456,6 +462,23 @@ export class ManagementToolHandler { } } + async handleRunAttachedFlow(args: RunAttachedFlowArgs): Promise<{ content: Array<{ type: string; text: string }> }> { + try { + const agentPresetId = getCurrentMcpAgentId(); + if (!agentPresetId) throw new Error("An authenticated agent is required to run an attached flow."); + const result = await this.nodeFlowAgentSkillService.runAttachedFlow({ + projectId: args.projectId, + flowId: args.flowId, + agentPresetId, + conversationId: getCurrentMcpThreadId(), + parameters: args.input, + }); + return { content: [{ type: "text", text: JSON.stringify({ result }, null, 2) }] }; + } catch (error) { + return this.formatError("node_flows", "run_attached_flow", error); + } + } + async handleManageMemory(args: ManageMemoryArgs): Promise<{ content: Array<{ type: string; text: string }> }> { try { const managementArgs = { domain: "memory", action: args.action, payload: args as unknown as Record, approval: args.approval }; diff --git a/src/mcp/management/node-flow-actions.ts b/src/mcp/management/node-flow-actions.ts index a35437f0b5..997806ad6c 100644 --- a/src/mcp/management/node-flow-actions.ts +++ b/src/mcp/management/node-flow-actions.ts @@ -13,11 +13,13 @@ import type { NodeWidgetSchema, } from "../../contracts/node-flow-types.js"; import type { NodeFlowService } from "../../services/node-flow-service.js"; +import { getCurrentMcpAgentId } from "../../server/mcp-agent-context.js"; import { managementValidationError, parseOptionalObject, parseOptionalNumber, parseOptionalString, + parseOptionalIntegerStrict, parseRequiredObject, parseRequiredString, } from "./payload-parsers.js"; @@ -33,6 +35,28 @@ export class NodeFlowActions { switch (args.action) { case "list": return this.listFlows(payload); + case "catalog": + return { result: this.nodeFlowService.catalog() }; + case "get_node_definition": + return this.getNodeDefinition(payload); + case "create_draft": + return this.createDraft(payload); + case "patch_draft": + return this.patchDraft(payload); + case "validate_draft": + return this.validateDraft(payload); + case "request_credential": + return this.requestCredential(payload); + case "inspect_bindings": + return this.inspectBindings(payload); + case "dry_run": + return this.dryRun(payload); + case "publish": + return this.publishDraft(args, payload); + case "compare_versions": + return this.compareVersions(payload); + case "rollback": + return this.rollback(args, payload); case "get": return this.getFlow(payload); case "create": @@ -45,19 +69,133 @@ export class NodeFlowActions { return this.validateFlow(payload); case "run": return await this.runFlow(payload); + case "cancel": + return this.cancelRun(payload); + case "retry": + return await this.retryRun(payload); case "list_runs": return this.listRuns(payload); case "get_run": - return this.getRun(payload); + case "inspect_run": + return this.getRun(payload, args.action === "inspect_run"); case "attach_to_agent": + case "attach": return this.attachToAgent(payload); case "detach_from_agent": + case "detach": return this.detachFromAgent(payload); + case "create_custom_node": + return await this.createCustomNode(payload); + case "update_custom_node": + return await this.updateCustomNode(payload); + case "validate_custom_node": + return await this.validateCustomNode(payload); default: throw new Error(`Unknown node flow action: ${args.action}`); } } + private getNodeDefinition(payload: Record): ManagementResponseEnvelope { + const nodeType = parseRequiredString(payload, "nodeType"); + const version = parseOptionalIntegerStrict(payload, "nodeVersion", { min: 1 }); + const definition = this.nodeFlowService.nodeDefinition(nodeType, version); + if (!definition) throw managementValidationError(`Node definition not found: ${nodeType}${version ? `@${version}` : ""}`, "nodeType"); + return { result: { definition } }; + } + + private createDraft(payload: Record): ManagementResponseEnvelope { + const graph = this.parseGraphWithWidgets(payload, true); + if (!graph) throw managementValidationError("graph object is required", "graph"); + const validation = this.nodeFlowService.validate(graph); + if (!validation.valid || !validation.graph) return { result: { status: "invalid", validationIssues: validation.errors } }; + return { result: { draft: this.nodeFlowService.createDraft(parseRequiredString(payload, "projectId"), { + title: parseRequiredString(payload, "name"), description: parseOptionalText(payload, "description"), graph: validation.graph, + }) } }; + } + + private patchDraft(payload: Record): ManagementResponseEnvelope { + const projectId = parseRequiredString(payload, "projectId"); + const flowId = parseRequiredString(payload, "flowId"); + const draftRevision = requiredInteger(payload, "draftRevision"); + const patch = parseOptionalObject>(payload, "patch") ?? {}; + const graph = this.parseGraphWithWidgets(patch, false) ?? this.parseGraphWithWidgets(payload, false); + const operations = Array.isArray(patch.operations) ? patch.operations : Array.isArray(payload.operations) ? payload.operations : undefined; + return { result: this.nodeFlowService.patchDraft(flowId, { + projectId, draftRevision, graph, + operations: operations as import("../../contracts/node-flow-types.js").NodeFlowGraphPatchOperation[] | undefined, + title: parseOptionalString(patch, "name") ?? parseOptionalString(payload, "name"), + description: parseOptionalText(patch, "description") ?? parseOptionalText(payload, "description"), + }) }; + } + + private validateDraft(payload: Record): ManagementResponseEnvelope { + return { result: { draft: this.nodeFlowService.validateDraft(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "flowId")) } }; + } + + private requestCredential(payload: Record): ManagementResponseEnvelope { + return { result: { request: this.nodeFlowService.requestCredential(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "flowId"), parseRequiredString(payload, "nodeId"), parseRequiredString(payload, "slot")) } }; + } + + private inspectBindings(payload: Record): ManagementResponseEnvelope { + return { result: this.nodeFlowService.inspectBindings(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "flowId")) }; + } + + private dryRun(payload: Record): ManagementResponseEnvelope { + return { result: this.nodeFlowService.dryRun(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "flowId"), parseOptionalObject(payload, "input") ?? {}) }; + } + + private publishDraft(args: ManageCodeUxArgs, payload: Record): ManagementResponseEnvelope { + const flowId = parseRequiredString(payload, "flowId"); + const draftRevision = requiredInteger(payload, "draftRevision"); + if (args.approval?.confirmed !== true) return { approvalRequired: true, approvalMessage: `Publish node flow ${flowId} draft revision ${draftRevision} after reviewing validation, credentials, capabilities, and side effects.` }; + return { result: { draft: this.nodeFlowService.publishDraft(parseRequiredString(payload, "projectId"), flowId, draftRevision, parseOptionalString(payload, "publishedBy") ?? "project-manager-mcp") } }; + } + + private compareVersions(payload: Record): ManagementResponseEnvelope { + return { result: this.nodeFlowService.compareVersions(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "flowId"), requiredInteger(payload, "fromVersion"), requiredInteger(payload, "toVersion")) }; + } + + private rollback(args: ManageCodeUxArgs, payload: Record): ManagementResponseEnvelope { + const flowId = parseRequiredString(payload, "flowId"); + const version = requiredInteger(payload, "version"); + if (args.approval?.confirmed !== true) return { approvalRequired: true, approvalMessage: `Create a new draft of node flow ${flowId} from version ${version}. The current draft remains in immutable history.` }; + return { result: { draft: this.nodeFlowService.rollback(parseRequiredString(payload, "projectId"), flowId, version, requiredInteger(payload, "draftRevision")) } }; + } + + private cancelRun(payload: Record): ManagementResponseEnvelope { + return { result: { run: formatRun(this.nodeFlowService.cancelRun(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "runId"))) } }; + } + + private async retryRun(payload: Record): Promise { + return { result: formatRunSummary(await this.nodeFlowService.retryRun(parseRequiredString(payload, "projectId"), parseRequiredString(payload, "runId"))) }; + } + + private async createCustomNode(payload: Record): Promise { + const node = await this.nodeFlowService.createCustomNode(parseRequiredString(payload, "projectId"), { + nodeId: parseRequiredString(payload, "nodeId"), name: parseRequiredString(payload, "name"), + description: parseOptionalText(payload, "description"), sourceRevision: parseRequiredString(payload, "sourceRevision"), + createdBy: parseOptionalString(payload, "actor") ?? "project-manager-mcp", + }); + return { result: { node } }; + } + + private async updateCustomNode(payload: Record): Promise { + const node = await this.nodeFlowService.updateCustomNode( + parseRequiredString(payload, "projectId"), parseRequiredString(payload, "nodeId"), + parseRequiredObject(payload, "manifest"), parseRequiredString(payload, "sourceRevision"), + ); + return { result: { node } }; + } + + private async validateCustomNode(payload: Record): Promise { + return { result: await this.nodeFlowService.validateCustomNode( + parseRequiredString(payload, "projectId"), parseRequiredString(payload, "nodeId"), + parseOptionalString(payload, "actor") ?? "project-manager-mcp", + parseOptionalString(payload, "invocationId") ?? `mcp-custom-node-${Date.now()}`, + parseOptionalString(payload, "correlationId") ?? `mcp-custom-node-${Date.now()}`, + ) }; + } + private listFlows(payload: Record): ManagementResponseEnvelope { const projectId = parseRequiredString(payload, "projectId"); const flows = this.nodeFlowService.list(projectId).flows.map(formatFlowSummary); @@ -70,7 +208,8 @@ export class NodeFlowActions { this.assertProjectMatch(payload, flow); return { result: { - flow: formatFlow(flow), + flow: formatFlowForCaller(flow), + ...(getCurrentMcpAgentId() ? { draft: this.nodeFlowService.validateDraft(flow.projectId, flow.id) } : {}), agentSkills: this.nodeFlowService.listAgentSkills(flow.id), }, }; @@ -92,7 +231,7 @@ export class NodeFlowActions { description: parseOptionalText(payload, "description"), graph: validation.graph, }); - return { result: { flow: formatFlow(flow) } }; + return { result: { flow: formatFlowForCaller(flow) } }; } private updateFlow(payload: Record): ManagementResponseEnvelope { @@ -110,7 +249,7 @@ export class NodeFlowActions { ...(description !== undefined ? { description } : {}), ...(validation?.graph ? { graph: validation.graph } : {}), }); - return { result: { flow: formatFlow(flow) } }; + return { result: { flow: formatFlowForCaller(flow) } }; } private deleteFlow(args: ManageCodeUxArgs, payload: Record): ManagementResponseEnvelope { @@ -152,16 +291,20 @@ export class NodeFlowActions { private listRuns(payload: Record): ManagementResponseEnvelope { const flowId = parseRequiredString(payload, "flowId"); + const projectId = parseOptionalString(payload, "projectId"); + if (projectId) this.assertProjectMatch(payload, this.requireFlow(flowId)); const runs = this.nodeFlowService.listRuns(flowId).runs.map(formatRun); return { result: { runs } }; } - private getRun(payload: Record): ManagementResponseEnvelope { + private getRun(payload: Record, requireProject = false): ManagementResponseEnvelope { const runId = parseRequiredString(payload, "runId"); + const projectId = requireProject ? parseRequiredString(payload, "projectId") : parseOptionalString(payload, "projectId"); const run = this.nodeFlowService.getRun(runId); if (!run) { throw new Error(`Node flow run not found: ${runId}`); } + if (projectId && run.projectId !== projectId) throw managementValidationError("Node flow run does not belong to the requested project.", "projectId"); return { result: { run: formatRun(run), @@ -221,6 +364,12 @@ export class NodeFlowActions { } } +function requiredInteger(payload: Record, key: string): number { + const value = parseOptionalIntegerStrict(payload, key, { min: 1 }); + if (value === undefined) throw managementValidationError(`${key} is required`, key); + return value; +} + function parseOptionalText(payload: Record, key: string): string | undefined { if (!(key in payload)) { return undefined; @@ -289,6 +438,10 @@ function formatFlow(flow: NodeFlowRecord): Record { }; } +function formatFlowForCaller(flow: NodeFlowRecord): Record { + return getCurrentMcpAgentId() ? formatFlowSummary(flow) : formatFlow(flow); +} + function formatRun(run: NodeFlowRunRecord): Record { return { ...run, diff --git a/src/repositories/custom-node-repository.ts b/src/repositories/custom-node-repository.ts index f4d41d6ce5..3a44dd77ee 100644 --- a/src/repositories/custom-node-repository.ts +++ b/src/repositories/custom-node-repository.ts @@ -65,6 +65,16 @@ export class CustomNodeRepository { .map((row) => this.mapNode(row)); } + updateDraft(nodeId: string, manifest: CustomNodeManifest, sourceRevision: string): CustomNodeRecord { + const node = this.requireNode(nodeId); + if (node.status === "published" || node.status === "validating") throw new ValidationError("Published or validating custom node revisions are immutable."); + if (manifest.id !== node.id || manifest.nodeType !== node.manifest.nodeType) throw new ValidationError("Custom node identity cannot be changed."); + if (!sourceRevision.trim()) throw new ValidationError("Custom node source revision is required."); + this.db.prepare("UPDATE custom_nodes SET status = 'draft', source_revision = ?, manifest_json = ?, validation_report_json = NULL, artifact_digest = NULL, updated_at = ? WHERE id = ?") + .run(sourceRevision.trim(), JSON.stringify(manifest), new Date().toISOString(), nodeId); + return this.requireNode(nodeId); + } + beginValidation(nodeId: string): CustomNodeRecord { const node = this.requireNode(nodeId); if (node.status === "published") throw new ValidationError("Published custom node revisions are immutable."); diff --git a/src/repositories/node-flow-repository.ts b/src/repositories/node-flow-repository.ts index c3648e3606..d40fea4196 100644 --- a/src/repositories/node-flow-repository.ts +++ b/src/repositories/node-flow-repository.ts @@ -144,7 +144,7 @@ export class NodeFlowRepository { return row ? this.mapFlowRow(row) : null; } - createFlow(projectId: string, input: CreateNodeFlowInput): NodeFlowRecord { + createFlow(projectId: string, input: CreateNodeFlowInput, options: { publish?: boolean; publishedBy?: string } = {}): NodeFlowRecord { this.requireProject(projectId); const now = new Date().toISOString(); const id = input.id?.trim() || randomUUID(); @@ -166,7 +166,9 @@ export class NodeFlowRepository { graphJson, createdAt: now, }); - this.insertPublication(id, projectId, 1, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, "system"); + if (options.publish !== false) { + this.insertPublication(id, projectId, 1, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, options.publishedBy ?? "system"); + } }); const created = this.requireFlow(id); @@ -174,7 +176,7 @@ export class NodeFlowRepository { return created; } - updateFlow(flowId: string, input: UpdateNodeFlowInput): NodeFlowRecord { + updateFlow(flowId: string, input: UpdateNodeFlowInput, options: { publish?: boolean; publishedBy?: string } = {}): NodeFlowRecord { const current = this.requireFlow(flowId); const now = new Date().toISOString(); const title = input.title === undefined ? current.title : this.requireTitle(input.title); @@ -198,7 +200,9 @@ export class NodeFlowRepository { graphJson, createdAt: now, }); - this.insertPublication(flowId, current.projectId, nextVersion, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, "system"); + if (options.publish !== false) { + this.insertPublication(flowId, current.projectId, nextVersion, graphJson, DEFAULT_NODE_FLOW_EXECUTION_POLICY, options.publishedBy ?? "system"); + } }); const updated = this.requireFlow(flowId); diff --git a/src/server/code-ux-server.ts b/src/server/code-ux-server.ts index ca7ac7b433..aa6e8e21e2 100644 --- a/src/server/code-ux-server.ts +++ b/src/server/code-ux-server.ts @@ -49,6 +49,7 @@ import { isProjectManagerClarificationAgent, isWorkerClarificationAgent, toAgentCodeUxToolAccess, + withAttachedFlowAccess, withClarificationAudienceAccess, } from "../services/agent-mcp-access.js"; import { JulesSourceResolver } from "../services/jules-source-resolver.js"; @@ -525,6 +526,9 @@ export class CodeUxServer { : persistentSkillRetrievalEnabled ? toAgentCodeUxToolAccess({ codeUxEnabled: false, codeUxToolToggles: [] }, true) : { codeUxEnabled: false, codeUxToolToggles: [] }; + if (this.nodeFlowService.listAgentSkillsForAgent(agent.projectId, agent.id).length > 0) { + resolvedAccess = withAttachedFlowAccess(resolvedAccess); + } if (workerEligible) { resolvedAccess = withClarificationAudienceAccess(resolvedAccess, "worker", "request_clarification"); } diff --git a/src/server/mcp-request-router.ts b/src/server/mcp-request-router.ts index 8470bebd40..6b2fefe52f 100644 --- a/src/server/mcp-request-router.ts +++ b/src/server/mcp-request-router.ts @@ -43,6 +43,7 @@ export const registerMcpRequestHandlers = (args: McpRequestRouterArgs): void => .register("scheduler_code_ux", async (input) => (await args.managementToolHandler.handleScheduler(input)) as McpToolResponse) .register("manage_agents", async (input) => (await args.managementToolHandler.handleManageAgents(input)) as McpToolResponse) .register("manage_node_flows", async (input) => (await args.managementToolHandler.handleManageNodeFlows(input)) as McpToolResponse) + .register("run_attached_flow", async (input) => (await args.managementToolHandler.handleRunAttachedFlow(input)) as McpToolResponse) .register("manage_memory", async (input) => (await args.managementToolHandler.handleManageMemory(input)) as McpToolResponse) .register("add_long_term_memory", async (input) => (await args.managementToolHandler.handleAddLongTermMemory(input)) as McpToolResponse) .register("manage_skills", async (input) => (await args.managementToolHandler.handleManageSkills(input)) as McpToolResponse) diff --git a/src/server/node-flow-routes.ts b/src/server/node-flow-routes.ts index 4032685d0b..f20c244148 100644 --- a/src/server/node-flow-routes.ts +++ b/src/server/node-flow-routes.ts @@ -19,6 +19,102 @@ function requireNodeFlowService(deps: DashboardDependencies): NonNullable { + res.json(requireNodeFlowService(deps).catalog()); + })); + + app.get("/api/node-flow-catalog/:nodeType", syncRoute((req, res) => { + const definition = requireNodeFlowService(deps).nodeDefinition( + requireTrimmedString(req.params.nodeType, "nodeType"), + parseOptionalInteger(req.query.version, 1, Number.MAX_SAFE_INTEGER, "version"), + ); + if (!definition) throw new HttpRouteError(404, "Node definition not found."); + res.json(definition); + })); + + app.post("/api/projects/:projectId/node-flow-drafts", syncRoute((req, res) => { + res.status(201).json(requireNodeFlowService(deps).createDraft( + requireTrimmedString(req.params.projectId, "projectId"), req.body as CreateNodeFlowInput, + )); + })); + + app.patch("/api/node-flow-drafts/:flowId", syncRoute((req, res) => { + const result = requireNodeFlowService(deps).patchDraft( + requireTrimmedString(req.params.flowId, "flowId"), req.body, + ); + res.status(result.conflict ? 409 : 200).json(result); + })); + + app.post("/api/node-flow-drafts/:flowId/validate", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).validateDraft( + requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + )); + })); + + app.post("/api/node-flow-drafts/:flowId/dry-run", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).dryRun( + requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), req.body?.input ?? {}, + )); + })); + + app.get("/api/node-flow-drafts/:flowId/bindings", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).inspectBindings( + requireTrimmedString(req.query.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + )); + })); + + app.post("/api/node-flow-drafts/:flowId/credential-requests", syncRoute((req, res) => { + res.status(201).json(requireNodeFlowService(deps).requestCredential( + requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + requireTrimmedString(req.body?.nodeId, "nodeId"), requireTrimmedString(req.body?.slot, "slot"), + )); + })); + + app.post("/api/projects/:projectId/custom-nodes", asyncRoute(async (req, res) => { + res.status(201).json(await requireNodeFlowService(deps).createCustomNode(requireTrimmedString(req.params.projectId, "projectId"), req.body)); + })); + + app.put("/api/projects/:projectId/custom-nodes/:nodeId", asyncRoute(async (req, res) => { + res.json(await requireNodeFlowService(deps).updateCustomNode( + requireTrimmedString(req.params.projectId, "projectId"), requireTrimmedString(req.params.nodeId, "nodeId"), req.body?.manifest, requireTrimmedString(req.body?.sourceRevision, "sourceRevision"), + )); + })); + + app.post("/api/projects/:projectId/custom-nodes/:nodeId/validate", asyncRoute(async (req, res) => { + res.json(await requireNodeFlowService(deps).validateCustomNode( + requireTrimmedString(req.params.projectId, "projectId"), requireTrimmedString(req.params.nodeId, "nodeId"), + requireTrimmedString(req.body?.actor, "actor"), requireTrimmedString(req.body?.invocationId, "invocationId"), requireTrimmedString(req.body?.correlationId, "correlationId"), + )); + })); + + app.post("/api/node-flow-drafts/:flowId/publish", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).publishDraft( + requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + parseRequiredBodyInteger(req.body?.draftRevision, "draftRevision"), requireTrimmedString(req.body?.publishedBy, "publishedBy"), + )); + })); + + app.get("/api/node-flows/:flowId/compare", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).compareVersions( + requireTrimmedString(req.query.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + parseRequiredBodyInteger(req.query.fromVersion, "fromVersion"), parseRequiredBodyInteger(req.query.toVersion, "toVersion"), + )); + })); + + app.post("/api/node-flows/:flowId/rollback", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).rollback( + requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.flowId, "flowId"), + parseRequiredBodyInteger(req.body?.version, "version"), parseRequiredBodyInteger(req.body?.draftRevision, "draftRevision"), + )); + })); + + app.post("/api/node-flow-runs/:runId/cancel", syncRoute((req, res) => { + res.json(requireNodeFlowService(deps).cancelRun(requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.runId, "runId"))); + })); + + app.post("/api/node-flow-runs/:runId/retry", asyncRoute(async (req, res) => { + res.status(201).json(await requireNodeFlowService(deps).retryRun(requireTrimmedString(req.body?.projectId, "projectId"), requireTrimmedString(req.params.runId, "runId"))); + })); app.get("/api/projects/:projectId/node-flows", syncRoute((req, res) => { res.json(requireNodeFlowService(deps).list(requireTrimmedString(req.params.projectId, "projectId"))); })); @@ -158,3 +254,9 @@ export function registerNodeFlowRoutes(app: Express, deps: DashboardDependencies res.status(201).json({ ...configured.trigger, pathToken: configured.pathToken, secret: configured.secret }); })); } + +function parseRequiredBodyInteger(value: unknown, label: string): number { + const parsed = typeof value === "string" && value.trim() ? Number(value) : value; + if (typeof parsed !== "number" || !Number.isInteger(parsed) || parsed < 1) throw new HttpRouteError(400, `${label} must be a positive integer.`); + return parsed; +} diff --git a/src/services/agent-mcp-access.ts b/src/services/agent-mcp-access.ts index 920fd0d217..1dd3f1bf91 100644 --- a/src/services/agent-mcp-access.ts +++ b/src/services/agent-mcp-access.ts @@ -117,6 +117,18 @@ export const withClarificationAudienceAccess = ( audienceToolNames: Array.from(new Set([...(access.audienceToolNames ?? []), toolName])), }); +/** Expose only the narrow attached-flow runner when an agent owns at least one flow attachment. */ +export const withAttachedFlowAccess = (access: AgentCodeUxToolAccess): AgentCodeUxToolAccess => { + const byName = new Map(access.codeUxToolToggles.map((toggle) => [toggle.name, toggle])); + byName.set("run_attached_flow", { name: "run_attached_flow", enabled: true, isInternal: true }); + return { + ...access, + codeUxEnabled: true, + audiences: Array.from(new Set([...(access.audiences ?? []), "worker" as const])), + codeUxToolToggles: Array.from(byName.values()), + }; +}; + const clarificationGatewayAccess = ( access: AgentMcpAccessConfig | null | undefined, toolName: Extract, diff --git a/src/services/custom-nodes/custom-node-project-service.ts b/src/services/custom-nodes/custom-node-project-service.ts index fb33698a2a..2aebbfde0f 100644 --- a/src/services/custom-nodes/custom-node-project-service.ts +++ b/src/services/custom-nodes/custom-node-project-service.ts @@ -61,6 +61,12 @@ export class CustomNodeProjectService { return parsed as CustomNodeManifest; } + async writeManifest(projectRoot: string, nodeId: string, manifest: CustomNodeManifest): Promise { + const root = this.resolveNodeRoot(projectRoot, nodeId); + if (manifest.id !== nodeId || manifest.nodeType !== `custom.${nodeId}`) throw new ValidationError("Custom node manifest identity does not match its project."); + await fs.writeFile(path.join(root, "node.json"), `${JSON.stringify(manifest, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + } + resolveNodeRoot(projectRoot: string, nodeId: string): string { if (!NODE_ID_PATTERN.test(nodeId)) throw new ValidationError("Invalid custom node id."); const base = path.resolve(projectRoot); diff --git a/src/services/node-flow-agent-skill-service.ts b/src/services/node-flow-agent-skill-service.ts new file mode 100644 index 0000000000..16de46ccfa --- /dev/null +++ b/src/services/node-flow-agent-skill-service.ts @@ -0,0 +1,57 @@ +import type { NodeFlowJsonObject, NodeFlowRunSummaryResponse, NodeFlowValueSchema } from "../contracts/node-flow-types.js"; +import { ValidationError, EntityNotFoundError } from "../repositories/repository-utils.js"; +import type { NodeFlowService } from "./node-flow-service.js"; + +export interface AttachedNodeFlowCapability { + flowId: string; + name: string; + description: string; + inputSchema: NodeFlowValueSchema; + operation: "run_attached_flow"; +} + +export class NodeFlowAgentSkillService { + constructor(private readonly nodeFlowService: NodeFlowService) {} + + listCapabilities(projectId: string, agentPresetId: string): AttachedNodeFlowCapability[] { + return this.nodeFlowService.listAgentSkillsForAgent(projectId, agentPresetId).map((attachment) => { + const flow = this.nodeFlowService.get(attachment.flowId); + if (!flow || flow.projectId !== projectId) { + throw new ValidationError("Attached node flow is outside the agent project."); + } + return { + flowId: flow.id, + name: attachment.skillName, + description: attachment.description, + inputSchema: flow.graph.schemas?.input ?? { type: "object" }, + operation: "run_attached_flow", + }; + }); + } + + async runAttachedFlow(input: { + projectId: string; + flowId: string; + agentPresetId: string; + conversationId?: string | null; + parameters?: NodeFlowJsonObject; + }): Promise { + const capability = this.listCapabilities(input.projectId, input.agentPresetId) + .find((item) => item.flowId === input.flowId); + if (!capability) throw new EntityNotFoundError("Node flow is not attached to the initiating agent."); + const review = this.nodeFlowService.validateDraft(input.projectId, input.flowId); + if (review.publishedVersion === null) throw new ValidationError("Attached node flow has not been published."); + if (review.requiredCredentials.some((credential) => credential.status !== "bound")) { + throw new ValidationError("Attached node flow credential policy is not satisfied."); + } + return await this.nodeFlowService.runFlow(input.projectId, input.flowId, input.parameters ?? {}, { + triggerType: "attached_flow", + triggerPayload: { + initiatingAgentId: input.agentPresetId, + ...(input.conversationId ? { conversationId: input.conversationId } : {}), + operation: "run_attached_flow", + }, + versionSelection: { mode: "latest_published" }, + }); + } +} diff --git a/src/services/node-flow-runtime-service.ts b/src/services/node-flow-runtime-service.ts index 72f7667521..9a7991676b 100644 --- a/src/services/node-flow-runtime-service.ts +++ b/src/services/node-flow-runtime-service.ts @@ -102,6 +102,10 @@ export class NodeFlowRuntimeService { }); } + requestCancellation(runId: string): NodeFlowRunRecord { + return this.deps.nodeFlowRepository.requestCancellation(runId); + } + async runFlow( projectId: string, flowId: string, @@ -187,7 +191,7 @@ export class NodeFlowRuntimeService { if (!node) { continue; } - if (options.signal?.aborted) { + if (options.signal?.aborted || this.deps.nodeFlowRepository.getRun(run.id)?.cancelRequestedAt) { terminalStatus = "cancelled"; terminalError ??= "Node flow run was cancelled."; await this.persistSkippedNode(context, node, "cancelled", terminalError); diff --git a/src/services/node-flow-service.ts b/src/services/node-flow-service.ts index 066e300c57..753612b958 100644 --- a/src/services/node-flow-service.ts +++ b/src/services/node-flow-service.ts @@ -1,14 +1,25 @@ import { EntityNotFoundError, ValidationError } from "../repositories/repository-utils.js"; import { NodeFlowRepository } from "../repositories/node-flow-repository.js"; import { normalizeNodeFlowGraph, validateNodeFlowGraph } from "../domain/node-flows/node-flow-validation.js"; +import { listNodeDefinitions, resolveNodeDefinition } from "../domain/node-flows/node-definition-registry.js"; +import type { CredentialBroker } from "./credentials/credential-broker.js"; +import type { CustomNodeManifest, CustomNodeRecord } from "../contracts/custom-node-types.js"; +import type { CustomNodeRepository } from "../repositories/custom-node-repository.js"; +import type { CustomNodeProjectService } from "./custom-nodes/custom-node-project-service.js"; +import type { CustomNodeBuildService } from "./custom-nodes/custom-node-build-service.js"; import type { AttachNodeFlowSkillInput, CreateNodeFlowInput, NodeFlowGraph, + NodeFlowGraphPatchOperation, NodeFlowJsonObject, NodeFlowListResponse, NodeFlowNodeRunListResponse, NodeFlowRecord, + NodeFlowDraftReview, + NodeFlowConcurrencyConflict, + NodeFlowRequiredCredential, + PatchNodeFlowDraftInput, NodeFlowRunListResponse, NodeFlowRunRecord, NodeFlowRunSummaryResponse, @@ -23,8 +34,26 @@ export class NodeFlowService { constructor( private readonly repository: NodeFlowRepository = new NodeFlowRepository(), private readonly runtimeService?: NodeFlowRuntimeService, + private readonly credentialBroker?: CredentialBroker, + private readonly customNodeAuthoring?: { + repository: CustomNodeRepository; + projectService: CustomNodeProjectService; + buildService: CustomNodeBuildService; + resolveProjectRoot(projectId: string): string; + }, ) {} + catalog(): { nodes: Array> } { + return { nodes: listNodeDefinitions().map((definition) => definitionSummary(definition)) }; + } + + nodeDefinition(type: string, version?: number): Record | null { + const definition = version === undefined + ? [...listNodeDefinitions()].filter((item) => item.type === type).sort((a, b) => b.version - a.version)[0] ?? null + : resolveNodeDefinition(type, version); + return definition ? definitionSummary(definition, true) : null; + } + list(projectId: string): NodeFlowListResponse { return { flows: this.repository.listFlows(projectId) }; } @@ -45,6 +74,156 @@ export class NodeFlowService { }); } + createDraft(projectId: string, input: CreateNodeFlowInput): NodeFlowDraftReview { + const title = normalizeRequiredText(input.title, "Node flow title"); + const description = normalizeOptionalText(input.description); + const { graph } = normalizeNodeFlowGraph(input.graph); + const flow = this.repository.createFlow(projectId, { id: input.id, title, description, graph }, { publish: false }); + return this.reviewDraft(flow); + } + + patchDraft(flowId: string, input: PatchNodeFlowDraftInput): { draft?: NodeFlowDraftReview; conflict?: NodeFlowConcurrencyConflict } { + const current = this.requireOwnedFlow(flowId, input.projectId); + if (current.version !== input.draftRevision) { + return { conflict: { + code: "draft_revision_conflict", + flowId, + expectedDraftRevision: input.draftRevision, + actualDraftRevision: current.version, + message: "The draft changed after it was read; reload the summary and reapply the patch.", + } }; + } + if (!input.graph && !input.operations?.length && input.title === undefined && input.description === undefined) { + throw new ValidationError("A graph patch, title, or description is required."); + } + const graph = input.graph ?? applyGraphPatch(current.graph, input.operations ?? []); + const validation = validateNodeFlowGraph(graph); + if (!validation.valid || !validation.graph) { + return { draft: this.reviewDraft({ ...current, graph }, validation) }; + } + const updated = this.repository.updateFlow(flowId, { + ...(input.title !== undefined ? { title: input.title } : {}), + ...(input.description !== undefined ? { description: input.description } : {}), + graph: validation.graph, + }, { publish: false }); + return { draft: this.reviewDraft(updated, validation) }; + } + + validateDraft(projectId: string, flowId: string): NodeFlowDraftReview { + return this.reviewDraft(this.requireOwnedFlow(flowId, projectId)); + } + + inspectBindings(projectId: string, flowId: string): Pick { + const review = this.reviewDraft(this.requireOwnedFlow(flowId, projectId)); + return { + requiredCredentials: review.requiredCredentials, + requestedCapabilities: review.requestedCapabilities, + policyFindings: review.policyFindings, + }; + } + + requestCredential(projectId: string, flowId: string, nodeId: string, slot: string): Record { + const review = this.reviewDraft(this.requireOwnedFlow(flowId, projectId)); + const requirement = review.requiredCredentials.find((item) => item.nodeId === nodeId && item.slot === slot); + if (!requirement) throw new ValidationError(`Credential slot is not declared: ${nodeId}.${slot}`); + return { projectId, flowId, ...requirement, requestStatus: requirement.status === "bound" ? "already_bound" : "requested" }; + } + + async createCustomNode(projectId: string, input: { nodeId: string; name: string; description?: string; sourceRevision: string; createdBy: string }): Promise { + const authoring = this.requireCustomNodeAuthoring(); + const generated = await authoring.projectService.generate({ + projectRoot: authoring.resolveProjectRoot(projectId), nodeId: input.nodeId, name: input.name, description: input.description, + }); + return authoring.repository.createDraft(projectId, { + manifest: generated.manifest, sourceRevision: normalizeRequiredText(input.sourceRevision, "sourceRevision"), createdBy: normalizeRequiredText(input.createdBy, "createdBy"), + }); + } + + async updateCustomNode(projectId: string, nodeId: string, manifest: CustomNodeManifest, sourceRevision: string): Promise { + const authoring = this.requireCustomNodeAuthoring(); + const current = authoring.repository.getNode(nodeId); + if (!current || current.projectId !== projectId) throw new EntityNotFoundError(`Custom node not found: ${nodeId}`); + await authoring.projectService.writeManifest(authoring.resolveProjectRoot(projectId), nodeId, manifest); + return authoring.repository.updateDraft(nodeId, manifest, sourceRevision); + } + + async validateCustomNode(projectId: string, nodeId: string, actor: string, invocationId: string, correlationId: string): Promise> { + const authoring = this.requireCustomNodeAuthoring(); + const current = authoring.repository.getNode(nodeId); + if (!current || current.projectId !== projectId) throw new EntityNotFoundError(`Custom node not found: ${nodeId}`); + const result = await authoring.buildService.validateAndBuild({ projectRoot: authoring.resolveProjectRoot(projectId), nodeId, creator: actor, invocationId, correlationId }); + return { nodeId, status: result.report.valid ? "passed" : "failed", validationIssues: result.report.issues, checks: result.report.checks, requestedCapabilities: current.manifest.capabilities, requiredCredentials: current.manifest.credentials }; + } + + dryRun(projectId: string, flowId: string, input: Record = {}): Record { + normalizeJsonObject(input, "input"); + const review = this.reviewDraft(this.requireOwnedFlow(flowId, projectId)); + const missingCredentials = review.requiredCredentials.filter((item) => item.status !== "bound"); + return { + status: review.valid && missingCredentials.length === 0 ? "ready" : "blocked", + draftRevision: review.draftRevision, + validationIssues: review.validationIssues, + policyFindings: review.policyFindings, + requiredCredentials: review.requiredCredentials, + requestedCapabilities: review.requestedCapabilities, + sideEffectDiffs: review.sideEffectDiffs, + result: { executed: false, inputKeys: Object.keys(input).sort(), output: null }, + }; + } + + publishDraft(projectId: string, flowId: string, draftRevision: number, publishedBy: string): NodeFlowDraftReview { + const flow = this.requireOwnedFlow(flowId, projectId); + if (flow.version !== draftRevision) throw new ValidationError(`Draft revision conflict: expected ${draftRevision}, actual ${flow.version}.`); + const review = this.reviewDraft(flow); + if (!review.valid) throw new ValidationError("Only a valid draft can be published."); + if (review.requiredCredentials.some((item) => item.status !== "bound")) { + throw new ValidationError("All required credentials must be bound before publication."); + } + this.repository.publishVersion(flowId, draftRevision, undefined, normalizeRequiredText(publishedBy, "publishedBy")); + return this.reviewDraft(flow); + } + + compareVersions(projectId: string, flowId: string, fromVersion: number, toVersion: number): Record { + this.requireOwnedFlow(flowId, projectId); + const from = this.repository.getVersion(flowId, fromVersion); + const to = this.repository.getVersion(flowId, toVersion); + if (!from || !to) throw new EntityNotFoundError("One or both node flow versions were not found."); + return { + flowId, fromVersion, toVersion, + nodeCount: { from: from.graph.nodes.length, to: to.graph.nodes.length }, + edgeCount: { from: from.graph.edges.length, to: to.graph.edges.length }, + addedNodeIds: to.graph.nodes.map((node) => node.id).filter((id) => !from.graph.nodes.some((node) => node.id === id)), + removedNodeIds: from.graph.nodes.map((node) => node.id).filter((id) => !to.graph.nodes.some((node) => node.id === id)), + sideEffectDiffs: reviewSideEffects(to.graph).filter((item) => !reviewSideEffects(from.graph).some((before) => before.nodeId === item.nodeId && before.sideEffect === item.sideEffect)), + }; + } + + rollback(projectId: string, flowId: string, version: number, draftRevision: number): NodeFlowDraftReview { + const current = this.requireOwnedFlow(flowId, projectId); + if (current.version !== draftRevision) throw new ValidationError(`Draft revision conflict: expected ${draftRevision}, actual ${current.version}.`); + const target = this.repository.getVersion(flowId, version); + if (!target) throw new EntityNotFoundError(`Node flow version not found: ${flowId}@${version}`); + return this.reviewDraft(this.repository.updateFlow(flowId, { + title: target.title, description: target.description, graph: target.graph, + }, { publish: false })); + } + + cancelRun(projectId: string, runId: string): NodeFlowRunRecord { + const run = this.requireOwnedRun(runId, projectId); + if (["succeeded", "failed", "cancelled"].includes(run.status)) throw new ValidationError(`Run ${runId} is already terminal.`); + return this.runtimeService?.requestCancellation(runId) ?? this.repository.requestCancellation(runId); + } + + async retryRun(projectId: string, runId: string): Promise { + const run = this.requireOwnedRun(runId, projectId); + if (!["failed", "cancelled", "attention_required"].includes(run.status)) throw new ValidationError("Only failed, cancelled, or attention-required runs can be retried."); + return await this.runFlow(projectId, run.flowId, run.input ?? {}, { + triggerType: "retry", + triggerPayload: { retriedRunId: run.id }, + versionSelection: { mode: "pinned", version: run.version }, + }); + } + update(flowId: string, input: UpdateNodeFlowInput): NodeFlowRecord { const update: UpdateNodeFlowInput = {}; if (input.title !== undefined) { @@ -127,6 +306,99 @@ export class NodeFlowService { options, ); } + + private requireOwnedFlow(flowId: string, projectId: string): NodeFlowRecord { + const flow = this.repository.getFlow(flowId); + if (!flow) throw new EntityNotFoundError(`Node flow not found: ${flowId}`); + if (flow.projectId !== projectId) throw new ValidationError("Node flow does not belong to the requested project."); + return flow; + } + + private requireOwnedRun(runId: string, projectId: string): NodeFlowRunRecord { + const run = this.repository.getRun(runId); + if (!run) throw new EntityNotFoundError(`Node flow run not found: ${runId}`); + if (run.projectId !== projectId) throw new ValidationError("Node flow run does not belong to the requested project."); + return run; + } + + private reviewDraft(flow: NodeFlowRecord, validation = validateNodeFlowGraph(flow.graph)): NodeFlowDraftReview { + const credentialMetadata = this.credentialBroker?.list(flow.projectId) ?? []; + const requiredCredentials: NodeFlowRequiredCredential[] = flow.graph.nodes.flatMap((node) => { + const definition = node.definition ? resolveNodeDefinition(node.definition.type, node.definition.version) : undefined; + return (definition?.credentials ?? []).filter((slot) => slot.required || node.credentialBindings?.some((binding) => binding.slot === slot.slot)).map((slot) => { + const credentialId = node.credentialBindings?.find((binding) => binding.slot === slot.slot)?.credentialId ?? null; + const credential = credentialMetadata.find((item) => item.id === credentialId); + const requiredCapabilities = ["read"]; + const allowed = credential && credential.status === "active" + && slot.allowedKinds.includes(credential.kind) + && requiredCapabilities.every((capability) => credential.capabilities.includes(capability)); + const status: NodeFlowRequiredCredential["status"] = credentialId ? (allowed ? "bound" : "denied") : "missing"; + return { nodeId: node.id, slot: slot.slot, allowedKinds: [...slot.allowedKinds], requiredCapabilities, required: slot.required, credentialId, status }; + }); + }); + const requestedCapabilities = [...new Set(flow.graph.nodes.flatMap((node) => node.capabilities ?? []))].sort(); + const policyFindings = reviewPolicy(flow.graph, requiredCredentials); + return { + flowId: flow.id, projectId: flow.projectId, name: flow.title, description: flow.description, + draftRevision: flow.version, nodeCount: flow.graph.nodes.length, edgeCount: flow.graph.edges.length, + valid: validation.valid && !policyFindings.some((finding) => finding.severity === "error"), + validationIssues: validation.errors, policyFindings, requiredCredentials, requestedCapabilities, + sideEffectDiffs: reviewSideEffects(flow.graph), + publishedVersion: this.repository.getPublication(flow.id)?.version ?? null, + }; + } + + private requireCustomNodeAuthoring(): NonNullable { + if (!this.customNodeAuthoring) throw new ValidationError("Custom-node authoring is not configured."); + return this.customNodeAuthoring; + } +} + +function definitionSummary(definition: ReturnType[number], includeSchema = false): Record { + return { + type: definition.type, version: definition.version, executable: definition.executable, + executionKind: definition.executionKind, label: definition.ui.label, description: definition.ui.description, + category: definition.ui.category, credentials: definition.credentials, capabilities: definition.capabilities, + sideEffect: definition.sideEffect, ports: definition.ports, + ...(includeSchema ? { configurationSchema: definition.configurationSchema, widgetSchema: definition.ui.widgetSchema, defaultPolicy: definition.defaultPolicy } : {}), + }; +} + +function applyGraphPatch(graph: NodeFlowGraph, operations: NodeFlowGraphPatchOperation[]): NodeFlowGraph { + let next: NodeFlowGraph = { ...graph, nodes: [...graph.nodes], edges: [...graph.edges] }; + for (const operation of operations) { + if (operation.op === "upsert_node") next.nodes = [...next.nodes.filter((node) => node.id !== operation.node.id), operation.node]; + else if (operation.op === "remove_node") { + next.nodes = next.nodes.filter((node) => node.id !== operation.nodeId); + next.edges = next.edges.filter((edge) => edge.fromNodeId !== operation.nodeId && edge.toNodeId !== operation.nodeId); + } else if (operation.op === "upsert_edge") { + const edgeKey = operation.edge.id ?? `${operation.edge.fromNodeId}:${operation.edge.toNodeId}:${operation.edge.fromHandle ?? ""}:${operation.edge.toHandle ?? ""}`; + next.edges = [...next.edges.filter((edge) => (edge.id ?? `${edge.fromNodeId}:${edge.toNodeId}:${edge.fromHandle ?? ""}:${edge.toHandle ?? ""}`) !== edgeKey), operation.edge]; + } else if (operation.op === "remove_edge") next.edges = next.edges.filter((edge) => operation.edgeId ? edge.id !== operation.edgeId : !(edge.fromNodeId === operation.fromNodeId && edge.toNodeId === operation.toNodeId)); + else if (operation.op === "set_input_schema") next = { ...next, inputSchema: operation.inputSchema ?? undefined }; + else if (operation.op === "set_metadata") next = { ...next, metadata: operation.metadata ?? undefined }; + } + return next; +} + +function reviewSideEffects(graph: NodeFlowGraph): NodeFlowDraftReview["sideEffectDiffs"] { + return graph.nodes.flatMap((node) => { + const definition = node.definition ? resolveNodeDefinition(node.definition.type, node.definition.version) : undefined; + const sideEffect = node.sideEffect ?? definition?.sideEffect ?? "none"; + return sideEffect === "none" ? [] : [{ nodeId: node.id, sideEffect, description: `${node.title} may perform a ${sideEffect} side effect.` }]; + }); +} + +function reviewPolicy(graph: NodeFlowGraph, credentials: NodeFlowDraftReview["requiredCredentials"]): NodeFlowDraftReview["policyFindings"] { + const findings: NodeFlowDraftReview["policyFindings"] = credentials.filter((item) => item.status === "denied" || (item.required && item.status === "missing")).map((item) => ({ + severity: "error", code: item.status === "missing" ? "missing_credential" : "credential_permission_denied", nodeId: item.nodeId, + message: `${item.nodeId}.${item.slot} requires an approved credential binding.`, + })); + for (const node of graph.nodes) { + const definition = node.definition ? resolveNodeDefinition(node.definition.type, node.definition.version) : undefined; + if ((node.sideEffect ?? definition?.sideEffect) === "external") findings.push({ severity: "warning", code: "external_side_effect", nodeId: node.id, message: "External side effects require publication review." }); + } + return findings; } function normalizeRequiredText(value: string | undefined, label: string): string { diff --git a/tests/backend/mcp/management-node-flow-actions.test.ts b/tests/backend/mcp/management-node-flow-actions.test.ts index dbd69abd52..f6292447e7 100644 --- a/tests/backend/mcp/management-node-flow-actions.test.ts +++ b/tests/backend/mcp/management-node-flow-actions.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { ManagementToolHandler } from "../../../src/mcp/management-tool-handler.js"; import type { NodeFlowGraph, NodeFlowRecord } from "../../../src/contracts/node-flow-types.js"; +import { runWithMcpAgentContext } from "../../../src/server/mcp-agent-context.js"; const validGraph: NodeFlowGraph = { nodes: [ @@ -205,4 +206,50 @@ describe("manage_node_flows", () => { visible: "ok", }); }); + + it("returns governed draft conflicts and structured dry-run findings", async () => { + const nodeFlowService = { + patchDraft: vi.fn(() => ({ conflict: { code: "draft_revision_conflict", expectedDraftRevision: 1, actualDraftRevision: 2 } })), + dryRun: vi.fn(() => ({ status: "blocked", validationIssues: [], policyFindings: [{ code: "missing_credential" }], requiredCredentials: [{ status: "missing" }], result: { executed: false } })), + }; + const handler = createHandler(nodeFlowService); + const conflict = parseResponse(await handler.handleManageNodeFlows({ action: "patch_draft", projectId: "project-1", flowId: "flow-1", draftRevision: 1, operations: [{ op: "set_metadata", metadata: {} }] })); + const dryRun = parseResponse(await handler.handleManageNodeFlows({ action: "dry_run", projectId: "project-1", flowId: "flow-1", input: { token: "never-returned" } })); + expect(conflict.result.conflict).toMatchObject({ code: "draft_revision_conflict", actualDraftRevision: 2 }); + expect(dryRun.result).toMatchObject({ status: "blocked", result: { executed: false } }); + }); + + it("requires exact approval for publish and rollback", async () => { + const nodeFlowService = { publishDraft: vi.fn(() => ({ draftRevision: 2 })), rollback: vi.fn(() => ({ draftRevision: 3 })) }; + const handler = createHandler(nodeFlowService); + const publishArgs = { action: "publish" as const, projectId: "project-1", flowId: "flow-1", draftRevision: 2, approval: { confirmed: true } }; + expect(parseResponse(await handler.handleManageNodeFlows(publishArgs)).approvalRequired).toBe(true); + expect(parseResponse(await handler.handleManageNodeFlows(publishArgs)).result.draft.draftRevision).toBe(2); + const rollbackArgs = { action: "rollback" as const, projectId: "project-1", flowId: "flow-1", draftRevision: 2, version: 1, approval: { confirmed: true } }; + expect(parseResponse(await handler.handleManageNodeFlows(rollbackArgs)).approvalRequired).toBe(true); + expect(parseResponse(await handler.handleManageNodeFlows(rollbackArgs)).result.draft.draftRevision).toBe(3); + }); + + it("validates required optimistic and operational fields", async () => { + const handler = createHandler({}); + const patch = await handler.handleManageNodeFlows({ action: "patch_draft", projectId: "project-1", flowId: "flow-1" }); + const cancel = await handler.handleManageNodeFlows({ action: "cancel", projectId: "project-1" }); + expect(parseResponse(patch).result).toMatchObject({ errorType: "validation", field: "draftRevision" }); + expect(parseResponse(cancel).result).toMatchObject({ errorType: "validation", field: "runId" }); + }); + + it("executes an attached flow with authenticated agent and conversation metadata", async () => { + const runFlow = vi.fn(async () => ({ run: { id: "run-1" }, nodeRuns: [], output: { ok: true } })); + const handler = createHandler({ + listAgentSkillsForAgent: () => [{ flowId: "flow-1", skillName: "Review", description: "" }], + get: () => ({ id: "flow-1", projectId: "project-1", graph: { nodes: [], edges: [] } }), + validateDraft: () => ({ publishedVersion: 1, requiredCredentials: [] }), + runFlow, + }); + const response = await runWithMcpAgentContext("agent-1", "thread-1", () => handler.handleRunAttachedFlow({ projectId: "project-1", flowId: "flow-1", input: { prompt: "review" } })); + expect(parseResponse(response).result.run.id).toBe("run-1"); + expect(runFlow).toHaveBeenCalledWith("project-1", "flow-1", { prompt: "review" }, expect.objectContaining({ + triggerPayload: expect.objectContaining({ initiatingAgentId: "agent-1", conversationId: "thread-1" }), + })); + }); }); diff --git a/tests/backend/mcp/mcp-management.test.ts b/tests/backend/mcp/mcp-management.test.ts index c2e63f6c59..b342f42e97 100644 --- a/tests/backend/mcp/mcp-management.test.ts +++ b/tests/backend/mcp/mcp-management.test.ts @@ -546,25 +546,45 @@ describe("ManagementToolHandler", () => { const properties = schema?.properties ?? {}; expect(properties.action?.enum).toEqual([ + "catalog", + "get_node_definition", + "create_draft", + "patch_draft", + "validate_draft", + "create_custom_node", + "update_custom_node", + "validate_custom_node", + "request_credential", + "inspect_bindings", + "dry_run", + "publish", + "compare_versions", + "rollback", + "run", + "cancel", + "retry", + "inspect_run", + "list_runs", "list", "get", "create", "update", "delete", "validate", - "run", - "list_runs", "get_run", "attach_to_agent", "detach_from_agent", + "attach", + "detach", ]); expect(properties.graph).toMatchObject({ type: "object" }); expect(properties.widgets).toMatchObject({ type: "object" }); expect(properties.input).toMatchObject({ type: "object" }); expect(properties.agentPresetId).toMatchObject({ type: "string" }); expect(properties.skillAlias).toMatchObject({ type: "string" }); - expect(tool?.description).toContain("Code UX-adapted flows"); - expect(tool?.description).toContain("dynamic widget schemas"); + expect(properties.draftRevision).toMatchObject({ type: "integer", minimum: 1 }); + expect(properties.operations).toMatchObject({ type: "array" }); + expect(tool?.description).toContain("Govern node-flow authoring"); }); it("exposes the dedicated long-term-memory MCP schema", () => { diff --git a/tests/backend/server/node-flow-routes.test.ts b/tests/backend/server/node-flow-routes.test.ts index 662fa498a1..f7a7a05f45 100644 --- a/tests/backend/server/node-flow-routes.test.ts +++ b/tests/backend/server/node-flow-routes.test.ts @@ -141,4 +141,14 @@ describe("node flow routes", () => { versionSelection: { mode: "latest_published" }, }); }); + + it("returns HTTP 409 for optimistic draft conflicts", async () => { + const nodeFlowService = { patchDraft: vi.fn(() => ({ conflict: { code: "draft_revision_conflict", expectedDraftRevision: 1, actualDraftRevision: 2 } })) }; + const app = express(); + app.use(express.json()); + registerNodeFlowRoutes(app, { nodeFlowService } as any); + const response = await request(app).patch("/api/node-flow-drafts/flow-1").send({ projectId: "project-1", draftRevision: 1, operations: [] }); + expect(response.status).toBe(409); + expect(response.body.conflict).toMatchObject({ code: "draft_revision_conflict", actualDraftRevision: 2 }); + }); }); diff --git a/tests/backend/services/agent-mcp-access.test.ts b/tests/backend/services/agent-mcp-access.test.ts index fe6c7d278e..1258b8c3a6 100644 --- a/tests/backend/services/agent-mcp-access.test.ts +++ b/tests/backend/services/agent-mcp-access.test.ts @@ -13,6 +13,7 @@ import { projectManagerClarificationAgentMcpAccess, toAgentCodeUxToolAccess, workerClarificationAgentMcpAccess, + withAttachedFlowAccess, } from "../../../src/services/agent-mcp-access.js"; import type { CustomMcpServer, McpToolToggle } from "../../../src/contracts/app-types.js"; import type { McpConnectionInfo } from "../../../src/contracts/mcp-connection-types.js"; @@ -56,6 +57,15 @@ describe("sanitizeAgentMcpAccess", () => { }); describe("agent MCP defaults", () => { + it("grants only the narrow attached-flow operation for flow-backed capabilities", () => { + const access = withAttachedFlowAccess({ codeUxEnabled: false, codeUxToolToggles: [] }); + expect(access.codeUxEnabled).toBe(true); + expect(access.audiences).toContain("worker"); + expect(access.codeUxToolToggles.filter((toggle) => toggle.enabled)).toEqual([ + { name: "run_attached_flow", enabled: true, isInternal: true }, + ]); + }); + it("recognizes assigned, manual, and worker-pool coding agents without admitting planning, QA, or unrelated agents", () => { const settings = { ...DEFAULT_DASHBOARD_SETTINGS, diff --git a/tests/backend/services/node-flow-agent-skill-service.test.ts b/tests/backend/services/node-flow-agent-skill-service.test.ts new file mode 100644 index 0000000000..4ae189f4a3 --- /dev/null +++ b/tests/backend/services/node-flow-agent-skill-service.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { NodeFlowAgentSkillService } from "../../../src/services/node-flow-agent-skill-service.js"; + +describe("NodeFlowAgentSkillService", () => { + it("advertises only narrow capability metadata and records initiating context", async () => { + const runFlow = vi.fn(async () => ({ run: { id: "run-1" }, nodeRuns: [], output: { ok: true } })); + const nodeFlowService = { + listAgentSkillsForAgent: vi.fn(() => [{ flowId: "flow-1", skillName: "Review", description: "Review input" }]), + get: vi.fn(() => ({ id: "flow-1", projectId: "project-1", graph: { nodes: [], edges: [], schemas: { input: { type: "object", properties: { prompt: { type: "string" } } } } } })), + validateDraft: vi.fn(() => ({ publishedVersion: 2, requiredCredentials: [] })), + runFlow, + }; + const service = new NodeFlowAgentSkillService(nodeFlowService as never); + + expect(service.listCapabilities("project-1", "agent-1")).toEqual([{ + flowId: "flow-1", name: "Review", description: "Review input", + inputSchema: { type: "object", properties: { prompt: { type: "string" } } }, operation: "run_attached_flow", + }]); + await service.runAttachedFlow({ projectId: "project-1", flowId: "flow-1", agentPresetId: "agent-1", conversationId: "thread-1", parameters: { prompt: "safe" } }); + expect(runFlow).toHaveBeenCalledWith("project-1", "flow-1", { prompt: "safe" }, expect.objectContaining({ + triggerType: "attached_flow", + triggerPayload: { initiatingAgentId: "agent-1", conversationId: "thread-1", operation: "run_attached_flow" }, + })); + expect(JSON.stringify(service.listCapabilities("project-1", "agent-1"))).not.toContain("credential"); + }); + + it("rejects unattached, unpublished, and credential-blocked flows", async () => { + const base = { get: vi.fn(() => ({ id: "flow-1", projectId: "project-1", graph: { nodes: [], edges: [] } })), runFlow: vi.fn() }; + await expect(new NodeFlowAgentSkillService({ ...base, listAgentSkillsForAgent: () => [] } as never).runAttachedFlow({ projectId: "project-1", flowId: "flow-1", agentPresetId: "agent-1" })).rejects.toThrow(/not attached/i); + await expect(new NodeFlowAgentSkillService({ ...base, listAgentSkillsForAgent: () => [{ flowId: "flow-1", skillName: "x", description: "" }], validateDraft: () => ({ publishedVersion: null, requiredCredentials: [] }) } as never).runAttachedFlow({ projectId: "project-1", flowId: "flow-1", agentPresetId: "agent-1" })).rejects.toThrow(/not been published/i); + await expect(new NodeFlowAgentSkillService({ ...base, listAgentSkillsForAgent: () => [{ flowId: "flow-1", skillName: "x", description: "" }], validateDraft: () => ({ publishedVersion: 1, requiredCredentials: [{ status: "missing" }] }) } as never).runAttachedFlow({ projectId: "project-1", flowId: "flow-1", agentPresetId: "agent-1" })).rejects.toThrow(/credential policy/i); + }); +}); diff --git a/tests/backend/services/node-flow-service.test.ts b/tests/backend/services/node-flow-service.test.ts index 1560a2d47b..c893c86983 100644 --- a/tests/backend/services/node-flow-service.test.ts +++ b/tests/backend/services/node-flow-service.test.ts @@ -90,4 +90,39 @@ describe("NodeFlowService", () => { expect.objectContaining({ code: "invalid_edge_endpoint" }), ])); }); + + it("applies optimistic draft patches without overwriting conflicting edits", async () => { + const { dir, projectRepository, service } = await createService(); + const project = projectRepository.createProject({ name: "Draft Project", sourceType: "local", sourceRef: dir }); + const draft = service.createDraft(project.id, { title: "Draft", graph: validGraph() }); + + const updated = service.patchDraft(draft.flowId, { + projectId: project.id, + draftRevision: draft.draftRevision, + operations: [{ op: "set_metadata", metadata: { purpose: "review" } }], + }); + const conflict = service.patchDraft(draft.flowId, { + projectId: project.id, + draftRevision: draft.draftRevision, + operations: [{ op: "set_metadata", metadata: { purpose: "overwrite" } }], + }); + + expect(updated.draft?.draftRevision).toBe(2); + expect(conflict.conflict).toMatchObject({ code: "draft_revision_conflict", expectedDraftRevision: 1, actualDraftRevision: 2 }); + expect(service.get(draft.flowId)?.graph.metadata).toEqual({ purpose: "review" }); + }); + + it("publishes reviewed drafts and creates rollback drafts without replacing history", async () => { + const { dir, projectRepository, service } = await createService(); + const project = projectRepository.createProject({ name: "Publish Project", sourceType: "local", sourceRef: dir }); + const draft = service.createDraft(project.id, { title: "Draft", graph: validGraph() }); + expect(draft.publishedVersion).toBeNull(); + const published = service.publishDraft(project.id, draft.flowId, 1, "reviewer"); + expect(published.publishedVersion).toBe(1); + const second = service.patchDraft(draft.flowId, { projectId: project.id, draftRevision: 1, title: "Second" }); + const rollback = service.rollback(project.id, draft.flowId, 1, second.draft!.draftRevision); + expect(rollback.draftRevision).toBe(3); + expect(rollback.name).toBe("Draft"); + expect(service.compareVersions(project.id, draft.flowId, 1, 3)).toMatchObject({ fromVersion: 1, toVersion: 3 }); + }); }); From c68c5014a361c7226767e1d0741e2c376a8d4bbe Mon Sep 17 00:00:00 2001 From: Code UX Date: Sun, 12 Jul 2026 06:33:12 +0000 Subject: [PATCH 07/25] feat(task T07): implement via codex --- dashboard/src/v2/NodesPage.tsx | 478 +++++------------- .../v2/components/nodes/NodeFlowInspector.tsx | 39 +- .../components/nodes/NodeGovernancePanel.tsx | 30 ++ .../src/v2/components/nodes/NodePalette.tsx | 127 ++--- .../v2/components/nodes/NodeRunDebugger.tsx | 17 + .../src/v2/lib/dashboard-feature-flags.ts | 13 +- dashboard/src/v2/lib/node-flow-api.ts | 111 ++++ dashboard/src/v2/types.ts | 16 + docs/dashboard/feature-flags.md | 4 +- docs/dashboard/node-flows.md | 96 +--- docs/dashboard/nodes-canvas.md | 80 +-- .../lib/dashboard-feature-flags.test.ts | 3 +- tests/dashboard/v2/node-flow-page.test.tsx | 47 +- tests/dashboard/v2/nodes-inspector.test.tsx | 18 +- tests/dashboard/v2/nodes-page.test.tsx | 202 ++------ 15 files changed, 451 insertions(+), 830 deletions(-) create mode 100644 dashboard/src/v2/components/nodes/NodeGovernancePanel.tsx create mode 100644 dashboard/src/v2/components/nodes/NodeRunDebugger.tsx diff --git a/dashboard/src/v2/NodesPage.tsx b/dashboard/src/v2/NodesPage.tsx index fe086346c6..f550fa51fc 100644 --- a/dashboard/src/v2/NodesPage.tsx +++ b/dashboard/src/v2/NodesPage.tsx @@ -1,374 +1,134 @@ import type { FunctionComponent } from "preact"; -import { useCallback, useEffect, useMemo, useState } from "preact/hooks"; -import { - AlertTriangle, - CheckCircle2, - ClipboardList, - Download, - FileJson, - RefreshCcw, - RotateCcw, - Upload, - Workflow, -} from "lucide-preact"; +import { useCallback, useEffect, useMemo, useRef, useState } from "preact/hooks"; +import { AlertTriangle, Plus, Save, Workflow } from "lucide-preact"; import { PageContainer } from "./components/layout/PageContainer.js"; import { PageHeader } from "./components/layout/PageHeader.js"; -import { NodeCanvas } from "./components/nodes/NodeCanvas.js"; -import { NodeInspector } from "./components/nodes/NodeInspector.js"; -import { NodePalette } from "./components/nodes/NodePalette.js"; -import { NodeValidationPanel } from "./components/nodes/NodeValidationPanel.js"; +import { EmptyState } from "./components/ui/EmptyState.js"; import { Button } from "./components/ui/Button.js"; -import type { - NodeCanvasConfigValue, - NodeCanvasEdge, - NodeCanvasGraph, - NodeCanvasNode, - NodesCanvasAction, -} from "./lib/nodes-canvas-state.js"; -import { - createInitialNodeCanvasGraph, - nodesCanvasReducer, - serializeNodeCanvasGraph, - validateNodeCanvasGraph, -} from "./lib/nodes-canvas-state.js"; +import { NodeFlowLibrary } from "./components/nodes/NodeFlowLibrary.js"; +import { NodeFlowCanvas } from "./components/nodes/NodeFlowCanvas.js"; +import { NodeFlowInspector } from "./components/nodes/NodeFlowInspector.js"; +import { NodePalette } from "./components/nodes/NodePalette.js"; +import { NodeGovernancePanel } from "./components/nodes/NodeGovernancePanel.js"; +import { NodeRunDebugger } from "./components/nodes/NodeRunDebugger.js"; +import { useProjectData } from "./context/project-data.js"; +import type { NodeDefinitionManifest, NodeFlowDraftReview, NodeFlowGraph, NodeFlowNode, NodeFlowNodeAttemptRecord, NodeFlowNodeRunRecord, NodeFlowRecord, NodeFlowRunRecord } from "./types.js"; +import { createDefaultNodeFlowGraph, isNodeFlowDirty, updateNodeInGraph } from "./lib/node-flow-view-models.js"; +import { deserializeNodeCanvasGraphWithMigration, toCanonicalNodeFlowGraph } from "./lib/nodes-canvas-state.js"; import { - applyNodeCanvasAgentCommand, - buildNodeCanvasAgentSummary, -} from "./lib/nodes-agent-surface.js"; + cancelNodeFlowRun, compareNodeFlowVersions, createNodeFlowDraft, deleteNodeFlow, dryRunNodeFlowDraft, + fetchNodeDefinition, fetchNodeFlow, fetchNodeFlowAttempts, fetchNodeFlowCatalog, fetchNodeFlowNodeRuns, + fetchNodeFlowRuns, fetchNodeFlows, patchNodeFlowDraft, publishNodeFlowDraft, requestNodeFlowCredential, + retryNodeFlowRun, rollbackNodeFlow, runNodeFlow, validateNodeFlowDraft, + type NodeDefinitionSummary, type NodeFlowDryRunResponse, type NodeFlowVersionDiff, +} from "./lib/node-flow-api.js"; export const NODES_CANVAS_STORAGE_KEY = "codeux:nodes-canvas:v1"; - -type FeedbackTone = "success" | "error" | "info" | "warning"; - -interface FeedbackState { - tone: FeedbackTone; - message: string; -} - -const panelClass = "rounded-[var(--radius-panel)] border border-black/[0.06] bg-white/70 p-4 shadow-[var(--elevation-soft)] dark:border-white/[0.06] dark:bg-white/[0.035]"; - -const toneClasses: Record = { - success: "border-status-green/20 bg-status-green/[0.08] text-slate-700 dark:text-slate-200", - error: "border-status-red/25 bg-status-red/[0.08] text-slate-700 dark:text-slate-200", - info: "border-signal-500/20 bg-signal-500/[0.08] text-slate-700 dark:text-slate-200", - warning: "border-amber-500/25 bg-amber-500/[0.08] text-slate-700 dark:text-slate-200", -}; - -const loadPersistedGraph = (): NodeCanvasGraph => { - if (typeof window === "undefined") { - return createInitialNodeCanvasGraph(); - } - - const persisted = window.localStorage.getItem(NODES_CANVAS_STORAGE_KEY); - if (!persisted) { - return createInitialNodeCanvasGraph(); - } - - const result = applyNodeCanvasAgentCommand(createInitialNodeCanvasGraph(), { - command: "replace_graph", - serializedGraph: persisted, - }); - - return result.issues.some((issue) => issue.field === "serializedGraph") - ? createInitialNodeCanvasGraph() - : result.graph; -}; - -const selectedNodeFromGraph = (graph: NodeCanvasGraph): NodeCanvasNode | null => { - const selectedId = graph.selection.nodeIds[0]; - return selectedId ? graph.nodes.find((node) => node.id === selectedId) ?? null : null; -}; - -const selectedEdgeFromGraph = (graph: NodeCanvasGraph): NodeCanvasEdge | null => { - const selectedId = graph.selection.edgeIds[0]; - return selectedId ? graph.edges.find((edge) => edge.id === selectedId) ?? null : null; -}; - -const updateNodePatch = ( - graph: NodeCanvasGraph, - nodeId: string, - patch: Partial>, -): NodeCanvasGraph => { - let nextGraph = graph; - if (patch.label !== undefined) { - nextGraph = nodesCanvasReducer(nextGraph, { type: "update_node_label", nodeId, label: patch.label }); - } - - if (patch.description !== undefined || patch.metadata !== undefined) { - nextGraph = nodesCanvasReducer(nextGraph, { - type: "replace_graph", - graph: { - ...nextGraph, - nodes: nextGraph.nodes.map((node) => node.id === nodeId - ? { - ...node, - ...(patch.description !== undefined ? { description: patch.description } : {}), - ...(patch.metadata !== undefined ? { metadata: { ...node.metadata, ...patch.metadata } } : {}), - } - : node), - }, - }); - } - - return nextGraph; -}; - -const formatSummaryJson = (graph: NodeCanvasGraph): string => ( - JSON.stringify(buildNodeCanvasAgentSummary(graph), null, 2) -); +const migrationMarker = (projectId: string): string => `codeux:nodes-canvas:imported:${projectId}`; +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : "The node-flow request failed."; export const NodesPage: FunctionComponent = () => { - const [graph, setGraph] = useState(() => loadPersistedGraph()); - const [exchangeJson, setExchangeJson] = useState(() => serializeNodeCanvasGraph(graph)); - const [feedback, setFeedback] = useState({ - tone: "info", - message: "Canvas restored from local browser storage.", - }); - const [enabledNodeIds, setEnabledNodeIds] = useState>(() => new Set(graph.nodes.map((node) => node.id))); - - const validationIssues = useMemo(() => validateNodeCanvasGraph(graph), [graph]); - const summary = useMemo(() => buildNodeCanvasAgentSummary(graph), [graph]); - const selectedNode = useMemo(() => selectedNodeFromGraph(graph), [graph]); - const selectedEdge = useMemo(() => selectedEdgeFromGraph(graph), [graph]); - const serializedGraph = useMemo(() => serializeNodeCanvasGraph(graph), [graph]); - const summaryJson = useMemo(() => formatSummaryJson(graph), [graph]); - - useEffect(() => { - if (typeof window === "undefined") { - return; - } - window.localStorage.setItem(NODES_CANVAS_STORAGE_KEY, serializedGraph); - }, [serializedGraph]); - - const dispatch = useCallback((action: NodesCanvasAction): void => { - setGraph((current) => nodesCanvasReducer(current, action)); + const { selectedProject, loading: projectLoading } = useProjectData(); + const projectId = selectedProject?.id ?? null; + const projectRef = useRef(projectId); projectRef.current = projectId; + const [flows, setFlows] = useState([]); + const [catalog, setCatalog] = useState([]); + const [selectedFlowId, setSelectedFlowId] = useState(null); + const [record, setRecord] = useState(null); + const [title, setTitle] = useState(""); + const [description, setDescription] = useState(""); + const [graph, setGraph] = useState(createDefaultNodeFlowGraph); + const [selectedNodeId, setSelectedNodeId] = useState(null); + const [definitions, setDefinitions] = useState>({}); + const [review, setReview] = useState(null); + const [dryRun, setDryRun] = useState(null); + const [diff, setDiff] = useState(null); + const [runs, setRuns] = useState([]); + const [selectedRunId, setSelectedRunId] = useState(null); + const [nodeRuns, setNodeRuns] = useState([]); + const [attempts, setAttempts] = useState([]); + const [loading, setLoading] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + + const selectedNode = useMemo(() => graph.nodes.find((node) => node.id === selectedNodeId) ?? null, [graph.nodes, selectedNodeId]); + const selectedDefinition = selectedNode?.definition ? definitions[`${selectedNode.definition.type}@${selectedNode.definition.version}`] ?? null : null; + const dirty = isNodeFlowDirty(record, title, description, graph); + + const applyRecord = useCallback((flow: NodeFlowRecord): void => { + setRecord(flow); setSelectedFlowId(flow.id); setTitle(flow.title); setDescription(flow.description); setGraph(flow.graph); + setSelectedNodeId(flow.graph.nodes[0]?.id ?? null); setReview(null); setDryRun(null); setDiff(null); }, []); - const handleExport = (): void => { - setExchangeJson(serializedGraph); - setFeedback({ - tone: validationIssues.length === 0 ? "success" : "warning", - message: `Exported ${summary.nodeCount} nodes and ${summary.edgeCount} edges as deterministic JSON.`, - }); - }; - - const handleImport = (): void => { - const result = applyNodeCanvasAgentCommand(graph, { - command: "replace_graph", - serializedGraph: exchangeJson, - }); - const importIssues = result.issues.filter((issue) => issue.entityId.startsWith("command[0]")); - if (importIssues.length > 0) { - setFeedback({ tone: "error", message: importIssues[0]?.message ?? "Import failed." }); - return; - } - - setGraph(result.graph); - setEnabledNodeIds(new Set(result.graph.nodes.map((node) => node.id))); - const nextIssueCount = validateNodeCanvasGraph(result.graph).length; - setFeedback({ - tone: nextIssueCount === 0 ? "success" : "warning", - message: nextIssueCount === 0 - ? "Imported graph JSON and saved it locally." - : `Imported graph JSON with ${nextIssueCount} validation issue${nextIssueCount === 1 ? "" : "s"}.`, - }); - }; - - const handleReset = (): void => { - const nextGraph = createInitialNodeCanvasGraph(); - setGraph(nextGraph); - setExchangeJson(serializeNodeCanvasGraph(nextGraph)); - setEnabledNodeIds(new Set(nextGraph.nodes.map((node) => node.id))); - setFeedback({ tone: "info", message: "Canvas reset to the starter workflow." }); - }; - - const handleClear = (): void => { - setGraph({ nodes: [], edges: [], selection: { nodeIds: [], edgeIds: [] } }); - setEnabledNodeIds(new Set()); - setFeedback({ tone: "info", message: "Canvas cleared. Add a node from the palette to start again." }); - }; - - const handleNodeChange = ( - nodeId: string, - patch: Partial>, - ): void => { - setGraph((current) => updateNodePatch(current, nodeId, patch)); - }; - - const handleNodeConfigChange = (nodeId: string, fieldId: string, value: NodeCanvasConfigValue): void => { - dispatch({ type: "update_node_config", nodeId, fieldId, value }); - }; - - const handleNodeEnabledChange = (nodeId: string, enabled: boolean): void => { - setEnabledNodeIds((current) => { - const next = new Set(current); - if (enabled) { - next.add(nodeId); - } else { - next.delete(nodeId); + const loadLibrary = useCallback(async (nextProjectId: string, signal?: AbortSignal): Promise => { + setLoading(true); setError(null); + try { + const [library, registry] = await Promise.all([fetchNodeFlows(nextProjectId, signal), fetchNodeFlowCatalog(signal)]); + if (projectRef.current !== nextProjectId) return; + let nextFlows = library.flows; + const legacy = typeof window !== "undefined" ? window.localStorage.getItem(NODES_CANVAS_STORAGE_KEY) : null; + if (legacy && !window.localStorage.getItem(migrationMarker(nextProjectId))) { + const importedGraph = toCanonicalNodeFlowGraph(deserializeNodeCanvasGraphWithMigration(legacy).graph); + const imported = await createNodeFlowDraft(nextProjectId, { title: "Imported Nodes Canvas", description: "One-time import from the legacy browser canvas.", graph: importedGraph }); + window.localStorage.setItem(migrationMarker(nextProjectId), imported.flowId); + window.localStorage.removeItem(NODES_CANVAS_STORAGE_KEY); + nextFlows = (await fetchNodeFlows(nextProjectId, signal)).flows; + setNotice("Legacy canvas imported once into this project's backend flow library."); } - return next; - }); - setFeedback({ tone: "info", message: `${enabled ? "Enabled" : "Disabled"} ${nodeId} for this editing session.` }); - }; - - const selectNode = (nodeId: string, append = false): void => dispatch({ type: "select_node", nodeId, append }); - const selectEdge = (edgeId: string, append = false): void => dispatch({ type: "select_edge", edgeId, append }); - - const statusIcon = validationIssues.length === 0 ? CheckCircle2 : AlertTriangle; - const StatusIcon = statusIcon; - - return ( - - Nodes Canvas} - subtitle="Compose local workflow graphs with typed nodes, reducer-backed edits, validation feedback, and JSON exchange for agents." - actions={( - <> - - - - - - )} - /> - -
-
-
-
-
-

- {validationIssues.length === 0 ? "Graph is structurally valid" : `${validationIssues.length} validation issue${validationIssues.length === 1 ? "" : "s"}`} -

-

- {summary.nodeCount} nodes, {summary.edgeCount} edges, {summary.selectedNodeIds.length + summary.selectedEdgeIds.length} selected. Saved locally under {NODES_CANVAS_STORAGE_KEY}. -

-
-
-

- {feedback.message} -

-
+ setCatalog(registry.nodes); setFlows(nextFlows); + const preferred = nextFlows.find((flow) => flow.id === selectedFlowId) ?? nextFlows[0] ?? null; + if (preferred) { + applyRecord(preferred); + setReview(await validateNodeFlowDraft(nextProjectId, preferred.id)); + } else { setRecord(null); setSelectedFlowId(null); setReview(null); } + } catch (requestError) { if (!signal?.aborted) setError(errorMessage(requestError)); } + finally { if (!signal?.aborted && projectRef.current === nextProjectId) setLoading(false); } + }, [applyRecord, selectedFlowId]); -
- { - dispatch(action); - setFeedback({ tone: "success", message: `Added ${action.kind} node.` }); - }} - /> - -
- dispatch({ type: "clear_selection" })} - onDeleteNode={(nodeId) => dispatch({ type: "delete_node", nodeId })} - onDeleteEdge={(edgeId) => dispatch({ type: "delete_edge", edgeId })} - onMoveNode={(nodeId, position) => dispatch({ type: "move_node", nodeId, position })} - /> - {graph.nodes.length === 0 ? ( -
-
- ) : null} - selectNode(nodeId)} - onSelectEdge={(edgeId) => selectEdge(edgeId)} - onFocusNode={(nodeId) => { - selectNode(nodeId); - setFeedback({ tone: "info", message: `Selected node ${nodeId}.` }); - }} - onFocusEdge={(edgeId) => { - selectEdge(edgeId); - setFeedback({ tone: "info", message: `Selected edge ${edgeId}.` }); - }} - /> -
- -
- selectNode(nodeId)} - onSelectEdge={(edgeId) => selectEdge(edgeId)} - /> - -
-
-
-

Exchange

-

Graph JSON

-
-
- -