From 0f624310741f3fdf3978fbfdcd8ecee8f2b46f51 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Fri, 28 Nov 2025 10:04:28 -0600 Subject: [PATCH 1/9] Workflows CFG extractor & build plumbing --- packages/builders/src/base-builder.ts | 55 +- packages/builders/src/standalone.ts | 24 +- packages/builders/src/workflows-extractor.ts | 1168 ++++++++++++++++++ packages/next/src/builder.ts | 57 +- workbench/example/workflows/99_e2e.ts | 4 +- 5 files changed, 1300 insertions(+), 8 deletions(-) create mode 100644 packages/builders/src/workflows-extractor.ts diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index b0929873b..795935cbb 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -26,6 +26,7 @@ const EMIT_SOURCEMAPS_FOR_DEBUGGING = */ export abstract class BaseBuilder { protected config: WorkflowConfig; + protected lastWorkflowManifest?: WorkflowManifest; constructor(config: WorkflowConfig) { this.config = config; @@ -253,6 +254,7 @@ export abstract class BaseBuilder { * Steps have full Node.js runtime access and handle side effects, API calls, etc. * * @param externalizeNonSteps - If true, only bundles step entry points and externalizes other code + * @returns Build context (for watch mode) and the collected workflow manifest */ protected async createStepsBundle({ inputFiles, @@ -268,7 +270,10 @@ export abstract class BaseBuilder { outfile: string; format?: 'cjs' | 'esm'; externalizeNonSteps?: boolean; - }): Promise { + }): Promise<{ + context: esbuild.BuildContext | undefined; + manifest: WorkflowManifest; + }> { // These need to handle watching for dev to scan for // new entries and changes to existing ones const { discoveredSteps: stepFiles } = await this.discoverEntries( @@ -389,10 +394,14 @@ export abstract class BaseBuilder { // Create .gitignore in .swc directory await this.createSwcGitignore(); + // Store the manifest for later use (e.g., graph generation in watch mode) + this.lastWorkflowManifest = workflowManifest; + if (this.config.watch) { - return esbuildCtx; + return { context: esbuildCtx, manifest: workflowManifest }; } await esbuildCtx.dispose(); + return { context: undefined, manifest: workflowManifest }; } /** @@ -838,4 +847,46 @@ export const OPTIONS = handler;`; // We're intentionally silently ignoring this error - creating .gitignore isn't critical } } + + /** + * Creates a workflows manifest JSON file by extracting from the bundled workflow file. + * The manifest contains React Flow-compatible graph data for visualizing workflows. + */ + protected async createWorkflowsManifest({ + workflowBundlePath, + outfile, + }: { + workflowBundlePath: string; + outfile: string; + }): Promise { + const buildStart = Date.now(); + console.log('Creating workflows manifest...'); + + try { + // Import the graph extractor + const { extractGraphFromBundle } = await import( + './workflows-extractor.js' + ); + + // Extract graph from the bundled workflow file + const workflowsManifest = + await extractGraphFromBundle(workflowBundlePath); + + // Write the workflows manifest + await this.ensureDirectory(outfile); + await writeFile(outfile, JSON.stringify(workflowsManifest, null, 2)); + + console.log( + `Created workflows manifest with ${ + Object.keys(workflowsManifest.workflows).length + } workflow(s)`, + `${Date.now() - buildStart}ms` + ); + } catch (error) { + console.warn( + 'Failed to extract workflows from bundle:', + error instanceof Error ? error.message : String(error) + ); + } + } } diff --git a/packages/builders/src/standalone.ts b/packages/builders/src/standalone.ts index c4ef2c936..11f6438af 100644 --- a/packages/builders/src/standalone.ts +++ b/packages/builders/src/standalone.ts @@ -14,6 +14,10 @@ export class StandaloneBuilder extends BaseBuilder { await this.buildWorkflowsBundle(options); await this.buildWebhookFunction(); + // Build workflows manifest from workflow bundle (post-bundle extraction) + const workflowBundlePath = this.resolvePath('.swc/workflows.js'); + await this.buildWorkflowsManifest({ workflowBundlePath }); + await this.createClientLibrary(); } @@ -25,18 +29,20 @@ export class StandaloneBuilder extends BaseBuilder { inputFiles: string[]; tsBaseUrl?: string; tsPaths?: Record; - }): Promise { + }) { console.log('Creating steps bundle at', this.config.stepsBundlePath); const stepsBundlePath = this.resolvePath(this.config.stepsBundlePath); await this.ensureDirectory(stepsBundlePath); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ outfile: stepsBundlePath, inputFiles, tsBaseUrl, tsPaths, }); + + return manifest; } private async buildWorkflowsBundle({ @@ -76,4 +82,18 @@ export class StandaloneBuilder extends BaseBuilder { outfile: webhookBundlePath, }); } + + private async buildWorkflowsManifest({ + workflowBundlePath, + }: { + workflowBundlePath: string; + }): Promise { + const workflowsManifestPath = this.resolvePath('.swc/workflows.json'); + await this.ensureDirectory(workflowsManifestPath); + + await this.createWorkflowsManifest({ + workflowBundlePath, + outfile: workflowsManifestPath, + }); + } } diff --git a/packages/builders/src/workflows-extractor.ts b/packages/builders/src/workflows-extractor.ts new file mode 100644 index 000000000..52358b92d --- /dev/null +++ b/packages/builders/src/workflows-extractor.ts @@ -0,0 +1,1168 @@ +import { readFile } from 'node:fs/promises'; +import type { + ArrowFunctionExpression, + BlockStatement, + CallExpression, + Expression, + FunctionDeclaration, + FunctionExpression, + Identifier, + MemberExpression, + Program, + Statement, + VariableDeclaration, +} from '@swc/core'; +import { parseSync } from '@swc/core'; + +/** + * Represents a function in the bundle (can be declaration, expression, or arrow) + */ +interface FunctionInfo { + name: string; + body: BlockStatement | Expression | null | undefined; + isStep: boolean; + stepId?: string; +} + +/** + * Graph manifest structure + */ +export interface WorkflowsManifest { + version: string; + workflows: Record; + debugInfo?: DebugInfo; +} + +export interface WorkflowGraph { + workflowId: string; + workflowName: string; + filePath: string; + nodes: GraphNode[]; + edges: GraphEdge[]; +} + +export interface GraphNode { + id: string; + type: string; + position: { x: number; y: number }; + data: { + label: string; + nodeKind: string; + stepId?: string; + line: number; + }; + metadata?: NodeMetadata; +} + +export interface NodeMetadata { + loopId?: string; + loopIsAwait?: boolean; + conditionalId?: string; + conditionalBranch?: 'Then' | 'Else'; + parallelGroupId?: string; + parallelMethod?: string; + /** Step is passed as a reference (callback/tool) rather than directly called */ + isStepReference?: boolean; + /** Context where the step reference was found (e.g., "tools.getWeather.execute") */ + referenceContext?: string; +} + +export interface GraphEdge { + id: string; + source: string; + target: string; + type: 'default' | 'loop' | 'conditional' | 'parallel'; + label?: string; +} + +export interface DebugInfo { + manifestPresent?: boolean; + manifestStepFiles?: number; + importsResolved?: number; + importsWithKind?: number; + importDetails?: Array<{ + localName: string; + source: string; + importedName: string; + kind?: string; + lookupCandidates: string[]; + }>; + error?: string; +} + +/** + * Extracts workflow graph from a bundled workflow file + */ +export async function extractGraphFromBundle( + bundlePath: string +): Promise { + const bundleCode = await readFile(bundlePath, 'utf-8'); + + try { + // The workflow bundle wraps the actual code in a template literal: + // const workflowCode = `...`; + // We need to parse the bundle AST first to properly extract the unescaped string + let actualWorkflowCode = bundleCode; + + // First, try to parse the bundle itself to extract workflowCode properly + const bundleAst = parseSync(bundleCode, { + syntax: 'ecmascript', + target: 'es2022', + }); + + // Find the workflowCode variable declaration + const workflowCodeValue = extractWorkflowCodeFromBundle(bundleAst); + if (workflowCodeValue) { + actualWorkflowCode = workflowCodeValue; + } + + // Now parse the actual workflow code + const ast = parseSync(actualWorkflowCode, { + syntax: 'ecmascript', + target: 'es2022', + }); + + // Extract step declarations + const stepDeclarations = extractStepDeclarations(actualWorkflowCode); + + // Build a map of ALL functions in the bundle (for transitive step resolution) + const functionMap = buildFunctionMap(ast, stepDeclarations); + + // Extract workflows with transitive step resolution + const workflows = extractWorkflows(ast, stepDeclarations, functionMap); + + return { + version: '1.0.0', + workflows, + }; + } catch (error) { + console.error('Failed to extract graph from bundle:', error); + // Return empty manifest on parsing errors + return { + version: '1.0.0', + workflows: {}, + debugInfo: { + error: error instanceof Error ? error.message : String(error), + }, + }; + } +} + +/** + * Extract the workflowCode string value from a parsed bundle AST + */ +function extractWorkflowCodeFromBundle(ast: Program): string | null { + for (const item of ast.body) { + if (item.type === 'VariableDeclaration') { + for (const decl of item.declarations) { + if ( + decl.id.type === 'Identifier' && + decl.id.value === 'workflowCode' && + decl.init + ) { + // Handle template literal + if (decl.init.type === 'TemplateLiteral') { + // Concatenate all quasis (the string parts of template literal) + return decl.init.quasis.map((q) => q.cooked || q.raw).join(''); + } + // Handle regular string literal + if (decl.init.type === 'StringLiteral') { + return decl.init.value; + } + } + } + } + } + return null; +} + +/** + * Extract step declarations using regex for speed + */ +function extractStepDeclarations( + bundleCode: string +): Map { + const stepDeclarations = new Map(); + + // Match: var stepName = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//path//name"); + const stepPattern = + /var (\w+) = globalThis\[Symbol\.for\("WORKFLOW_USE_STEP"\)\]\("([^"]+)"\)/g; + + // Track line numbers + const lines = bundleCode.split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + stepPattern.lastIndex = 0; + const match = stepPattern.exec(line); + if (match) { + const [, varName, stepId] = match; + stepDeclarations.set(varName, { + stepId, + line: i + 1, + }); + } + } + + return stepDeclarations; +} + +/** + * Build a map of all functions in the bundle for transitive step resolution + */ +function buildFunctionMap( + ast: Program, + stepDeclarations: Map +): Map { + const functionMap = new Map(); + + for (const item of ast.body) { + // Handle function declarations: function foo() {} + if (item.type === 'FunctionDeclaration') { + const func = item as FunctionDeclaration; + if (func.identifier) { + const name = func.identifier.value; + const isStep = stepDeclarations.has(name); + functionMap.set(name, { + name, + body: func.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } + } + + // Handle variable declarations: const foo = function() {} or const foo = () => {} + if (item.type === 'VariableDeclaration') { + const varDecl = item as VariableDeclaration; + for (const decl of varDecl.declarations) { + if (decl.id.type === 'Identifier' && decl.init) { + const name = decl.id.value; + const isStep = stepDeclarations.has(name); + + if (decl.init.type === 'FunctionExpression') { + const funcExpr = decl.init as FunctionExpression; + functionMap.set(name, { + name, + body: funcExpr.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } else if (decl.init.type === 'ArrowFunctionExpression') { + const arrowFunc = decl.init as ArrowFunctionExpression; + functionMap.set(name, { + name, + body: arrowFunc.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } + } + } + } + } + + return functionMap; +} + +/** + * Extract workflows from AST + */ +function extractWorkflows( + ast: Program, + stepDeclarations: Map, + functionMap: Map +): Record { + const workflows: Record = {}; + + // Find all function declarations + for (const item of ast.body) { + if (item.type === 'FunctionDeclaration') { + const func = item as FunctionDeclaration; + if (!func.identifier) continue; + + const workflowName = func.identifier.value; + + // Check if this function has a workflowId property assignment + // Look for: functionName.workflowId = "workflow//path//name"; + const workflowId = findWorkflowId(ast, workflowName); + if (!workflowId) continue; + + // Extract file path from workflowId + // Format: "workflow//path/to/file.ts//functionName" + const parts = workflowId.split('//'); + const filePath = parts.length > 1 ? parts[1] : ''; + + // Analyze the function body with transitive step resolution + const graph = analyzeWorkflowFunction( + func, + workflowName, + workflowId, + filePath, + stepDeclarations, + functionMap + ); + + workflows[workflowName] = graph; + } + } + + return workflows; +} + +/** + * Find workflowId assignment for a function + */ +function findWorkflowId(ast: Program, functionName: string): string | null { + for (const item of ast.body) { + if (item.type === 'ExpressionStatement') { + const expr = item.expression; + if (expr.type === 'AssignmentExpression') { + const left = expr.left; + if (left.type === 'MemberExpression') { + const obj = left.object; + const prop = left.property; + if ( + obj.type === 'Identifier' && + obj.value === functionName && + prop.type === 'Identifier' && + prop.value === 'workflowId' + ) { + const right = expr.right; + if (right.type === 'StringLiteral') { + return right.value; + } + } + } + } + } + } + return null; +} + +/** + * Analyze a workflow function and build its graph + */ +function analyzeWorkflowFunction( + func: FunctionDeclaration, + workflowName: string, + workflowId: string, + filePath: string, + stepDeclarations: Map, + functionMap: Map +): WorkflowGraph { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + + // Add start node + nodes.push({ + id: 'start', + type: 'workflowStart', + position: { x: 250, y: 0 }, + data: { + label: `Start: ${workflowName}`, + nodeKind: 'workflow_start', + line: func.span.start, + }, + }); + + // Context for control flow analysis + const context: AnalysisContext = { + parallelCounter: 0, + loopCounter: 0, + conditionalCounter: 0, + nodeCounter: 0, + yPosition: 100, + inLoop: null, + inConditional: null, + }; + + let prevExitIds = ['start']; + + // Analyze function body + if (func.body?.stmts) { + for (const stmt of func.body.stmts) { + const result = analyzeStatement( + stmt, + stepDeclarations, + context, + functionMap + ); + + // Add all nodes and edges from this statement + nodes.push(...result.nodes); + edges.push(...result.edges); + + // Connect previous exits to this statement's entries + for (const prevId of prevExitIds) { + for (const entryId of result.entryNodeIds) { + // Check if edge already exists + const edgeId = `e_${prevId}_${entryId}`; + if (!edges.find((e) => e.id === edgeId)) { + const targetNode = result.nodes.find((n) => n.id === entryId); + const edgeType = targetNode?.metadata?.parallelGroupId + ? 'parallel' + : targetNode?.metadata?.loopId + ? 'loop' + : 'default'; + edges.push({ + id: edgeId, + source: prevId, + target: entryId, + type: edgeType, + }); + } + } + } + + // Update prev exits for next iteration + if (result.exitNodeIds.length > 0) { + prevExitIds = result.exitNodeIds; + } + } + } + + // Add end node + const endY = context.yPosition; + nodes.push({ + id: 'end', + type: 'workflowEnd', + position: { x: 250, y: endY }, + data: { + label: 'Return', + nodeKind: 'workflow_end', + line: func.span.end, + }, + }); + + // Connect last exits to end + for (const prevId of prevExitIds) { + edges.push({ + id: `e_${prevId}_end`, + source: prevId, + target: 'end', + type: 'default', + }); + } + + return { + workflowId, + workflowName, + filePath, + nodes, + edges, + }; +} + +interface AnalysisContext { + parallelCounter: number; + loopCounter: number; + conditionalCounter: number; + nodeCounter: number; + yPosition: number; + inLoop: string | null; + inConditional: string | null; +} + +interface AnalysisResult { + nodes: GraphNode[]; + edges: GraphEdge[]; + entryNodeIds: string[]; // Nodes that should receive edge from previous + exitNodeIds: string[]; // Nodes that should send edge to next +} + +/** + * Analyze a statement and extract step calls with proper CFG structure + */ +function analyzeStatement( + stmt: Statement, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map +): AnalysisResult { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + let entryNodeIds: string[] = []; + let exitNodeIds: string[] = []; + + // Variable declaration (const result = await step()) + if (stmt.type === 'VariableDeclaration') { + const varDecl = stmt as VariableDeclaration; + for (const decl of varDecl.declarations) { + if (decl.init) { + const result = analyzeExpression( + decl.init, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + if (entryNodeIds.length === 0) { + entryNodeIds = result.entryNodeIds; + } else { + // Connect previous exits to new entries + for (const prevId of exitNodeIds) { + for (const entryId of result.entryNodeIds) { + edges.push({ + id: `e_${prevId}_${entryId}`, + source: prevId, + target: entryId, + type: 'default', + }); + } + } + } + exitNodeIds = result.exitNodeIds; + } + } + } + + // Expression statement (await step()) + if (stmt.type === 'ExpressionStatement') { + const result = analyzeExpression( + stmt.expression, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + entryNodeIds = result.entryNodeIds; + exitNodeIds = result.exitNodeIds; + } + + // If statement + if (stmt.type === 'IfStatement') { + const savedConditional = context.inConditional; + const conditionalId = `cond_${context.conditionalCounter++}`; + context.inConditional = conditionalId; + + // Analyze consequent (then branch) + if (stmt.consequent.type === 'BlockStatement') { + const branchResult = analyzeBlock( + stmt.consequent.stmts, + stepDeclarations, + context, + functionMap + ); + + // Mark all nodes with conditional metadata for 'Then' branch + for (const node of branchResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.conditionalId = conditionalId; + node.metadata.conditionalBranch = 'Then'; + } + + nodes.push(...branchResult.nodes); + edges.push(...branchResult.edges); + if (entryNodeIds.length === 0) { + entryNodeIds = branchResult.entryNodeIds; + } + exitNodeIds.push(...branchResult.exitNodeIds); + } + + // Analyze alternate (else branch) + if (stmt.alternate?.type === 'BlockStatement') { + const branchResult = analyzeBlock( + stmt.alternate.stmts, + stepDeclarations, + context, + functionMap + ); + + // Mark all nodes with conditional metadata for 'Else' branch + for (const node of branchResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.conditionalId = conditionalId; + node.metadata.conditionalBranch = 'Else'; + } + + nodes.push(...branchResult.nodes); + edges.push(...branchResult.edges); + // Add else branch entries to entryNodeIds for proper edge connection + if (entryNodeIds.length === 0) { + entryNodeIds = branchResult.entryNodeIds; + } else { + entryNodeIds.push(...branchResult.entryNodeIds); + } + exitNodeIds.push(...branchResult.exitNodeIds); + } + + context.inConditional = savedConditional; + } + + // While/For loops + if (stmt.type === 'WhileStatement' || stmt.type === 'ForStatement') { + const loopId = `loop_${context.loopCounter++}`; + const savedLoop = context.inLoop; + context.inLoop = loopId; + + const body = + stmt.type === 'WhileStatement' ? stmt.body : (stmt as any).body; + if (body.type === 'BlockStatement') { + const loopResult = analyzeBlock( + body.stmts, + stepDeclarations, + context, + functionMap + ); + + // Mark all nodes with loop metadata + for (const node of loopResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.loopId = loopId; + } + + nodes.push(...loopResult.nodes); + edges.push(...loopResult.edges); + entryNodeIds = loopResult.entryNodeIds; + exitNodeIds = loopResult.exitNodeIds; + + // Add loop back-edge from last nodes to first nodes + for (const exitId of loopResult.exitNodeIds) { + for (const entryId of loopResult.entryNodeIds) { + edges.push({ + id: `e_${exitId}_back_${entryId}`, + source: exitId, + target: entryId, + type: 'loop', + }); + } + } + } + + context.inLoop = savedLoop; + } + + // For-of loops (including `for await...of`) + if (stmt.type === 'ForOfStatement') { + const loopId = `loop_${context.loopCounter++}`; + const savedLoop = context.inLoop; + context.inLoop = loopId; + + const isAwait = (stmt as any).isAwait || (stmt as any).await; + const body = (stmt as any).body; + + if (body.type === 'BlockStatement') { + const loopResult = analyzeBlock( + body.stmts, + stepDeclarations, + context, + functionMap + ); + + // Mark all nodes with loop metadata + for (const node of loopResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.loopId = loopId; + node.metadata.loopIsAwait = isAwait; + } + + nodes.push(...loopResult.nodes); + edges.push(...loopResult.edges); + entryNodeIds = loopResult.entryNodeIds; + exitNodeIds = loopResult.exitNodeIds; + + // Add loop back-edge from last nodes to first nodes + for (const exitId of loopResult.exitNodeIds) { + for (const entryId of loopResult.entryNodeIds) { + edges.push({ + id: `e_${exitId}_back_${entryId}`, + source: exitId, + target: entryId, + type: 'loop', + }); + } + } + } + + context.inLoop = savedLoop; + } + + // Return statement with expression + if (stmt.type === 'ReturnStatement' && (stmt as any).argument) { + const result = analyzeExpression( + (stmt as any).argument, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + entryNodeIds = result.entryNodeIds; + exitNodeIds = result.exitNodeIds; + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze a block of statements with proper sequential chaining + */ +function analyzeBlock( + stmts: Statement[], + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map +): AnalysisResult { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + let entryNodeIds: string[] = []; + let currentExitIds: string[] = []; + + for (const stmt of stmts) { + const result = analyzeStatement( + stmt, + stepDeclarations, + context, + functionMap + ); + + if (result.nodes.length === 0) continue; + + nodes.push(...result.nodes); + edges.push(...result.edges); + + // Set entry nodes from first statement with nodes + if (entryNodeIds.length === 0 && result.entryNodeIds.length > 0) { + entryNodeIds = result.entryNodeIds; + } + + // Connect previous exits to current entries + if (currentExitIds.length > 0 && result.entryNodeIds.length > 0) { + for (const prevId of currentExitIds) { + for (const entryId of result.entryNodeIds) { + const targetNode = result.nodes.find((n) => n.id === entryId); + const edgeType = targetNode?.metadata?.parallelGroupId + ? 'parallel' + : 'default'; + edges.push({ + id: `e_${prevId}_${entryId}`, + source: prevId, + target: entryId, + type: edgeType, + }); + } + } + } + + // Update exit nodes + if (result.exitNodeIds.length > 0) { + currentExitIds = result.exitNodeIds; + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds: currentExitIds }; +} + +/** + * Analyze an expression and extract step calls (including transitive calls through helper functions) + */ +function analyzeExpression( + expr: Expression, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map, + visitedFunctions: Set = new Set() // Prevent infinite recursion +): AnalysisResult { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + // Await expression + if (expr.type === 'AwaitExpression') { + const awaitedExpr = expr.argument; + if (awaitedExpr.type === 'CallExpression') { + const callExpr = awaitedExpr as CallExpression; + + // Check for Promise.all/race/allSettled + if (callExpr.callee.type === 'MemberExpression') { + const member = callExpr.callee as MemberExpression; + if ( + member.object.type === 'Identifier' && + (member.object as Identifier).value === 'Promise' && + member.property.type === 'Identifier' + ) { + const method = (member.property as Identifier).value; + if (['all', 'race', 'allSettled'].includes(method)) { + // Create a new parallel group for this Promise.all + const parallelId = `parallel_${context.parallelCounter++}`; + + // Analyze array elements + if (callExpr.arguments.length > 0) { + const arg = callExpr.arguments[0].expression; + if (arg.type === 'ArrayExpression') { + for (const element of arg.elements) { + if (element?.expression) { + const elemResult = analyzeExpression( + element.expression, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + + // Set parallel metadata on all nodes from this element + for (const node of elemResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.parallelGroupId = parallelId; + node.metadata.parallelMethod = method; + // Preserve loop context if we're inside a loop + if (context.inLoop) { + node.metadata.loopId = context.inLoop; + } + } + + nodes.push(...elemResult.nodes); + edges.push(...elemResult.edges); + entryNodeIds.push(...elemResult.entryNodeIds); + exitNodeIds.push(...elemResult.exitNodeIds); + } + } + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + } + } + + // Regular call - check if it's a step or a helper function + if (callExpr.callee.type === 'Identifier') { + const funcName = (callExpr.callee as Identifier).value; + const stepInfo = stepDeclarations.get(funcName); + + if (stepInfo) { + // Direct step call + const nodeId = `node_${context.nodeCounter++}`; + const metadata: NodeMetadata = {}; + + if (context.inLoop) { + metadata.loopId = context.inLoop; + } + if (context.inConditional) { + metadata.conditionalId = context.inConditional; + } + + const node: GraphNode = { + id: nodeId, + type: 'step', + position: { x: 250, y: context.yPosition }, + data: { + label: funcName, + nodeKind: 'step', + stepId: stepInfo.stepId, + line: expr.span.start, + }, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + }; + + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + context.yPosition += 100; + } else { + // Check if it's a helper function - analyze transitively + const transitiveResult = analyzeTransitiveCall( + funcName, + stepDeclarations, + context, + functionMap, + visitedFunctions, + expr.span.start + ); + nodes.push(...transitiveResult.nodes); + edges.push(...transitiveResult.edges); + entryNodeIds.push(...transitiveResult.entryNodeIds); + exitNodeIds.push(...transitiveResult.exitNodeIds); + } + } + } + } + + // Non-awaited call expression + if (expr.type === 'CallExpression') { + const callExpr = expr as CallExpression; + if (callExpr.callee.type === 'Identifier') { + const funcName = (callExpr.callee as Identifier).value; + const stepInfo = stepDeclarations.get(funcName); + + if (stepInfo) { + // Direct step call + const nodeId = `node_${context.nodeCounter++}`; + const metadata: NodeMetadata = {}; + + if (context.inLoop) { + metadata.loopId = context.inLoop; + } + if (context.inConditional) { + metadata.conditionalId = context.inConditional; + } + + const node: GraphNode = { + id: nodeId, + type: 'step', + position: { x: 250, y: context.yPosition }, + data: { + label: funcName, + nodeKind: 'step', + stepId: stepInfo.stepId, + line: expr.span.start, + }, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + }; + + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + context.yPosition += 100; + } else { + // Check if it's a helper function - analyze transitively + const transitiveResult = analyzeTransitiveCall( + funcName, + stepDeclarations, + context, + functionMap, + visitedFunctions, + expr.span.start + ); + nodes.push(...transitiveResult.nodes); + edges.push(...transitiveResult.edges); + entryNodeIds.push(...transitiveResult.entryNodeIds); + exitNodeIds.push(...transitiveResult.exitNodeIds); + } + } + } + + // Check for step references in object literals (e.g., { execute: stepFunc, tools: { ... } }) + if (expr.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + expr, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + + // Check for step references in function call arguments + if (expr.type === 'CallExpression') { + const callExpr = expr as CallExpression; + for (const arg of callExpr.arguments) { + if (arg.expression) { + // Check if argument is a step reference + if (arg.expression.type === 'Identifier') { + const argName = (arg.expression as Identifier).value; + const stepInfo = stepDeclarations.get(argName); + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const node: GraphNode = { + id: nodeId, + type: 'step', + position: { x: 250, y: context.yPosition }, + data: { + label: `${argName} (ref)`, + nodeKind: 'step', + stepId: stepInfo.stepId, + line: arg.expression.span.start, + }, + metadata: { + isStepReference: true, + referenceContext: 'function argument', + }, + }; + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + context.yPosition += 100; + } + } + // Check for object literals in arguments + if (arg.expression.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + arg.expression, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + } + } + } + + // Check for step references in 'new' expressions (e.g., new DurableAgent({ tools: ... })) + if (expr.type === 'NewExpression') { + const newExpr = expr as any; + if (newExpr.arguments) { + for (const arg of newExpr.arguments) { + if (arg.expression?.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + arg.expression, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + } + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze an object expression for step references (e.g., { execute: stepFunc }) + */ +function analyzeObjectForStepReferences( + obj: any, + stepDeclarations: Map, + context: AnalysisContext, + path: string +): AnalysisResult { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + if (!obj.properties) return { nodes, edges, entryNodeIds, exitNodeIds }; + + for (const prop of obj.properties) { + if (prop.type !== 'KeyValueProperty') continue; + + // Get property key name + let keyName = ''; + if (prop.key.type === 'Identifier') { + keyName = prop.key.value; + } else if (prop.key.type === 'StringLiteral') { + keyName = prop.key.value; + } + + const currentPath = path ? `${path}.${keyName}` : keyName; + + // Check if the value is a step reference + if (prop.value.type === 'Identifier') { + const valueName = prop.value.value; + const stepInfo = stepDeclarations.get(valueName); + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const node: GraphNode = { + id: nodeId, + type: 'step', + position: { x: 250, y: context.yPosition }, + data: { + label: `${valueName} (tool)`, + nodeKind: 'step', + stepId: stepInfo.stepId, + line: prop.value.span.start, + }, + metadata: { + isStepReference: true, + referenceContext: currentPath, + }, + }; + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + context.yPosition += 100; + } + } + + // Recursively check nested objects + if (prop.value.type === 'ObjectExpression') { + const nestedResult = analyzeObjectForStepReferences( + prop.value, + stepDeclarations, + context, + currentPath + ); + nodes.push(...nestedResult.nodes); + edges.push(...nestedResult.edges); + entryNodeIds.push(...nestedResult.entryNodeIds); + exitNodeIds.push(...nestedResult.exitNodeIds); + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze a transitive function call to find step calls within helper functions + */ +function analyzeTransitiveCall( + funcName: string, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map, + visitedFunctions: Set, + _callLine: number // Reserved for future debug info +): AnalysisResult { + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + // Prevent infinite recursion + if (visitedFunctions.has(funcName)) { + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + + // Look up the function in our map + const funcInfo = functionMap.get(funcName); + if (!funcInfo || funcInfo.isStep) { + // Not a helper function or already a step + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + + // Mark as visited to prevent cycles + visitedFunctions.add(funcName); + + try { + // Analyze the function body + if (funcInfo.body) { + if (funcInfo.body.type === 'BlockStatement') { + // Function body is a block statement + const bodyResult = analyzeBlock( + funcInfo.body.stmts, + stepDeclarations, + context, + functionMap + ); + nodes.push(...bodyResult.nodes); + edges.push(...bodyResult.edges); + entryNodeIds.push(...bodyResult.entryNodeIds); + exitNodeIds.push(...bodyResult.exitNodeIds); + } else { + // Arrow function with expression body + const exprResult = analyzeExpression( + funcInfo.body, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + nodes.push(...exprResult.nodes); + edges.push(...exprResult.edges); + entryNodeIds.push(...exprResult.entryNodeIds); + exitNodeIds.push(...exprResult.exitNodeIds); + } + } + } finally { + // Unmark after analysis to allow the same function to be called multiple times + // (just not recursively) + visitedFunctions.delete(funcName); + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 28a52e681..8a883de54 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -43,9 +43,23 @@ export async function getNextBuilder() { tsPaths: tsConfig.paths, }; - const stepsBuildContext = await this.buildStepsFunction(options); + const { context: stepsBuildContext } = + await this.buildStepsFunction(options); const workflowsBundle = await this.buildWorkflowsFunction(options); await this.buildWebhookRoute({ workflowGeneratedDir }); + + // Write workflows manifest to workflow data directory (post-bundle extraction) + const workflowDataDir = join( + this.config.workingDir, + '.next/workflow-data' + ); + await mkdir(workflowDataDir, { recursive: true }); + const workflowBundlePath = join(workflowGeneratedDir, 'flow/route.js'); + await this.createWorkflowsManifest({ + workflowBundlePath, + outfile: join(workflowDataDir, 'workflows.json'), + }); + await this.writeFunctionsConfig(outputDir); if (this.config.watch) { @@ -150,7 +164,8 @@ export async function getNextBuilder() { options.inputFiles = newInputFiles; await stepsCtx.dispose(); - const newStepsCtx = await this.buildStepsFunction(options); + const { context: newStepsCtx } = + await this.buildStepsFunction(options); if (!newStepsCtx) { throw new Error( 'Invariant: expected steps build context after rebuild' @@ -166,6 +181,25 @@ export async function getNextBuilder() { ); } workflowsCtx = newWorkflowsCtx; + + // Rebuild graph manifest to workflow data directory + try { + const workflowDataDir = join( + this.config.workingDir, + '.next/workflow-data' + ); + await mkdir(workflowDataDir, { recursive: true }); + const workflowBundlePath = join( + workflowGeneratedDir, + 'flow/route.js' + ); + await this.createWorkflowsManifest({ + workflowBundlePath, + outfile: join(workflowDataDir, 'workflows.json'), + }); + } catch (error) { + console.error('Failed to rebuild graph manifest:', error); + } }; const logBuildMessages = ( @@ -220,6 +254,25 @@ export async function getNextBuilder() { 'Rebuilt workflow bundle', `${Date.now() - rebuiltWorkflowStart}ms` ); + + // Rebuild graph manifest to workflow data directory (post-bundle extraction) + try { + const workflowDataDir = join( + this.config.workingDir, + '.next/workflow-data' + ); + await mkdir(workflowDataDir, { recursive: true }); + const workflowBundlePath = join( + workflowGeneratedDir, + 'flow/route.js' + ); + await this.createWorkflowsManifest({ + workflowBundlePath, + outfile: join(workflowDataDir, 'workflows.json'), + }); + } catch (error) { + console.error('Failed to rebuild graph manifest:', error); + } }; const isWatchableFile = (path: string) => diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index df5e0b22c..4124a3a0f 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -521,13 +521,13 @@ export async function stepFunctionWithClosureWorkflow() { const prefix = 'Result: '; // Create a step function that captures closure variables - const calculate = async (x: number) => { + const computeValue = async (x: number) => { 'use step'; return `${prefix}${x * multiplier}`; }; // Pass the step function (with closure vars) to another step - const result = await stepThatCallsStepFn(calculate, 7); + const result = await stepThatCallsStepFn(computeValue, 7); return result; } From d5378468710ba277f593b46b1cd5f4de93ffeac9 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Fri, 28 Nov 2025 10:08:08 -0600 Subject: [PATCH 2/9] Adding changeset --- .changeset/smart-insects-smile.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/smart-insects-smile.md diff --git a/.changeset/smart-insects-smile.md b/.changeset/smart-insects-smile.md new file mode 100644 index 000000000..e16ab46f1 --- /dev/null +++ b/.changeset/smart-insects-smile.md @@ -0,0 +1,6 @@ +--- +"@workflow/builders": patch +"@workflow/next": patch +--- + +Add CFG extractor for extracting workflow graph data from bundles From e5f4cd827db19ac210324433a755e43fd7619074 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 14:51:54 -0800 Subject: [PATCH 3/9] Extend manifest.debug.json to include workflow CFG metadata and cleanup partial generations --- packages/builders/src/base-builder.ts | 119 ++++-- packages/builders/src/standalone.ts | 22 +- .../builders/src/vercel-build-output-api.ts | 7 + packages/builders/src/workflows-extractor.ts | 378 +++++++----------- packages/next/src/builder.ts | 37 +- packages/nitro/src/builders.ts | 7 + packages/sveltekit/src/builder.ts | 7 + 7 files changed, 266 insertions(+), 311 deletions(-) diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 795935cbb..bdb12a89e 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -12,6 +12,7 @@ import { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.j import { createNodeModuleErrorPlugin } from './node-module-esbuild-plugin.js'; import { createSwcPlugin } from './swc-esbuild-plugin.js'; import type { WorkflowConfig } from './types.js'; +import { extractWorkflowGraphs } from './workflows-extractor.js'; const enhancedResolve = promisify(enhancedResolveOriginal); @@ -381,16 +382,6 @@ export abstract class BaseBuilder { this.logEsbuildMessages(stepsResult, 'steps bundle creation'); console.log('Created steps bundle', `${Date.now() - stepsBundleStart}ms`); - const partialWorkflowManifest = { - steps: workflowManifest.steps, - }; - // always write to debug file - await this.writeDebugFile( - join(dirname(outfile), 'manifest'), - partialWorkflowManifest, - true - ); - // Create .gitignore in .swc directory await this.createSwcGitignore(); @@ -510,16 +501,6 @@ export abstract class BaseBuilder { `${Date.now() - bundleStartTime}ms` ); - const partialWorkflowManifest = { - workflows: workflowManifest.workflows, - }; - - await this.writeDebugFile( - join(dirname(outfile), 'manifest'), - partialWorkflowManifest, - true - ); - if (this.config.workflowManifestPath) { const resolvedPath = resolve( process.cwd(), @@ -849,44 +830,104 @@ export const OPTIONS = handler;`; } /** - * Creates a workflows manifest JSON file by extracting from the bundled workflow file. - * The manifest contains React Flow-compatible graph data for visualizing workflows. + * Creates a manifest JSON file containing step/workflow metadata + * and graph data for visualization. */ - protected async createWorkflowsManifest({ + protected async createManifest({ workflowBundlePath, - outfile, + manifestDir, }: { workflowBundlePath: string; - outfile: string; + manifestDir: string; }): Promise { const buildStart = Date.now(); - console.log('Creating workflows manifest...'); + console.log('Creating manifest...'); try { - // Import the graph extractor - const { extractGraphFromBundle } = await import( - './workflows-extractor.js' + const workflowGraphs = await extractWorkflowGraphs(workflowBundlePath); + const source = this.lastWorkflowManifest || {}; + + const steps = this.convertStepsManifest(source.steps); + const workflows = this.convertWorkflowsManifest( + source.workflows, + workflowGraphs ); - // Extract graph from the bundled workflow file - const workflowsManifest = - await extractGraphFromBundle(workflowBundlePath); + const manifest = { version: '1.0.0', steps, workflows }; + + await mkdir(manifestDir, { recursive: true }); + await writeFile( + join(manifestDir, 'manifest.json'), + JSON.stringify(manifest, null, 2) + ); - // Write the workflows manifest - await this.ensureDirectory(outfile); - await writeFile(outfile, JSON.stringify(workflowsManifest, null, 2)); + const stepCount = Object.values(steps).reduce( + (acc, s) => acc + Object.keys(s).length, + 0 + ); + const workflowCount = Object.values(workflows).reduce( + (acc, w) => acc + Object.keys(w).length, + 0 + ); console.log( - `Created workflows manifest with ${ - Object.keys(workflowsManifest.workflows).length - } workflow(s)`, + `Created manifest with ${stepCount} step(s) and ${workflowCount} workflow(s)`, `${Date.now() - buildStart}ms` ); } catch (error) { console.warn( - 'Failed to extract workflows from bundle:', + 'Failed to create manifest:', error instanceof Error ? error.message : String(error) ); } } + + private convertStepsManifest( + steps: WorkflowManifest['steps'] + ): Record> { + const result: Record> = {}; + if (!steps) return result; + + for (const [filePath, entries] of Object.entries(steps)) { + result[filePath] = {}; + for (const [name, data] of Object.entries(entries)) { + result[filePath][name] = { stepId: data.stepId }; + } + } + return result; + } + + private convertWorkflowsManifest( + workflows: WorkflowManifest['workflows'], + graphs: Record< + string, + Record + > + ): Record< + string, + Record< + string, + { workflowId: string; graph: { nodes: any[]; edges: any[] } } + > + > { + const result: Record< + string, + Record< + string, + { workflowId: string; graph: { nodes: any[]; edges: any[] } } + > + > = {}; + if (!workflows) return result; + + for (const [filePath, entries] of Object.entries(workflows)) { + result[filePath] = {}; + for (const [name, data] of Object.entries(entries)) { + result[filePath][name] = { + workflowId: data.workflowId, + graph: graphs[filePath]?.[name]?.graph || { nodes: [], edges: [] }, + }; + } + } + return result; + } } diff --git a/packages/builders/src/standalone.ts b/packages/builders/src/standalone.ts index 11f6438af..ec3d1187c 100644 --- a/packages/builders/src/standalone.ts +++ b/packages/builders/src/standalone.ts @@ -14,9 +14,13 @@ export class StandaloneBuilder extends BaseBuilder { await this.buildWorkflowsBundle(options); await this.buildWebhookFunction(); - // Build workflows manifest from workflow bundle (post-bundle extraction) + // Build unified manifest from workflow bundle const workflowBundlePath = this.resolvePath('.swc/workflows.js'); - await this.buildWorkflowsManifest({ workflowBundlePath }); + const manifestDir = this.resolvePath('.swc'); + await this.createManifest({ + workflowBundlePath, + manifestDir, + }); await this.createClientLibrary(); } @@ -82,18 +86,4 @@ export class StandaloneBuilder extends BaseBuilder { outfile: webhookBundlePath, }); } - - private async buildWorkflowsManifest({ - workflowBundlePath, - }: { - workflowBundlePath: string; - }): Promise { - const workflowsManifestPath = this.resolvePath('.swc/workflows.json'); - await this.ensureDirectory(workflowsManifestPath); - - await this.createWorkflowsManifest({ - workflowBundlePath, - outfile: workflowsManifestPath, - }); - } } diff --git a/packages/builders/src/vercel-build-output-api.ts b/packages/builders/src/vercel-build-output-api.ts index be8869675..cda642efe 100644 --- a/packages/builders/src/vercel-build-output-api.ts +++ b/packages/builders/src/vercel-build-output-api.ts @@ -25,6 +25,13 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { await this.buildWebhookFunction(options); await this.createBuildOutputConfig(outputDir); + // Generate unified manifest + const workflowBundlePath = join(workflowGeneratedDir, 'flow.func/index.js'); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + }); + await this.createClientLibrary(); } diff --git a/packages/builders/src/workflows-extractor.ts b/packages/builders/src/workflows-extractor.ts index 52358b92d..dedcaa3f5 100644 --- a/packages/builders/src/workflows-extractor.ts +++ b/packages/builders/src/workflows-extractor.ts @@ -14,9 +14,10 @@ import type { } from '@swc/core'; import { parseSync } from '@swc/core'; -/** - * Represents a function in the bundle (can be declaration, expression, or arrow) - */ +// ============================================================================= +// Internal Types (used during extraction only) +// ============================================================================= + interface FunctionInfo { name: string; body: BlockStatement | Expression | null | undefined; @@ -24,36 +25,25 @@ interface FunctionInfo { stepId?: string; } -/** - * Graph manifest structure - */ -export interface WorkflowsManifest { - version: string; - workflows: Record; - debugInfo?: DebugInfo; -} - -export interface WorkflowGraph { - workflowId: string; - workflowName: string; - filePath: string; - nodes: GraphNode[]; - edges: GraphEdge[]; +interface AnalysisContext { + parallelCounter: number; + loopCounter: number; + conditionalCounter: number; + nodeCounter: number; + inLoop: string | null; + inConditional: string | null; } -export interface GraphNode { - id: string; - type: string; - position: { x: number; y: number }; - data: { - label: string; - nodeKind: string; - stepId?: string; - line: number; - }; - metadata?: NodeMetadata; +interface AnalysisResult { + nodes: ManifestNode[]; + edges: ManifestEdge[]; + entryNodeIds: string[]; + exitNodeIds: string[]; } +/** + * Node metadata for control flow semantics + */ export interface NodeMetadata { loopId?: string; loopIsAwait?: boolean; @@ -67,7 +57,24 @@ export interface NodeMetadata { referenceContext?: string; } -export interface GraphEdge { +/** + * Graph node for workflow visualization + */ +export interface ManifestNode { + id: string; + type: string; + data: { + label: string; + nodeKind: string; + stepId?: string; + }; + metadata?: NodeMetadata; +} + +/** + * Graph edge for workflow control flow + */ +export interface ManifestEdge { id: string; source: string; target: string; @@ -75,76 +82,86 @@ export interface GraphEdge { label?: string; } -export interface DebugInfo { - manifestPresent?: boolean; - manifestStepFiles?: number; - importsResolved?: number; - importsWithKind?: number; - importDetails?: Array<{ - localName: string; - source: string; - importedName: string; - kind?: string; - lookupCandidates: string[]; - }>; - error?: string; +/** + * Graph data for a single workflow + */ +export interface WorkflowGraphData { + nodes: ManifestNode[]; + edges: ManifestEdge[]; +} + +/** + * Step entry in the manifest + */ +export interface ManifestStepEntry { + stepId: string; +} + +/** + * Workflow entry in the manifest (includes graph data) + */ +export interface ManifestWorkflowEntry { + workflowId: string; + graph: WorkflowGraphData; +} + +/** + * Unified manifest structure - single source of truth for all workflow metadata + */ +export interface UnifiedManifest { + version: string; + steps: { + [filePath: string]: { + [stepName: string]: ManifestStepEntry; + }; + }; + workflows: { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; + }; } +// ============================================================================= +// Extraction Functions +// ============================================================================= + /** - * Extracts workflow graph from a bundled workflow file + * Extracts workflow graphs from a bundled workflow file. + * Returns workflow entries organized by file path, ready for merging into UnifiedManifest. */ -export async function extractGraphFromBundle( - bundlePath: string -): Promise { +export async function extractWorkflowGraphs(bundlePath: string): Promise<{ + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; +}> { const bundleCode = await readFile(bundlePath, 'utf-8'); try { - // The workflow bundle wraps the actual code in a template literal: - // const workflowCode = `...`; - // We need to parse the bundle AST first to properly extract the unescaped string let actualWorkflowCode = bundleCode; - // First, try to parse the bundle itself to extract workflowCode properly const bundleAst = parseSync(bundleCode, { syntax: 'ecmascript', target: 'es2022', }); - // Find the workflowCode variable declaration const workflowCodeValue = extractWorkflowCodeFromBundle(bundleAst); if (workflowCodeValue) { actualWorkflowCode = workflowCodeValue; } - // Now parse the actual workflow code const ast = parseSync(actualWorkflowCode, { syntax: 'ecmascript', target: 'es2022', }); - // Extract step declarations const stepDeclarations = extractStepDeclarations(actualWorkflowCode); - - // Build a map of ALL functions in the bundle (for transitive step resolution) const functionMap = buildFunctionMap(ast, stepDeclarations); - // Extract workflows with transitive step resolution - const workflows = extractWorkflows(ast, stepDeclarations, functionMap); - - return { - version: '1.0.0', - workflows, - }; + return extractWorkflows(ast, stepDeclarations, functionMap); } catch (error) { - console.error('Failed to extract graph from bundle:', error); - // Return empty manifest on parsing errors - return { - version: '1.0.0', - workflows: {}, - debugInfo: { - error: error instanceof Error ? error.message : String(error), - }, - }; + console.error('Failed to extract workflow graphs from bundle:', error); + return {}; } } @@ -160,12 +177,9 @@ function extractWorkflowCodeFromBundle(ast: Program): string | null { decl.id.value === 'workflowCode' && decl.init ) { - // Handle template literal if (decl.init.type === 'TemplateLiteral') { - // Concatenate all quasis (the string parts of template literal) return decl.init.quasis.map((q) => q.cooked || q.raw).join(''); } - // Handle regular string literal if (decl.init.type === 'StringLiteral') { return decl.init.value; } @@ -181,25 +195,19 @@ function extractWorkflowCodeFromBundle(ast: Program): string | null { */ function extractStepDeclarations( bundleCode: string -): Map { - const stepDeclarations = new Map(); +): Map { + const stepDeclarations = new Map(); - // Match: var stepName = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//path//name"); const stepPattern = /var (\w+) = globalThis\[Symbol\.for\("WORKFLOW_USE_STEP"\)\]\("([^"]+)"\)/g; - // Track line numbers const lines = bundleCode.split('\n'); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; + for (const line of lines) { stepPattern.lastIndex = 0; const match = stepPattern.exec(line); if (match) { const [, varName, stepId] = match; - stepDeclarations.set(varName, { - stepId, - line: i + 1, - }); + stepDeclarations.set(varName, { stepId }); } } @@ -211,12 +219,11 @@ function extractStepDeclarations( */ function buildFunctionMap( ast: Program, - stepDeclarations: Map + stepDeclarations: Map ): Map { const functionMap = new Map(); for (const item of ast.body) { - // Handle function declarations: function foo() {} if (item.type === 'FunctionDeclaration') { const func = item as FunctionDeclaration; if (func.identifier) { @@ -231,7 +238,6 @@ function buildFunctionMap( } } - // Handle variable declarations: const foo = function() {} or const foo = () => {} if (item.type === 'VariableDeclaration') { const varDecl = item as VariableDeclaration; for (const decl of varDecl.declarations) { @@ -269,44 +275,51 @@ function buildFunctionMap( */ function extractWorkflows( ast: Program, - stepDeclarations: Map, + stepDeclarations: Map, functionMap: Map -): Record { - const workflows: Record = {}; +): { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; +} { + const result: { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; + } = {}; - // Find all function declarations for (const item of ast.body) { if (item.type === 'FunctionDeclaration') { const func = item as FunctionDeclaration; if (!func.identifier) continue; const workflowName = func.identifier.value; - - // Check if this function has a workflowId property assignment - // Look for: functionName.workflowId = "workflow//path//name"; const workflowId = findWorkflowId(ast, workflowName); if (!workflowId) continue; - // Extract file path from workflowId - // Format: "workflow//path/to/file.ts//functionName" + // Extract file path from workflowId: "workflow//path/to/file.ts//functionName" const parts = workflowId.split('//'); - const filePath = parts.length > 1 ? parts[1] : ''; + const filePath = parts.length > 1 ? parts[1] : 'unknown'; - // Analyze the function body with transitive step resolution const graph = analyzeWorkflowFunction( func, workflowName, - workflowId, - filePath, stepDeclarations, functionMap ); - workflows[workflowName] = graph; + if (!result[filePath]) { + result[filePath] = {}; + } + + result[filePath][workflowName] = { + workflowId, + graph, + }; } } - return workflows; + return result; } /** @@ -345,40 +358,33 @@ function findWorkflowId(ast: Program, functionName: string): string | null { function analyzeWorkflowFunction( func: FunctionDeclaration, workflowName: string, - workflowId: string, - filePath: string, - stepDeclarations: Map, + stepDeclarations: Map, functionMap: Map -): WorkflowGraph { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; +): WorkflowGraphData { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; // Add start node nodes.push({ id: 'start', type: 'workflowStart', - position: { x: 250, y: 0 }, data: { label: `Start: ${workflowName}`, nodeKind: 'workflow_start', - line: func.span.start, }, }); - // Context for control flow analysis const context: AnalysisContext = { parallelCounter: 0, loopCounter: 0, conditionalCounter: 0, nodeCounter: 0, - yPosition: 100, inLoop: null, inConditional: null, }; let prevExitIds = ['start']; - // Analyze function body if (func.body?.stmts) { for (const stmt of func.body.stmts) { const result = analyzeStatement( @@ -388,14 +394,11 @@ function analyzeWorkflowFunction( functionMap ); - // Add all nodes and edges from this statement nodes.push(...result.nodes); edges.push(...result.edges); - // Connect previous exits to this statement's entries for (const prevId of prevExitIds) { for (const entryId of result.entryNodeIds) { - // Check if edge already exists const edgeId = `e_${prevId}_${entryId}`; if (!edges.find((e) => e.id === edgeId)) { const targetNode = result.nodes.find((n) => n.id === entryId); @@ -414,7 +417,6 @@ function analyzeWorkflowFunction( } } - // Update prev exits for next iteration if (result.exitNodeIds.length > 0) { prevExitIds = result.exitNodeIds; } @@ -422,19 +424,15 @@ function analyzeWorkflowFunction( } // Add end node - const endY = context.yPosition; nodes.push({ id: 'end', type: 'workflowEnd', - position: { x: 250, y: endY }, data: { label: 'Return', nodeKind: 'workflow_end', - line: func.span.end, }, }); - // Connect last exits to end for (const prevId of prevExitIds) { edges.push({ id: `e_${prevId}_end`, @@ -444,30 +442,7 @@ function analyzeWorkflowFunction( }); } - return { - workflowId, - workflowName, - filePath, - nodes, - edges, - }; -} - -interface AnalysisContext { - parallelCounter: number; - loopCounter: number; - conditionalCounter: number; - nodeCounter: number; - yPosition: number; - inLoop: string | null; - inConditional: string | null; -} - -interface AnalysisResult { - nodes: GraphNode[]; - edges: GraphEdge[]; - entryNodeIds: string[]; // Nodes that should receive edge from previous - exitNodeIds: string[]; // Nodes that should send edge to next + return { nodes, edges }; } /** @@ -475,16 +450,15 @@ interface AnalysisResult { */ function analyzeStatement( stmt: Statement, - stepDeclarations: Map, + stepDeclarations: Map, context: AnalysisContext, functionMap: Map ): AnalysisResult { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; let entryNodeIds: string[] = []; let exitNodeIds: string[] = []; - // Variable declaration (const result = await step()) if (stmt.type === 'VariableDeclaration') { const varDecl = stmt as VariableDeclaration; for (const decl of varDecl.declarations) { @@ -500,7 +474,6 @@ function analyzeStatement( if (entryNodeIds.length === 0) { entryNodeIds = result.entryNodeIds; } else { - // Connect previous exits to new entries for (const prevId of exitNodeIds) { for (const entryId of result.entryNodeIds) { edges.push({ @@ -517,7 +490,6 @@ function analyzeStatement( } } - // Expression statement (await step()) if (stmt.type === 'ExpressionStatement') { const result = analyzeExpression( stmt.expression, @@ -531,13 +503,11 @@ function analyzeStatement( exitNodeIds = result.exitNodeIds; } - // If statement if (stmt.type === 'IfStatement') { const savedConditional = context.inConditional; const conditionalId = `cond_${context.conditionalCounter++}`; context.inConditional = conditionalId; - // Analyze consequent (then branch) if (stmt.consequent.type === 'BlockStatement') { const branchResult = analyzeBlock( stmt.consequent.stmts, @@ -546,7 +516,6 @@ function analyzeStatement( functionMap ); - // Mark all nodes with conditional metadata for 'Then' branch for (const node of branchResult.nodes) { if (!node.metadata) node.metadata = {}; node.metadata.conditionalId = conditionalId; @@ -561,7 +530,6 @@ function analyzeStatement( exitNodeIds.push(...branchResult.exitNodeIds); } - // Analyze alternate (else branch) if (stmt.alternate?.type === 'BlockStatement') { const branchResult = analyzeBlock( stmt.alternate.stmts, @@ -570,7 +538,6 @@ function analyzeStatement( functionMap ); - // Mark all nodes with conditional metadata for 'Else' branch for (const node of branchResult.nodes) { if (!node.metadata) node.metadata = {}; node.metadata.conditionalId = conditionalId; @@ -579,7 +546,6 @@ function analyzeStatement( nodes.push(...branchResult.nodes); edges.push(...branchResult.edges); - // Add else branch entries to entryNodeIds for proper edge connection if (entryNodeIds.length === 0) { entryNodeIds = branchResult.entryNodeIds; } else { @@ -591,7 +557,6 @@ function analyzeStatement( context.inConditional = savedConditional; } - // While/For loops if (stmt.type === 'WhileStatement' || stmt.type === 'ForStatement') { const loopId = `loop_${context.loopCounter++}`; const savedLoop = context.inLoop; @@ -607,7 +572,6 @@ function analyzeStatement( functionMap ); - // Mark all nodes with loop metadata for (const node of loopResult.nodes) { if (!node.metadata) node.metadata = {}; node.metadata.loopId = loopId; @@ -618,7 +582,6 @@ function analyzeStatement( entryNodeIds = loopResult.entryNodeIds; exitNodeIds = loopResult.exitNodeIds; - // Add loop back-edge from last nodes to first nodes for (const exitId of loopResult.exitNodeIds) { for (const entryId of loopResult.entryNodeIds) { edges.push({ @@ -634,7 +597,6 @@ function analyzeStatement( context.inLoop = savedLoop; } - // For-of loops (including `for await...of`) if (stmt.type === 'ForOfStatement') { const loopId = `loop_${context.loopCounter++}`; const savedLoop = context.inLoop; @@ -651,7 +613,6 @@ function analyzeStatement( functionMap ); - // Mark all nodes with loop metadata for (const node of loopResult.nodes) { if (!node.metadata) node.metadata = {}; node.metadata.loopId = loopId; @@ -663,7 +624,6 @@ function analyzeStatement( entryNodeIds = loopResult.entryNodeIds; exitNodeIds = loopResult.exitNodeIds; - // Add loop back-edge from last nodes to first nodes for (const exitId of loopResult.exitNodeIds) { for (const entryId of loopResult.entryNodeIds) { edges.push({ @@ -679,7 +639,6 @@ function analyzeStatement( context.inLoop = savedLoop; } - // Return statement with expression if (stmt.type === 'ReturnStatement' && (stmt as any).argument) { const result = analyzeExpression( (stmt as any).argument, @@ -701,12 +660,12 @@ function analyzeStatement( */ function analyzeBlock( stmts: Statement[], - stepDeclarations: Map, + stepDeclarations: Map, context: AnalysisContext, functionMap: Map ): AnalysisResult { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; let entryNodeIds: string[] = []; let currentExitIds: string[] = []; @@ -723,12 +682,10 @@ function analyzeBlock( nodes.push(...result.nodes); edges.push(...result.edges); - // Set entry nodes from first statement with nodes if (entryNodeIds.length === 0 && result.entryNodeIds.length > 0) { entryNodeIds = result.entryNodeIds; } - // Connect previous exits to current entries if (currentExitIds.length > 0 && result.entryNodeIds.length > 0) { for (const prevId of currentExitIds) { for (const entryId of result.entryNodeIds) { @@ -746,7 +703,6 @@ function analyzeBlock( } } - // Update exit nodes if (result.exitNodeIds.length > 0) { currentExitIds = result.exitNodeIds; } @@ -756,21 +712,20 @@ function analyzeBlock( } /** - * Analyze an expression and extract step calls (including transitive calls through helper functions) + * Analyze an expression and extract step calls */ function analyzeExpression( expr: Expression, - stepDeclarations: Map, + stepDeclarations: Map, context: AnalysisContext, functionMap: Map, - visitedFunctions: Set = new Set() // Prevent infinite recursion + visitedFunctions: Set = new Set() ): AnalysisResult { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; const entryNodeIds: string[] = []; const exitNodeIds: string[] = []; - // Await expression if (expr.type === 'AwaitExpression') { const awaitedExpr = expr.argument; if (awaitedExpr.type === 'CallExpression') { @@ -786,10 +741,8 @@ function analyzeExpression( ) { const method = (member.property as Identifier).value; if (['all', 'race', 'allSettled'].includes(method)) { - // Create a new parallel group for this Promise.all const parallelId = `parallel_${context.parallelCounter++}`; - // Analyze array elements if (callExpr.arguments.length > 0) { const arg = callExpr.arguments[0].expression; if (arg.type === 'ArrayExpression') { @@ -803,12 +756,10 @@ function analyzeExpression( visitedFunctions ); - // Set parallel metadata on all nodes from this element for (const node of elemResult.nodes) { if (!node.metadata) node.metadata = {}; node.metadata.parallelGroupId = parallelId; node.metadata.parallelMethod = method; - // Preserve loop context if we're inside a loop if (context.inLoop) { node.metadata.loopId = context.inLoop; } @@ -834,7 +785,6 @@ function analyzeExpression( const stepInfo = stepDeclarations.get(funcName); if (stepInfo) { - // Direct step call const nodeId = `node_${context.nodeCounter++}`; const metadata: NodeMetadata = {}; @@ -845,15 +795,13 @@ function analyzeExpression( metadata.conditionalId = context.inConditional; } - const node: GraphNode = { + const node: ManifestNode = { id: nodeId, type: 'step', - position: { x: 250, y: context.yPosition }, data: { label: funcName, nodeKind: 'step', stepId: stepInfo.stepId, - line: expr.span.start, }, metadata: Object.keys(metadata).length > 0 ? metadata : undefined, }; @@ -861,16 +809,13 @@ function analyzeExpression( nodes.push(node); entryNodeIds.push(nodeId); exitNodeIds.push(nodeId); - context.yPosition += 100; } else { - // Check if it's a helper function - analyze transitively const transitiveResult = analyzeTransitiveCall( funcName, stepDeclarations, context, functionMap, - visitedFunctions, - expr.span.start + visitedFunctions ); nodes.push(...transitiveResult.nodes); edges.push(...transitiveResult.edges); @@ -889,7 +834,6 @@ function analyzeExpression( const stepInfo = stepDeclarations.get(funcName); if (stepInfo) { - // Direct step call const nodeId = `node_${context.nodeCounter++}`; const metadata: NodeMetadata = {}; @@ -900,15 +844,13 @@ function analyzeExpression( metadata.conditionalId = context.inConditional; } - const node: GraphNode = { + const node: ManifestNode = { id: nodeId, type: 'step', - position: { x: 250, y: context.yPosition }, data: { label: funcName, nodeKind: 'step', stepId: stepInfo.stepId, - line: expr.span.start, }, metadata: Object.keys(metadata).length > 0 ? metadata : undefined, }; @@ -916,16 +858,13 @@ function analyzeExpression( nodes.push(node); entryNodeIds.push(nodeId); exitNodeIds.push(nodeId); - context.yPosition += 100; } else { - // Check if it's a helper function - analyze transitively const transitiveResult = analyzeTransitiveCall( funcName, stepDeclarations, context, functionMap, - visitedFunctions, - expr.span.start + visitedFunctions ); nodes.push(...transitiveResult.nodes); edges.push(...transitiveResult.edges); @@ -935,7 +874,7 @@ function analyzeExpression( } } - // Check for step references in object literals (e.g., { execute: stepFunc, tools: { ... } }) + // Check for step references in object literals if (expr.type === 'ObjectExpression') { const refResult = analyzeObjectForStepReferences( expr, @@ -954,21 +893,18 @@ function analyzeExpression( const callExpr = expr as CallExpression; for (const arg of callExpr.arguments) { if (arg.expression) { - // Check if argument is a step reference if (arg.expression.type === 'Identifier') { const argName = (arg.expression as Identifier).value; const stepInfo = stepDeclarations.get(argName); if (stepInfo) { const nodeId = `node_${context.nodeCounter++}`; - const node: GraphNode = { + const node: ManifestNode = { id: nodeId, type: 'step', - position: { x: 250, y: context.yPosition }, data: { label: `${argName} (ref)`, nodeKind: 'step', stepId: stepInfo.stepId, - line: arg.expression.span.start, }, metadata: { isStepReference: true, @@ -978,10 +914,8 @@ function analyzeExpression( nodes.push(node); entryNodeIds.push(nodeId); exitNodeIds.push(nodeId); - context.yPosition += 100; } } - // Check for object literals in arguments if (arg.expression.type === 'ObjectExpression') { const refResult = analyzeObjectForStepReferences( arg.expression, @@ -998,7 +932,7 @@ function analyzeExpression( } } - // Check for step references in 'new' expressions (e.g., new DurableAgent({ tools: ... })) + // Check for step references in 'new' expressions if (expr.type === 'NewExpression') { const newExpr = expr as any; if (newExpr.arguments) { @@ -1023,16 +957,16 @@ function analyzeExpression( } /** - * Analyze an object expression for step references (e.g., { execute: stepFunc }) + * Analyze an object expression for step references */ function analyzeObjectForStepReferences( obj: any, - stepDeclarations: Map, + stepDeclarations: Map, context: AnalysisContext, path: string ): AnalysisResult { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; const entryNodeIds: string[] = []; const exitNodeIds: string[] = []; @@ -1041,7 +975,6 @@ function analyzeObjectForStepReferences( for (const prop of obj.properties) { if (prop.type !== 'KeyValueProperty') continue; - // Get property key name let keyName = ''; if (prop.key.type === 'Identifier') { keyName = prop.key.value; @@ -1051,21 +984,18 @@ function analyzeObjectForStepReferences( const currentPath = path ? `${path}.${keyName}` : keyName; - // Check if the value is a step reference if (prop.value.type === 'Identifier') { const valueName = prop.value.value; const stepInfo = stepDeclarations.get(valueName); if (stepInfo) { const nodeId = `node_${context.nodeCounter++}`; - const node: GraphNode = { + const node: ManifestNode = { id: nodeId, type: 'step', - position: { x: 250, y: context.yPosition }, data: { label: `${valueName} (tool)`, nodeKind: 'step', stepId: stepInfo.stepId, - line: prop.value.span.start, }, metadata: { isStepReference: true, @@ -1075,11 +1005,9 @@ function analyzeObjectForStepReferences( nodes.push(node); entryNodeIds.push(nodeId); exitNodeIds.push(nodeId); - context.yPosition += 100; } } - // Recursively check nested objects if (prop.value.type === 'ObjectExpression') { const nestedResult = analyzeObjectForStepReferences( prop.value, @@ -1102,37 +1030,30 @@ function analyzeObjectForStepReferences( */ function analyzeTransitiveCall( funcName: string, - stepDeclarations: Map, + stepDeclarations: Map, context: AnalysisContext, functionMap: Map, - visitedFunctions: Set, - _callLine: number // Reserved for future debug info + visitedFunctions: Set ): AnalysisResult { - const nodes: GraphNode[] = []; - const edges: GraphEdge[] = []; + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; const entryNodeIds: string[] = []; const exitNodeIds: string[] = []; - // Prevent infinite recursion if (visitedFunctions.has(funcName)) { return { nodes, edges, entryNodeIds, exitNodeIds }; } - // Look up the function in our map const funcInfo = functionMap.get(funcName); if (!funcInfo || funcInfo.isStep) { - // Not a helper function or already a step return { nodes, edges, entryNodeIds, exitNodeIds }; } - // Mark as visited to prevent cycles visitedFunctions.add(funcName); try { - // Analyze the function body if (funcInfo.body) { if (funcInfo.body.type === 'BlockStatement') { - // Function body is a block statement const bodyResult = analyzeBlock( funcInfo.body.stmts, stepDeclarations, @@ -1144,7 +1065,6 @@ function analyzeTransitiveCall( entryNodeIds.push(...bodyResult.entryNodeIds); exitNodeIds.push(...bodyResult.exitNodeIds); } else { - // Arrow function with expression body const exprResult = analyzeExpression( funcInfo.body, stepDeclarations, @@ -1159,8 +1079,6 @@ function analyzeTransitiveCall( } } } finally { - // Unmark after analysis to allow the same function to be called multiple times - // (just not recursively) visitedFunctions.delete(funcName); } diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 8a883de54..9c494d1e3 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -48,16 +48,11 @@ export async function getNextBuilder() { const workflowsBundle = await this.buildWorkflowsFunction(options); await this.buildWebhookRoute({ workflowGeneratedDir }); - // Write workflows manifest to workflow data directory (post-bundle extraction) - const workflowDataDir = join( - this.config.workingDir, - '.next/workflow-data' - ); - await mkdir(workflowDataDir, { recursive: true }); + // Write unified manifest to workflow generated directory const workflowBundlePath = join(workflowGeneratedDir, 'flow/route.js'); - await this.createWorkflowsManifest({ + await this.createManifest({ workflowBundlePath, - outfile: join(workflowDataDir, 'workflows.json'), + manifestDir: workflowGeneratedDir, }); await this.writeFunctionsConfig(outputDir); @@ -182,23 +177,18 @@ export async function getNextBuilder() { } workflowsCtx = newWorkflowsCtx; - // Rebuild graph manifest to workflow data directory + // Rebuild unified manifest try { - const workflowDataDir = join( - this.config.workingDir, - '.next/workflow-data' - ); - await mkdir(workflowDataDir, { recursive: true }); const workflowBundlePath = join( workflowGeneratedDir, 'flow/route.js' ); - await this.createWorkflowsManifest({ + await this.createManifest({ workflowBundlePath, - outfile: join(workflowDataDir, 'workflows.json'), + manifestDir: workflowGeneratedDir, }); } catch (error) { - console.error('Failed to rebuild graph manifest:', error); + console.error('Failed to rebuild manifest:', error); } }; @@ -255,23 +245,18 @@ export async function getNextBuilder() { `${Date.now() - rebuiltWorkflowStart}ms` ); - // Rebuild graph manifest to workflow data directory (post-bundle extraction) + // Rebuild unified manifest try { - const workflowDataDir = join( - this.config.workingDir, - '.next/workflow-data' - ); - await mkdir(workflowDataDir, { recursive: true }); const workflowBundlePath = join( workflowGeneratedDir, 'flow/route.js' ); - await this.createWorkflowsManifest({ + await this.createManifest({ workflowBundlePath, - outfile: join(workflowDataDir, 'workflows.json'), + manifestDir: workflowGeneratedDir, }); } catch (error) { - console.error('Failed to rebuild graph manifest:', error); + console.error('Failed to rebuild manifest:', error); } }; diff --git a/packages/nitro/src/builders.ts b/packages/nitro/src/builders.ts index 8b36d24d1..114941712 100644 --- a/packages/nitro/src/builders.ts +++ b/packages/nitro/src/builders.ts @@ -69,5 +69,12 @@ export class LocalBuilder extends BaseBuilder { outfile: webhookRouteFile, bundle: false, }); + + // Generate manifest + const workflowBundlePath = join(this.#outDir, 'workflows.mjs'); + await this.createManifest({ + workflowBundlePath, + manifestDir: this.#outDir, + }); } } diff --git a/packages/sveltekit/src/builder.ts b/packages/sveltekit/src/builder.ts index 52b4c8895..b7d971c2b 100644 --- a/packages/sveltekit/src/builder.ts +++ b/packages/sveltekit/src/builder.ts @@ -60,6 +60,13 @@ export class SvelteKitBuilder extends BaseBuilder { await this.buildStepsRoute(options); await this.buildWorkflowsRoute(options); await this.buildWebhookRoute({ workflowGeneratedDir }); + + // Generate unified manifest + const workflowBundlePath = join(workflowGeneratedDir, 'flow/+server.js'); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + }); } private async buildStepsRoute({ From 68b23e0fbd7a0f79c17c2d47cd9afcc9f524ff97 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 14:56:43 -0800 Subject: [PATCH 4/9] Extend manifest.debug.json to include workflow CFG metadata and cleanup partial generations --- packages/builders/src/workflows-extractor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/builders/src/workflows-extractor.ts b/packages/builders/src/workflows-extractor.ts index dedcaa3f5..7e54100d5 100644 --- a/packages/builders/src/workflows-extractor.ts +++ b/packages/builders/src/workflows-extractor.ts @@ -106,9 +106,9 @@ export interface ManifestWorkflowEntry { } /** - * Unified manifest structure - single source of truth for all workflow metadata + * Manifest structure - single source of truth for all workflow metadata */ -export interface UnifiedManifest { +export interface Manifest { version: string; steps: { [filePath: string]: { @@ -128,7 +128,7 @@ export interface UnifiedManifest { /** * Extracts workflow graphs from a bundled workflow file. - * Returns workflow entries organized by file path, ready for merging into UnifiedManifest. + * Returns workflow entries organized by file path, ready for merging into Manifest. */ export async function extractWorkflowGraphs(bundlePath: string): Promise<{ [filePath: string]: { From 1c45af610af106d0bcbf78dd19b189dbb60370df Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 15:35:32 -0800 Subject: [PATCH 5/9] Add unit test coverage for manifest --- packages/builders/src/base-builder.ts | 15 +- packages/builders/src/standalone.ts | 3 +- .../builders/src/vercel-build-output-api.ts | 9 +- packages/core/e2e/bench.bench.ts | 4 +- packages/core/e2e/dev.test.ts | 4 +- packages/core/e2e/manifest.test.ts | 130 ++++++++++++++++++ packages/next/src/builder.ts | 9 +- packages/nitro/src/builders.ts | 3 +- packages/sveltekit/src/builder.ts | 6 +- 9 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 packages/core/e2e/manifest.test.ts diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index bdb12a89e..eb9fc43ae 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -27,7 +27,6 @@ const EMIT_SOURCEMAPS_FOR_DEBUGGING = */ export abstract class BaseBuilder { protected config: WorkflowConfig; - protected lastWorkflowManifest?: WorkflowManifest; constructor(config: WorkflowConfig) { this.config = config; @@ -385,9 +384,6 @@ export abstract class BaseBuilder { // Create .gitignore in .swc directory await this.createSwcGitignore(); - // Store the manifest for later use (e.g., graph generation in watch mode) - this.lastWorkflowManifest = workflowManifest; - if (this.config.watch) { return { context: esbuildCtx, manifest: workflowManifest }; } @@ -836,29 +832,30 @@ export const OPTIONS = handler;`; protected async createManifest({ workflowBundlePath, manifestDir, + manifest, }: { workflowBundlePath: string; manifestDir: string; + manifest: WorkflowManifest; }): Promise { const buildStart = Date.now(); console.log('Creating manifest...'); try { const workflowGraphs = await extractWorkflowGraphs(workflowBundlePath); - const source = this.lastWorkflowManifest || {}; - const steps = this.convertStepsManifest(source.steps); + const steps = this.convertStepsManifest(manifest.steps); const workflows = this.convertWorkflowsManifest( - source.workflows, + manifest.workflows, workflowGraphs ); - const manifest = { version: '1.0.0', steps, workflows }; + const output = { version: '1.0.0', steps, workflows }; await mkdir(manifestDir, { recursive: true }); await writeFile( join(manifestDir, 'manifest.json'), - JSON.stringify(manifest, null, 2) + JSON.stringify(output, null, 2) ); const stepCount = Object.values(steps).reduce( diff --git a/packages/builders/src/standalone.ts b/packages/builders/src/standalone.ts index ec3d1187c..36d1f60d2 100644 --- a/packages/builders/src/standalone.ts +++ b/packages/builders/src/standalone.ts @@ -10,7 +10,7 @@ export class StandaloneBuilder extends BaseBuilder { tsBaseUrl: tsConfig.baseUrl, tsPaths: tsConfig.paths, }; - await this.buildStepsBundle(options); + const manifest = await this.buildStepsBundle(options); await this.buildWorkflowsBundle(options); await this.buildWebhookFunction(); @@ -20,6 +20,7 @@ export class StandaloneBuilder extends BaseBuilder { await this.createManifest({ workflowBundlePath, manifestDir, + manifest, }); await this.createClientLibrary(); diff --git a/packages/builders/src/vercel-build-output-api.ts b/packages/builders/src/vercel-build-output-api.ts index cda642efe..943e7492a 100644 --- a/packages/builders/src/vercel-build-output-api.ts +++ b/packages/builders/src/vercel-build-output-api.ts @@ -20,7 +20,7 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { tsBaseUrl: tsConfig.baseUrl, tsPaths: tsConfig.paths, }; - await this.buildStepsFunction(options); + const manifest = await this.buildStepsFunction(options); await this.buildWorkflowsFunction(options); await this.buildWebhookFunction(options); await this.createBuildOutputConfig(outputDir); @@ -30,6 +30,7 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { await this.createManifest({ workflowBundlePath, manifestDir: workflowGeneratedDir, + manifest, }); await this.createClientLibrary(); @@ -45,13 +46,13 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { workflowGeneratedDir: string; tsBaseUrl?: string; tsPaths?: Record; - }): Promise { + }) { console.log('Creating Vercel Build Output API steps function'); const stepsFuncDir = join(workflowGeneratedDir, 'step.func'); await mkdir(stepsFuncDir, { recursive: true }); // Create steps bundle - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ inputFiles, outfile: join(stepsFuncDir, 'index.js'), tsBaseUrl, @@ -64,6 +65,8 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { shouldAddSourcemapSupport: true, experimentalTriggers: [STEP_QUEUE_TRIGGER], }); + + return manifest; } private async buildWorkflowsFunction({ diff --git a/packages/core/e2e/bench.bench.ts b/packages/core/e2e/bench.bench.ts index ce57115a2..82c962919 100644 --- a/packages/core/e2e/bench.bench.ts +++ b/packages/core/e2e/bench.bench.ts @@ -1,8 +1,8 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { withResolvers } from '@workflow/utils'; import { bench, describe } from 'vitest'; import { dehydrateWorkflowArguments } from '../src/serialization'; -import fs from 'fs'; -import path from 'path'; const deploymentUrl = process.env.DEPLOYMENT_URL; if (!deploymentUrl) { diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 04052c040..4411d7ea4 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -1,5 +1,5 @@ -import fs from 'fs/promises'; -import path from 'path'; +import fs from 'node:fs/promises'; +import path from 'node:path'; import { afterEach, describe, expect, test } from 'vitest'; import { getWorkbenchAppPath } from './utils'; diff --git a/packages/core/e2e/manifest.test.ts b/packages/core/e2e/manifest.test.ts new file mode 100644 index 000000000..03e250a92 --- /dev/null +++ b/packages/core/e2e/manifest.test.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { getWorkbenchAppPath } from './utils'; + +interface ManifestStep { + stepId: string; +} + +interface ManifestWorkflow { + workflowId: string; + graph: { + nodes: Array<{ + id: string; + type: string; + data: { + label: string; + nodeKind: string; + stepId?: string; + }; + }>; + edges: Array<{ + id: string; + source: string; + target: string; + }>; + }; +} + +interface Manifest { + version: string; + steps: Record>; + workflows: Record>; +} + +// Map project names to their manifest paths +const MANIFEST_PATHS: Record = { + 'nextjs-webpack': 'app/.well-known/workflow/v1/manifest.json', + 'nextjs-turbopack': 'app/.well-known/workflow/v1/manifest.json', + nitro: 'node_modules/.nitro/workflow/manifest.json', + vite: 'node_modules/.nitro/workflow/manifest.json', + sveltekit: 'src/routes/.well-known/workflow/v1/manifest.json', + nuxt: 'node_modules/.nitro/workflow/manifest.json', + hono: 'node_modules/.nitro/workflow/manifest.json', + express: 'node_modules/.nitro/workflow/manifest.json', +}; + +function validateSteps(steps: Manifest['steps']) { + expect(steps).toBeDefined(); + expect(typeof steps).toBe('object'); + + const stepFiles = Object.keys(steps); + expect(stepFiles.length).toBeGreaterThan(0); + + for (const filePath of stepFiles) { + const fileSteps = steps[filePath]; + for (const [stepName, stepData] of Object.entries(fileSteps)) { + expect(stepData.stepId).toBeDefined(); + expect(stepData.stepId).toContain('step//'); + expect(stepData.stepId).toContain(stepName); + } + } +} + +function validateWorkflowGraph(graph: ManifestWorkflow['graph']) { + expect(graph).toBeDefined(); + expect(graph.nodes).toBeDefined(); + expect(Array.isArray(graph.nodes)).toBe(true); + expect(graph.edges).toBeDefined(); + expect(Array.isArray(graph.edges)).toBe(true); + + for (const node of graph.nodes) { + expect(node.id).toBeDefined(); + expect(node.type).toBeDefined(); + expect(node.data).toBeDefined(); + expect(node.data.label).toBeDefined(); + expect(node.data.nodeKind).toBeDefined(); + } + + for (const edge of graph.edges) { + expect(edge.id).toBeDefined(); + expect(edge.source).toBeDefined(); + expect(edge.target).toBeDefined(); + } + + const nodeTypes = graph.nodes.map((n) => n.type); + expect(nodeTypes).toContain('workflowStart'); + expect(nodeTypes).toContain('workflowEnd'); +} + +function validateWorkflows(workflows: Manifest['workflows']) { + expect(workflows).toBeDefined(); + expect(typeof workflows).toBe('object'); + + const workflowFiles = Object.keys(workflows); + expect(workflowFiles.length).toBeGreaterThan(0); + + for (const filePath of workflowFiles) { + const fileWorkflows = workflows[filePath]; + for (const [workflowName, workflowData] of Object.entries(fileWorkflows)) { + expect(workflowData.workflowId).toBeDefined(); + expect(workflowData.workflowId).toContain('workflow//'); + expect(workflowData.workflowId).toContain(workflowName); + validateWorkflowGraph(workflowData.graph); + } + } +} + +describe.each(Object.keys(MANIFEST_PATHS))('manifest generation', (project) => { + test( + `${project}: manifest.json exists and has valid structure`, + { timeout: 30_000 }, + async () => { + // Skip if we're targeting a specific app + if (process.env.APP_NAME && project !== process.env.APP_NAME) { + return; + } + + const appPath = getWorkbenchAppPath(project); + const manifestPath = path.join(appPath, MANIFEST_PATHS[project]); + + const manifestContent = await fs.readFile(manifestPath, 'utf8'); + const manifest: Manifest = JSON.parse(manifestContent); + + expect(manifest.version).toBe('1.0.0'); + validateSteps(manifest.steps); + validateWorkflows(manifest.workflows); + } + ); +}); diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 9c494d1e3..b86dde5ec 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -43,7 +43,7 @@ export async function getNextBuilder() { tsPaths: tsConfig.paths, }; - const { context: stepsBuildContext } = + const { context: stepsBuildContext, manifest } = await this.buildStepsFunction(options); const workflowsBundle = await this.buildWorkflowsFunction(options); await this.buildWebhookRoute({ workflowGeneratedDir }); @@ -53,6 +53,7 @@ export async function getNextBuilder() { await this.createManifest({ workflowBundlePath, manifestDir: workflowGeneratedDir, + manifest, }); await this.writeFunctionsConfig(outputDir); @@ -69,6 +70,7 @@ export async function getNextBuilder() { let stepsCtx = stepsBuildContext; let workflowsCtx = workflowsBundle; + let currentManifest = manifest; const normalizePath = (pathname: string) => pathname.replace(/\\/g, '/'); @@ -159,7 +161,7 @@ export async function getNextBuilder() { options.inputFiles = newInputFiles; await stepsCtx.dispose(); - const { context: newStepsCtx } = + const { context: newStepsCtx, manifest: newManifest } = await this.buildStepsFunction(options); if (!newStepsCtx) { throw new Error( @@ -167,6 +169,7 @@ export async function getNextBuilder() { ); } stepsCtx = newStepsCtx; + currentManifest = newManifest; await workflowsCtx.interimBundleCtx.dispose(); const newWorkflowsCtx = await this.buildWorkflowsFunction(options); @@ -186,6 +189,7 @@ export async function getNextBuilder() { await this.createManifest({ workflowBundlePath, manifestDir: workflowGeneratedDir, + manifest: currentManifest, }); } catch (error) { console.error('Failed to rebuild manifest:', error); @@ -254,6 +258,7 @@ export async function getNextBuilder() { await this.createManifest({ workflowBundlePath, manifestDir: workflowGeneratedDir, + manifest: currentManifest, }); } catch (error) { console.error('Failed to rebuild manifest:', error); diff --git a/packages/nitro/src/builders.ts b/packages/nitro/src/builders.ts index 114941712..d9c95409a 100644 --- a/packages/nitro/src/builders.ts +++ b/packages/nitro/src/builders.ts @@ -56,7 +56,7 @@ export class LocalBuilder extends BaseBuilder { inputFiles, }); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ outfile: join(this.#outDir, 'steps.mjs'), externalizeNonSteps: true, format: 'esm', @@ -75,6 +75,7 @@ export class LocalBuilder extends BaseBuilder { await this.createManifest({ workflowBundlePath, manifestDir: this.#outDir, + manifest, }); } } diff --git a/packages/sveltekit/src/builder.ts b/packages/sveltekit/src/builder.ts index b7d971c2b..60678b612 100644 --- a/packages/sveltekit/src/builder.ts +++ b/packages/sveltekit/src/builder.ts @@ -57,7 +57,7 @@ export class SvelteKitBuilder extends BaseBuilder { }; // Generate the three SvelteKit route handlers - await this.buildStepsRoute(options); + const manifest = await this.buildStepsRoute(options); await this.buildWorkflowsRoute(options); await this.buildWebhookRoute({ workflowGeneratedDir }); @@ -66,6 +66,7 @@ export class SvelteKitBuilder extends BaseBuilder { await this.createManifest({ workflowBundlePath, manifestDir: workflowGeneratedDir, + manifest, }); } @@ -84,7 +85,7 @@ export class SvelteKitBuilder extends BaseBuilder { const stepsRouteDir = join(workflowGeneratedDir, 'step'); await mkdir(stepsRouteDir, { recursive: true }); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ format: 'esm', inputFiles, outfile: join(stepsRouteDir, '+server.js'), @@ -108,6 +109,7 @@ export const POST = async ({request}) => { ); await writeFile(stepsRouteFile, stepsRouteContent); + return manifest; } private async buildWorkflowsRoute({ From 8e21d56fe709001ff3bef706d58311859446b595 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 18:46:37 -0800 Subject: [PATCH 6/9] fix --- workbench/example/workflows/99_e2e.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 4124a3a0f..df5e0b22c 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -521,13 +521,13 @@ export async function stepFunctionWithClosureWorkflow() { const prefix = 'Result: '; // Create a step function that captures closure variables - const computeValue = async (x: number) => { + const calculate = async (x: number) => { 'use step'; return `${prefix}${x * multiplier}`; }; // Pass the step function (with closure vars) to another step - const result = await stepThatCallsStepFn(computeValue, 7); + const result = await stepThatCallsStepFn(calculate, 7); return result; } From b002d81e502f5a5ec767bb4f75f03e17e8ffd81c Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 18:48:52 -0800 Subject: [PATCH 7/9] Add changeset --- .changeset/huge-rabbits-travel.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .changeset/huge-rabbits-travel.md diff --git a/.changeset/huge-rabbits-travel.md b/.changeset/huge-rabbits-travel.md new file mode 100644 index 000000000..47d07a0d3 --- /dev/null +++ b/.changeset/huge-rabbits-travel.md @@ -0,0 +1,14 @@ +--- +"@workflow/world-postgres": patch +"@workflow/world-local": patch +"@workflow/sveltekit": patch +"@workflow/builders": patch +"@workflow/nitro": patch +"@workflow/utils": patch +"@workflow/world": patch +"@workflow/core": patch +"@workflow/next": patch +"@workflow/web": patch +--- + +Refactor manifest generation to pass manifest as a parameter instead of using instance state. Add e2e tests for manifest validation across all builders. From 252a0ddfa489ecfb46a40b932cdc3b206d5fcfb8 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 18:49:15 -0800 Subject: [PATCH 8/9] Add changeset --- .changeset/smart-insects-smile.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changeset/smart-insects-smile.md diff --git a/.changeset/smart-insects-smile.md b/.changeset/smart-insects-smile.md deleted file mode 100644 index e16ab46f1..000000000 --- a/.changeset/smart-insects-smile.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@workflow/builders": patch -"@workflow/next": patch ---- - -Add CFG extractor for extracting workflow graph data from bundles From 8a3f152d2d9052764f7883c216d0eed129e2a2ee Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Mon, 1 Dec 2025 18:50:45 -0800 Subject: [PATCH 9/9] Add changeset --- .changeset/huge-rabbits-travel.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/huge-rabbits-travel.md b/.changeset/huge-rabbits-travel.md index 47d07a0d3..c4323b7b0 100644 --- a/.changeset/huge-rabbits-travel.md +++ b/.changeset/huge-rabbits-travel.md @@ -11,4 +11,4 @@ "@workflow/web": patch --- -Refactor manifest generation to pass manifest as a parameter instead of using instance state. Add e2e tests for manifest validation across all builders. +Added Control Flow Graph extraction from Workflows and extended manifest.json's schema to incorporate the graph structure into it. Refactored manifest generation to pass manifest as a parameter instead of using instance state. Add e2e tests for manifest validation across all builders.