diff --git a/.changeset/huge-rabbits-travel.md b/.changeset/huge-rabbits-travel.md new file mode 100644 index 000000000..c4323b7b0 --- /dev/null +++ b/.changeset/huge-rabbits-travel.md @@ -0,0 +1,14 @@ +--- +"@workflow/world-postgres": patch +"@workflow/world-local": patch +"@workflow/sveltekit": patch +"@workflow/builders": patch +"@workflow/nitro": patch +"@workflow/utils": patch +"@workflow/world": patch +"@workflow/core": patch +"@workflow/next": patch +"@workflow/web": patch +--- + +Added Control Flow Graph extraction from Workflows and extended manifest.json's schema to incorporate the graph structure into it. Refactored manifest generation to pass manifest as a parameter instead of using instance state. Add e2e tests for manifest validation across all builders. diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index b0929873b..eb9fc43ae 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -12,6 +12,7 @@ import { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.j import { createNodeModuleErrorPlugin } from './node-module-esbuild-plugin.js'; import { createSwcPlugin } from './swc-esbuild-plugin.js'; import type { WorkflowConfig } from './types.js'; +import { extractWorkflowGraphs } from './workflows-extractor.js'; const enhancedResolve = promisify(enhancedResolveOriginal); @@ -253,6 +254,7 @@ export abstract class BaseBuilder { * Steps have full Node.js runtime access and handle side effects, API calls, etc. * * @param externalizeNonSteps - If true, only bundles step entry points and externalizes other code + * @returns Build context (for watch mode) and the collected workflow manifest */ protected async createStepsBundle({ inputFiles, @@ -268,7 +270,10 @@ export abstract class BaseBuilder { outfile: string; format?: 'cjs' | 'esm'; externalizeNonSteps?: boolean; - }): Promise { + }): Promise<{ + context: esbuild.BuildContext | undefined; + manifest: WorkflowManifest; + }> { // These need to handle watching for dev to scan for // new entries and changes to existing ones const { discoveredSteps: stepFiles } = await this.discoverEntries( @@ -376,23 +381,14 @@ export abstract class BaseBuilder { this.logEsbuildMessages(stepsResult, 'steps bundle creation'); console.log('Created steps bundle', `${Date.now() - stepsBundleStart}ms`); - const partialWorkflowManifest = { - steps: workflowManifest.steps, - }; - // always write to debug file - await this.writeDebugFile( - join(dirname(outfile), 'manifest'), - partialWorkflowManifest, - true - ); - // Create .gitignore in .swc directory await this.createSwcGitignore(); if (this.config.watch) { - return esbuildCtx; + return { context: esbuildCtx, manifest: workflowManifest }; } await esbuildCtx.dispose(); + return { context: undefined, manifest: workflowManifest }; } /** @@ -501,16 +497,6 @@ export abstract class BaseBuilder { `${Date.now() - bundleStartTime}ms` ); - const partialWorkflowManifest = { - workflows: workflowManifest.workflows, - }; - - await this.writeDebugFile( - join(dirname(outfile), 'manifest'), - partialWorkflowManifest, - true - ); - if (this.config.workflowManifestPath) { const resolvedPath = resolve( process.cwd(), @@ -838,4 +824,107 @@ export const OPTIONS = handler;`; // We're intentionally silently ignoring this error - creating .gitignore isn't critical } } + + /** + * Creates a manifest JSON file containing step/workflow metadata + * and graph data for visualization. + */ + protected async createManifest({ + workflowBundlePath, + manifestDir, + manifest, + }: { + workflowBundlePath: string; + manifestDir: string; + manifest: WorkflowManifest; + }): Promise { + const buildStart = Date.now(); + console.log('Creating manifest...'); + + try { + const workflowGraphs = await extractWorkflowGraphs(workflowBundlePath); + + const steps = this.convertStepsManifest(manifest.steps); + const workflows = this.convertWorkflowsManifest( + manifest.workflows, + workflowGraphs + ); + + const output = { version: '1.0.0', steps, workflows }; + + await mkdir(manifestDir, { recursive: true }); + await writeFile( + join(manifestDir, 'manifest.json'), + JSON.stringify(output, null, 2) + ); + + const stepCount = Object.values(steps).reduce( + (acc, s) => acc + Object.keys(s).length, + 0 + ); + const workflowCount = Object.values(workflows).reduce( + (acc, w) => acc + Object.keys(w).length, + 0 + ); + + console.log( + `Created manifest with ${stepCount} step(s) and ${workflowCount} workflow(s)`, + `${Date.now() - buildStart}ms` + ); + } catch (error) { + console.warn( + 'Failed to create manifest:', + error instanceof Error ? error.message : String(error) + ); + } + } + + private convertStepsManifest( + steps: WorkflowManifest['steps'] + ): Record> { + const result: Record> = {}; + if (!steps) return result; + + for (const [filePath, entries] of Object.entries(steps)) { + result[filePath] = {}; + for (const [name, data] of Object.entries(entries)) { + result[filePath][name] = { stepId: data.stepId }; + } + } + return result; + } + + private convertWorkflowsManifest( + workflows: WorkflowManifest['workflows'], + graphs: Record< + string, + Record + > + ): Record< + string, + Record< + string, + { workflowId: string; graph: { nodes: any[]; edges: any[] } } + > + > { + const result: Record< + string, + Record< + string, + { workflowId: string; graph: { nodes: any[]; edges: any[] } } + > + > = {}; + if (!workflows) return result; + + for (const [filePath, entries] of Object.entries(workflows)) { + result[filePath] = {}; + for (const [name, data] of Object.entries(entries)) { + result[filePath][name] = { + workflowId: data.workflowId, + graph: graphs[filePath]?.[name]?.graph || { nodes: [], edges: [] }, + }; + } + } + return result; + } } diff --git a/packages/builders/src/standalone.ts b/packages/builders/src/standalone.ts index c4ef2c936..36d1f60d2 100644 --- a/packages/builders/src/standalone.ts +++ b/packages/builders/src/standalone.ts @@ -10,10 +10,19 @@ export class StandaloneBuilder extends BaseBuilder { tsBaseUrl: tsConfig.baseUrl, tsPaths: tsConfig.paths, }; - await this.buildStepsBundle(options); + const manifest = await this.buildStepsBundle(options); await this.buildWorkflowsBundle(options); await this.buildWebhookFunction(); + // Build unified manifest from workflow bundle + const workflowBundlePath = this.resolvePath('.swc/workflows.js'); + const manifestDir = this.resolvePath('.swc'); + await this.createManifest({ + workflowBundlePath, + manifestDir, + manifest, + }); + await this.createClientLibrary(); } @@ -25,18 +34,20 @@ export class StandaloneBuilder extends BaseBuilder { inputFiles: string[]; tsBaseUrl?: string; tsPaths?: Record; - }): Promise { + }) { console.log('Creating steps bundle at', this.config.stepsBundlePath); const stepsBundlePath = this.resolvePath(this.config.stepsBundlePath); await this.ensureDirectory(stepsBundlePath); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ outfile: stepsBundlePath, inputFiles, tsBaseUrl, tsPaths, }); + + return manifest; } private async buildWorkflowsBundle({ diff --git a/packages/builders/src/vercel-build-output-api.ts b/packages/builders/src/vercel-build-output-api.ts index be8869675..943e7492a 100644 --- a/packages/builders/src/vercel-build-output-api.ts +++ b/packages/builders/src/vercel-build-output-api.ts @@ -20,11 +20,19 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { tsBaseUrl: tsConfig.baseUrl, tsPaths: tsConfig.paths, }; - await this.buildStepsFunction(options); + const manifest = await this.buildStepsFunction(options); await this.buildWorkflowsFunction(options); await this.buildWebhookFunction(options); await this.createBuildOutputConfig(outputDir); + // Generate unified manifest + const workflowBundlePath = join(workflowGeneratedDir, 'flow.func/index.js'); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + manifest, + }); + await this.createClientLibrary(); } @@ -38,13 +46,13 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { workflowGeneratedDir: string; tsBaseUrl?: string; tsPaths?: Record; - }): Promise { + }) { console.log('Creating Vercel Build Output API steps function'); const stepsFuncDir = join(workflowGeneratedDir, 'step.func'); await mkdir(stepsFuncDir, { recursive: true }); // Create steps bundle - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ inputFiles, outfile: join(stepsFuncDir, 'index.js'), tsBaseUrl, @@ -57,6 +65,8 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder { shouldAddSourcemapSupport: true, experimentalTriggers: [STEP_QUEUE_TRIGGER], }); + + return manifest; } private async buildWorkflowsFunction({ diff --git a/packages/builders/src/workflows-extractor.ts b/packages/builders/src/workflows-extractor.ts new file mode 100644 index 000000000..7e54100d5 --- /dev/null +++ b/packages/builders/src/workflows-extractor.ts @@ -0,0 +1,1086 @@ +import { readFile } from 'node:fs/promises'; +import type { + ArrowFunctionExpression, + BlockStatement, + CallExpression, + Expression, + FunctionDeclaration, + FunctionExpression, + Identifier, + MemberExpression, + Program, + Statement, + VariableDeclaration, +} from '@swc/core'; +import { parseSync } from '@swc/core'; + +// ============================================================================= +// Internal Types (used during extraction only) +// ============================================================================= + +interface FunctionInfo { + name: string; + body: BlockStatement | Expression | null | undefined; + isStep: boolean; + stepId?: string; +} + +interface AnalysisContext { + parallelCounter: number; + loopCounter: number; + conditionalCounter: number; + nodeCounter: number; + inLoop: string | null; + inConditional: string | null; +} + +interface AnalysisResult { + nodes: ManifestNode[]; + edges: ManifestEdge[]; + entryNodeIds: string[]; + exitNodeIds: string[]; +} + +/** + * Node metadata for control flow semantics + */ +export interface NodeMetadata { + loopId?: string; + loopIsAwait?: boolean; + conditionalId?: string; + conditionalBranch?: 'Then' | 'Else'; + parallelGroupId?: string; + parallelMethod?: string; + /** Step is passed as a reference (callback/tool) rather than directly called */ + isStepReference?: boolean; + /** Context where the step reference was found (e.g., "tools.getWeather.execute") */ + referenceContext?: string; +} + +/** + * Graph node for workflow visualization + */ +export interface ManifestNode { + id: string; + type: string; + data: { + label: string; + nodeKind: string; + stepId?: string; + }; + metadata?: NodeMetadata; +} + +/** + * Graph edge for workflow control flow + */ +export interface ManifestEdge { + id: string; + source: string; + target: string; + type: 'default' | 'loop' | 'conditional' | 'parallel'; + label?: string; +} + +/** + * Graph data for a single workflow + */ +export interface WorkflowGraphData { + nodes: ManifestNode[]; + edges: ManifestEdge[]; +} + +/** + * Step entry in the manifest + */ +export interface ManifestStepEntry { + stepId: string; +} + +/** + * Workflow entry in the manifest (includes graph data) + */ +export interface ManifestWorkflowEntry { + workflowId: string; + graph: WorkflowGraphData; +} + +/** + * Manifest structure - single source of truth for all workflow metadata + */ +export interface Manifest { + version: string; + steps: { + [filePath: string]: { + [stepName: string]: ManifestStepEntry; + }; + }; + workflows: { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; + }; +} + +// ============================================================================= +// Extraction Functions +// ============================================================================= + +/** + * Extracts workflow graphs from a bundled workflow file. + * Returns workflow entries organized by file path, ready for merging into Manifest. + */ +export async function extractWorkflowGraphs(bundlePath: string): Promise<{ + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; +}> { + const bundleCode = await readFile(bundlePath, 'utf-8'); + + try { + let actualWorkflowCode = bundleCode; + + const bundleAst = parseSync(bundleCode, { + syntax: 'ecmascript', + target: 'es2022', + }); + + const workflowCodeValue = extractWorkflowCodeFromBundle(bundleAst); + if (workflowCodeValue) { + actualWorkflowCode = workflowCodeValue; + } + + const ast = parseSync(actualWorkflowCode, { + syntax: 'ecmascript', + target: 'es2022', + }); + + const stepDeclarations = extractStepDeclarations(actualWorkflowCode); + const functionMap = buildFunctionMap(ast, stepDeclarations); + + return extractWorkflows(ast, stepDeclarations, functionMap); + } catch (error) { + console.error('Failed to extract workflow graphs from bundle:', error); + return {}; + } +} + +/** + * Extract the workflowCode string value from a parsed bundle AST + */ +function extractWorkflowCodeFromBundle(ast: Program): string | null { + for (const item of ast.body) { + if (item.type === 'VariableDeclaration') { + for (const decl of item.declarations) { + if ( + decl.id.type === 'Identifier' && + decl.id.value === 'workflowCode' && + decl.init + ) { + if (decl.init.type === 'TemplateLiteral') { + return decl.init.quasis.map((q) => q.cooked || q.raw).join(''); + } + if (decl.init.type === 'StringLiteral') { + return decl.init.value; + } + } + } + } + } + return null; +} + +/** + * Extract step declarations using regex for speed + */ +function extractStepDeclarations( + bundleCode: string +): Map { + const stepDeclarations = new Map(); + + const stepPattern = + /var (\w+) = globalThis\[Symbol\.for\("WORKFLOW_USE_STEP"\)\]\("([^"]+)"\)/g; + + const lines = bundleCode.split('\n'); + for (const line of lines) { + stepPattern.lastIndex = 0; + const match = stepPattern.exec(line); + if (match) { + const [, varName, stepId] = match; + stepDeclarations.set(varName, { stepId }); + } + } + + return stepDeclarations; +} + +/** + * Build a map of all functions in the bundle for transitive step resolution + */ +function buildFunctionMap( + ast: Program, + stepDeclarations: Map +): Map { + const functionMap = new Map(); + + for (const item of ast.body) { + if (item.type === 'FunctionDeclaration') { + const func = item as FunctionDeclaration; + if (func.identifier) { + const name = func.identifier.value; + const isStep = stepDeclarations.has(name); + functionMap.set(name, { + name, + body: func.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } + } + + if (item.type === 'VariableDeclaration') { + const varDecl = item as VariableDeclaration; + for (const decl of varDecl.declarations) { + if (decl.id.type === 'Identifier' && decl.init) { + const name = decl.id.value; + const isStep = stepDeclarations.has(name); + + if (decl.init.type === 'FunctionExpression') { + const funcExpr = decl.init as FunctionExpression; + functionMap.set(name, { + name, + body: funcExpr.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } else if (decl.init.type === 'ArrowFunctionExpression') { + const arrowFunc = decl.init as ArrowFunctionExpression; + functionMap.set(name, { + name, + body: arrowFunc.body, + isStep, + stepId: isStep ? stepDeclarations.get(name)?.stepId : undefined, + }); + } + } + } + } + } + + return functionMap; +} + +/** + * Extract workflows from AST + */ +function extractWorkflows( + ast: Program, + stepDeclarations: Map, + functionMap: Map +): { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; +} { + const result: { + [filePath: string]: { + [workflowName: string]: ManifestWorkflowEntry; + }; + } = {}; + + for (const item of ast.body) { + if (item.type === 'FunctionDeclaration') { + const func = item as FunctionDeclaration; + if (!func.identifier) continue; + + const workflowName = func.identifier.value; + const workflowId = findWorkflowId(ast, workflowName); + if (!workflowId) continue; + + // Extract file path from workflowId: "workflow//path/to/file.ts//functionName" + const parts = workflowId.split('//'); + const filePath = parts.length > 1 ? parts[1] : 'unknown'; + + const graph = analyzeWorkflowFunction( + func, + workflowName, + stepDeclarations, + functionMap + ); + + if (!result[filePath]) { + result[filePath] = {}; + } + + result[filePath][workflowName] = { + workflowId, + graph, + }; + } + } + + return result; +} + +/** + * Find workflowId assignment for a function + */ +function findWorkflowId(ast: Program, functionName: string): string | null { + for (const item of ast.body) { + if (item.type === 'ExpressionStatement') { + const expr = item.expression; + if (expr.type === 'AssignmentExpression') { + const left = expr.left; + if (left.type === 'MemberExpression') { + const obj = left.object; + const prop = left.property; + if ( + obj.type === 'Identifier' && + obj.value === functionName && + prop.type === 'Identifier' && + prop.value === 'workflowId' + ) { + const right = expr.right; + if (right.type === 'StringLiteral') { + return right.value; + } + } + } + } + } + } + return null; +} + +/** + * Analyze a workflow function and build its graph + */ +function analyzeWorkflowFunction( + func: FunctionDeclaration, + workflowName: string, + stepDeclarations: Map, + functionMap: Map +): WorkflowGraphData { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + + // Add start node + nodes.push({ + id: 'start', + type: 'workflowStart', + data: { + label: `Start: ${workflowName}`, + nodeKind: 'workflow_start', + }, + }); + + const context: AnalysisContext = { + parallelCounter: 0, + loopCounter: 0, + conditionalCounter: 0, + nodeCounter: 0, + inLoop: null, + inConditional: null, + }; + + let prevExitIds = ['start']; + + if (func.body?.stmts) { + for (const stmt of func.body.stmts) { + const result = analyzeStatement( + stmt, + stepDeclarations, + context, + functionMap + ); + + nodes.push(...result.nodes); + edges.push(...result.edges); + + for (const prevId of prevExitIds) { + for (const entryId of result.entryNodeIds) { + const edgeId = `e_${prevId}_${entryId}`; + if (!edges.find((e) => e.id === edgeId)) { + const targetNode = result.nodes.find((n) => n.id === entryId); + const edgeType = targetNode?.metadata?.parallelGroupId + ? 'parallel' + : targetNode?.metadata?.loopId + ? 'loop' + : 'default'; + edges.push({ + id: edgeId, + source: prevId, + target: entryId, + type: edgeType, + }); + } + } + } + + if (result.exitNodeIds.length > 0) { + prevExitIds = result.exitNodeIds; + } + } + } + + // Add end node + nodes.push({ + id: 'end', + type: 'workflowEnd', + data: { + label: 'Return', + nodeKind: 'workflow_end', + }, + }); + + for (const prevId of prevExitIds) { + edges.push({ + id: `e_${prevId}_end`, + source: prevId, + target: 'end', + type: 'default', + }); + } + + return { nodes, edges }; +} + +/** + * Analyze a statement and extract step calls with proper CFG structure + */ +function analyzeStatement( + stmt: Statement, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map +): AnalysisResult { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + let entryNodeIds: string[] = []; + let exitNodeIds: string[] = []; + + if (stmt.type === 'VariableDeclaration') { + const varDecl = stmt as VariableDeclaration; + for (const decl of varDecl.declarations) { + if (decl.init) { + const result = analyzeExpression( + decl.init, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + if (entryNodeIds.length === 0) { + entryNodeIds = result.entryNodeIds; + } else { + for (const prevId of exitNodeIds) { + for (const entryId of result.entryNodeIds) { + edges.push({ + id: `e_${prevId}_${entryId}`, + source: prevId, + target: entryId, + type: 'default', + }); + } + } + } + exitNodeIds = result.exitNodeIds; + } + } + } + + if (stmt.type === 'ExpressionStatement') { + const result = analyzeExpression( + stmt.expression, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + entryNodeIds = result.entryNodeIds; + exitNodeIds = result.exitNodeIds; + } + + if (stmt.type === 'IfStatement') { + const savedConditional = context.inConditional; + const conditionalId = `cond_${context.conditionalCounter++}`; + context.inConditional = conditionalId; + + if (stmt.consequent.type === 'BlockStatement') { + const branchResult = analyzeBlock( + stmt.consequent.stmts, + stepDeclarations, + context, + functionMap + ); + + for (const node of branchResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.conditionalId = conditionalId; + node.metadata.conditionalBranch = 'Then'; + } + + nodes.push(...branchResult.nodes); + edges.push(...branchResult.edges); + if (entryNodeIds.length === 0) { + entryNodeIds = branchResult.entryNodeIds; + } + exitNodeIds.push(...branchResult.exitNodeIds); + } + + if (stmt.alternate?.type === 'BlockStatement') { + const branchResult = analyzeBlock( + stmt.alternate.stmts, + stepDeclarations, + context, + functionMap + ); + + for (const node of branchResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.conditionalId = conditionalId; + node.metadata.conditionalBranch = 'Else'; + } + + nodes.push(...branchResult.nodes); + edges.push(...branchResult.edges); + if (entryNodeIds.length === 0) { + entryNodeIds = branchResult.entryNodeIds; + } else { + entryNodeIds.push(...branchResult.entryNodeIds); + } + exitNodeIds.push(...branchResult.exitNodeIds); + } + + context.inConditional = savedConditional; + } + + if (stmt.type === 'WhileStatement' || stmt.type === 'ForStatement') { + const loopId = `loop_${context.loopCounter++}`; + const savedLoop = context.inLoop; + context.inLoop = loopId; + + const body = + stmt.type === 'WhileStatement' ? stmt.body : (stmt as any).body; + if (body.type === 'BlockStatement') { + const loopResult = analyzeBlock( + body.stmts, + stepDeclarations, + context, + functionMap + ); + + for (const node of loopResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.loopId = loopId; + } + + nodes.push(...loopResult.nodes); + edges.push(...loopResult.edges); + entryNodeIds = loopResult.entryNodeIds; + exitNodeIds = loopResult.exitNodeIds; + + for (const exitId of loopResult.exitNodeIds) { + for (const entryId of loopResult.entryNodeIds) { + edges.push({ + id: `e_${exitId}_back_${entryId}`, + source: exitId, + target: entryId, + type: 'loop', + }); + } + } + } + + context.inLoop = savedLoop; + } + + if (stmt.type === 'ForOfStatement') { + const loopId = `loop_${context.loopCounter++}`; + const savedLoop = context.inLoop; + context.inLoop = loopId; + + const isAwait = (stmt as any).isAwait || (stmt as any).await; + const body = (stmt as any).body; + + if (body.type === 'BlockStatement') { + const loopResult = analyzeBlock( + body.stmts, + stepDeclarations, + context, + functionMap + ); + + for (const node of loopResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.loopId = loopId; + node.metadata.loopIsAwait = isAwait; + } + + nodes.push(...loopResult.nodes); + edges.push(...loopResult.edges); + entryNodeIds = loopResult.entryNodeIds; + exitNodeIds = loopResult.exitNodeIds; + + for (const exitId of loopResult.exitNodeIds) { + for (const entryId of loopResult.entryNodeIds) { + edges.push({ + id: `e_${exitId}_back_${entryId}`, + source: exitId, + target: entryId, + type: 'loop', + }); + } + } + } + + context.inLoop = savedLoop; + } + + if (stmt.type === 'ReturnStatement' && (stmt as any).argument) { + const result = analyzeExpression( + (stmt as any).argument, + stepDeclarations, + context, + functionMap + ); + nodes.push(...result.nodes); + edges.push(...result.edges); + entryNodeIds = result.entryNodeIds; + exitNodeIds = result.exitNodeIds; + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze a block of statements with proper sequential chaining + */ +function analyzeBlock( + stmts: Statement[], + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map +): AnalysisResult { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + let entryNodeIds: string[] = []; + let currentExitIds: string[] = []; + + for (const stmt of stmts) { + const result = analyzeStatement( + stmt, + stepDeclarations, + context, + functionMap + ); + + if (result.nodes.length === 0) continue; + + nodes.push(...result.nodes); + edges.push(...result.edges); + + if (entryNodeIds.length === 0 && result.entryNodeIds.length > 0) { + entryNodeIds = result.entryNodeIds; + } + + if (currentExitIds.length > 0 && result.entryNodeIds.length > 0) { + for (const prevId of currentExitIds) { + for (const entryId of result.entryNodeIds) { + const targetNode = result.nodes.find((n) => n.id === entryId); + const edgeType = targetNode?.metadata?.parallelGroupId + ? 'parallel' + : 'default'; + edges.push({ + id: `e_${prevId}_${entryId}`, + source: prevId, + target: entryId, + type: edgeType, + }); + } + } + } + + if (result.exitNodeIds.length > 0) { + currentExitIds = result.exitNodeIds; + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds: currentExitIds }; +} + +/** + * Analyze an expression and extract step calls + */ +function analyzeExpression( + expr: Expression, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map, + visitedFunctions: Set = new Set() +): AnalysisResult { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + if (expr.type === 'AwaitExpression') { + const awaitedExpr = expr.argument; + if (awaitedExpr.type === 'CallExpression') { + const callExpr = awaitedExpr as CallExpression; + + // Check for Promise.all/race/allSettled + if (callExpr.callee.type === 'MemberExpression') { + const member = callExpr.callee as MemberExpression; + if ( + member.object.type === 'Identifier' && + (member.object as Identifier).value === 'Promise' && + member.property.type === 'Identifier' + ) { + const method = (member.property as Identifier).value; + if (['all', 'race', 'allSettled'].includes(method)) { + const parallelId = `parallel_${context.parallelCounter++}`; + + if (callExpr.arguments.length > 0) { + const arg = callExpr.arguments[0].expression; + if (arg.type === 'ArrayExpression') { + for (const element of arg.elements) { + if (element?.expression) { + const elemResult = analyzeExpression( + element.expression, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + + for (const node of elemResult.nodes) { + if (!node.metadata) node.metadata = {}; + node.metadata.parallelGroupId = parallelId; + node.metadata.parallelMethod = method; + if (context.inLoop) { + node.metadata.loopId = context.inLoop; + } + } + + nodes.push(...elemResult.nodes); + edges.push(...elemResult.edges); + entryNodeIds.push(...elemResult.entryNodeIds); + exitNodeIds.push(...elemResult.exitNodeIds); + } + } + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + } + } + + // Regular call - check if it's a step or a helper function + if (callExpr.callee.type === 'Identifier') { + const funcName = (callExpr.callee as Identifier).value; + const stepInfo = stepDeclarations.get(funcName); + + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const metadata: NodeMetadata = {}; + + if (context.inLoop) { + metadata.loopId = context.inLoop; + } + if (context.inConditional) { + metadata.conditionalId = context.inConditional; + } + + const node: ManifestNode = { + id: nodeId, + type: 'step', + data: { + label: funcName, + nodeKind: 'step', + stepId: stepInfo.stepId, + }, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + }; + + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + } else { + const transitiveResult = analyzeTransitiveCall( + funcName, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + nodes.push(...transitiveResult.nodes); + edges.push(...transitiveResult.edges); + entryNodeIds.push(...transitiveResult.entryNodeIds); + exitNodeIds.push(...transitiveResult.exitNodeIds); + } + } + } + } + + // Non-awaited call expression + if (expr.type === 'CallExpression') { + const callExpr = expr as CallExpression; + if (callExpr.callee.type === 'Identifier') { + const funcName = (callExpr.callee as Identifier).value; + const stepInfo = stepDeclarations.get(funcName); + + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const metadata: NodeMetadata = {}; + + if (context.inLoop) { + metadata.loopId = context.inLoop; + } + if (context.inConditional) { + metadata.conditionalId = context.inConditional; + } + + const node: ManifestNode = { + id: nodeId, + type: 'step', + data: { + label: funcName, + nodeKind: 'step', + stepId: stepInfo.stepId, + }, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + }; + + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + } else { + const transitiveResult = analyzeTransitiveCall( + funcName, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + nodes.push(...transitiveResult.nodes); + edges.push(...transitiveResult.edges); + entryNodeIds.push(...transitiveResult.entryNodeIds); + exitNodeIds.push(...transitiveResult.exitNodeIds); + } + } + } + + // Check for step references in object literals + if (expr.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + expr, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + + // Check for step references in function call arguments + if (expr.type === 'CallExpression') { + const callExpr = expr as CallExpression; + for (const arg of callExpr.arguments) { + if (arg.expression) { + if (arg.expression.type === 'Identifier') { + const argName = (arg.expression as Identifier).value; + const stepInfo = stepDeclarations.get(argName); + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const node: ManifestNode = { + id: nodeId, + type: 'step', + data: { + label: `${argName} (ref)`, + nodeKind: 'step', + stepId: stepInfo.stepId, + }, + metadata: { + isStepReference: true, + referenceContext: 'function argument', + }, + }; + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + } + } + if (arg.expression.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + arg.expression, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + } + } + } + + // Check for step references in 'new' expressions + if (expr.type === 'NewExpression') { + const newExpr = expr as any; + if (newExpr.arguments) { + for (const arg of newExpr.arguments) { + if (arg.expression?.type === 'ObjectExpression') { + const refResult = analyzeObjectForStepReferences( + arg.expression, + stepDeclarations, + context, + '' + ); + nodes.push(...refResult.nodes); + edges.push(...refResult.edges); + entryNodeIds.push(...refResult.entryNodeIds); + exitNodeIds.push(...refResult.exitNodeIds); + } + } + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze an object expression for step references + */ +function analyzeObjectForStepReferences( + obj: any, + stepDeclarations: Map, + context: AnalysisContext, + path: string +): AnalysisResult { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + if (!obj.properties) return { nodes, edges, entryNodeIds, exitNodeIds }; + + for (const prop of obj.properties) { + if (prop.type !== 'KeyValueProperty') continue; + + let keyName = ''; + if (prop.key.type === 'Identifier') { + keyName = prop.key.value; + } else if (prop.key.type === 'StringLiteral') { + keyName = prop.key.value; + } + + const currentPath = path ? `${path}.${keyName}` : keyName; + + if (prop.value.type === 'Identifier') { + const valueName = prop.value.value; + const stepInfo = stepDeclarations.get(valueName); + if (stepInfo) { + const nodeId = `node_${context.nodeCounter++}`; + const node: ManifestNode = { + id: nodeId, + type: 'step', + data: { + label: `${valueName} (tool)`, + nodeKind: 'step', + stepId: stepInfo.stepId, + }, + metadata: { + isStepReference: true, + referenceContext: currentPath, + }, + }; + nodes.push(node); + entryNodeIds.push(nodeId); + exitNodeIds.push(nodeId); + } + } + + if (prop.value.type === 'ObjectExpression') { + const nestedResult = analyzeObjectForStepReferences( + prop.value, + stepDeclarations, + context, + currentPath + ); + nodes.push(...nestedResult.nodes); + edges.push(...nestedResult.edges); + entryNodeIds.push(...nestedResult.entryNodeIds); + exitNodeIds.push(...nestedResult.exitNodeIds); + } + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} + +/** + * Analyze a transitive function call to find step calls within helper functions + */ +function analyzeTransitiveCall( + funcName: string, + stepDeclarations: Map, + context: AnalysisContext, + functionMap: Map, + visitedFunctions: Set +): AnalysisResult { + const nodes: ManifestNode[] = []; + const edges: ManifestEdge[] = []; + const entryNodeIds: string[] = []; + const exitNodeIds: string[] = []; + + if (visitedFunctions.has(funcName)) { + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + + const funcInfo = functionMap.get(funcName); + if (!funcInfo || funcInfo.isStep) { + return { nodes, edges, entryNodeIds, exitNodeIds }; + } + + visitedFunctions.add(funcName); + + try { + if (funcInfo.body) { + if (funcInfo.body.type === 'BlockStatement') { + const bodyResult = analyzeBlock( + funcInfo.body.stmts, + stepDeclarations, + context, + functionMap + ); + nodes.push(...bodyResult.nodes); + edges.push(...bodyResult.edges); + entryNodeIds.push(...bodyResult.entryNodeIds); + exitNodeIds.push(...bodyResult.exitNodeIds); + } else { + const exprResult = analyzeExpression( + funcInfo.body, + stepDeclarations, + context, + functionMap, + visitedFunctions + ); + nodes.push(...exprResult.nodes); + edges.push(...exprResult.edges); + entryNodeIds.push(...exprResult.entryNodeIds); + exitNodeIds.push(...exprResult.exitNodeIds); + } + } + } finally { + visitedFunctions.delete(funcName); + } + + return { nodes, edges, entryNodeIds, exitNodeIds }; +} diff --git a/packages/core/e2e/bench.bench.ts b/packages/core/e2e/bench.bench.ts index ce57115a2..82c962919 100644 --- a/packages/core/e2e/bench.bench.ts +++ b/packages/core/e2e/bench.bench.ts @@ -1,8 +1,8 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { withResolvers } from '@workflow/utils'; import { bench, describe } from 'vitest'; import { dehydrateWorkflowArguments } from '../src/serialization'; -import fs from 'fs'; -import path from 'path'; const deploymentUrl = process.env.DEPLOYMENT_URL; if (!deploymentUrl) { diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 04052c040..4411d7ea4 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -1,5 +1,5 @@ -import fs from 'fs/promises'; -import path from 'path'; +import fs from 'node:fs/promises'; +import path from 'node:path'; import { afterEach, describe, expect, test } from 'vitest'; import { getWorkbenchAppPath } from './utils'; diff --git a/packages/core/e2e/manifest.test.ts b/packages/core/e2e/manifest.test.ts new file mode 100644 index 000000000..03e250a92 --- /dev/null +++ b/packages/core/e2e/manifest.test.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { getWorkbenchAppPath } from './utils'; + +interface ManifestStep { + stepId: string; +} + +interface ManifestWorkflow { + workflowId: string; + graph: { + nodes: Array<{ + id: string; + type: string; + data: { + label: string; + nodeKind: string; + stepId?: string; + }; + }>; + edges: Array<{ + id: string; + source: string; + target: string; + }>; + }; +} + +interface Manifest { + version: string; + steps: Record>; + workflows: Record>; +} + +// Map project names to their manifest paths +const MANIFEST_PATHS: Record = { + 'nextjs-webpack': 'app/.well-known/workflow/v1/manifest.json', + 'nextjs-turbopack': 'app/.well-known/workflow/v1/manifest.json', + nitro: 'node_modules/.nitro/workflow/manifest.json', + vite: 'node_modules/.nitro/workflow/manifest.json', + sveltekit: 'src/routes/.well-known/workflow/v1/manifest.json', + nuxt: 'node_modules/.nitro/workflow/manifest.json', + hono: 'node_modules/.nitro/workflow/manifest.json', + express: 'node_modules/.nitro/workflow/manifest.json', +}; + +function validateSteps(steps: Manifest['steps']) { + expect(steps).toBeDefined(); + expect(typeof steps).toBe('object'); + + const stepFiles = Object.keys(steps); + expect(stepFiles.length).toBeGreaterThan(0); + + for (const filePath of stepFiles) { + const fileSteps = steps[filePath]; + for (const [stepName, stepData] of Object.entries(fileSteps)) { + expect(stepData.stepId).toBeDefined(); + expect(stepData.stepId).toContain('step//'); + expect(stepData.stepId).toContain(stepName); + } + } +} + +function validateWorkflowGraph(graph: ManifestWorkflow['graph']) { + expect(graph).toBeDefined(); + expect(graph.nodes).toBeDefined(); + expect(Array.isArray(graph.nodes)).toBe(true); + expect(graph.edges).toBeDefined(); + expect(Array.isArray(graph.edges)).toBe(true); + + for (const node of graph.nodes) { + expect(node.id).toBeDefined(); + expect(node.type).toBeDefined(); + expect(node.data).toBeDefined(); + expect(node.data.label).toBeDefined(); + expect(node.data.nodeKind).toBeDefined(); + } + + for (const edge of graph.edges) { + expect(edge.id).toBeDefined(); + expect(edge.source).toBeDefined(); + expect(edge.target).toBeDefined(); + } + + const nodeTypes = graph.nodes.map((n) => n.type); + expect(nodeTypes).toContain('workflowStart'); + expect(nodeTypes).toContain('workflowEnd'); +} + +function validateWorkflows(workflows: Manifest['workflows']) { + expect(workflows).toBeDefined(); + expect(typeof workflows).toBe('object'); + + const workflowFiles = Object.keys(workflows); + expect(workflowFiles.length).toBeGreaterThan(0); + + for (const filePath of workflowFiles) { + const fileWorkflows = workflows[filePath]; + for (const [workflowName, workflowData] of Object.entries(fileWorkflows)) { + expect(workflowData.workflowId).toBeDefined(); + expect(workflowData.workflowId).toContain('workflow//'); + expect(workflowData.workflowId).toContain(workflowName); + validateWorkflowGraph(workflowData.graph); + } + } +} + +describe.each(Object.keys(MANIFEST_PATHS))('manifest generation', (project) => { + test( + `${project}: manifest.json exists and has valid structure`, + { timeout: 30_000 }, + async () => { + // Skip if we're targeting a specific app + if (process.env.APP_NAME && project !== process.env.APP_NAME) { + return; + } + + const appPath = getWorkbenchAppPath(project); + const manifestPath = path.join(appPath, MANIFEST_PATHS[project]); + + const manifestContent = await fs.readFile(manifestPath, 'utf8'); + const manifest: Manifest = JSON.parse(manifestContent); + + expect(manifest.version).toBe('1.0.0'); + validateSteps(manifest.steps); + validateWorkflows(manifest.workflows); + } + ); +}); diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 28a52e681..b86dde5ec 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -43,9 +43,19 @@ export async function getNextBuilder() { tsPaths: tsConfig.paths, }; - const stepsBuildContext = await this.buildStepsFunction(options); + const { context: stepsBuildContext, manifest } = + await this.buildStepsFunction(options); const workflowsBundle = await this.buildWorkflowsFunction(options); await this.buildWebhookRoute({ workflowGeneratedDir }); + + // Write unified manifest to workflow generated directory + const workflowBundlePath = join(workflowGeneratedDir, 'flow/route.js'); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + manifest, + }); + await this.writeFunctionsConfig(outputDir); if (this.config.watch) { @@ -60,6 +70,7 @@ export async function getNextBuilder() { let stepsCtx = stepsBuildContext; let workflowsCtx = workflowsBundle; + let currentManifest = manifest; const normalizePath = (pathname: string) => pathname.replace(/\\/g, '/'); @@ -150,13 +161,15 @@ export async function getNextBuilder() { options.inputFiles = newInputFiles; await stepsCtx.dispose(); - const newStepsCtx = await this.buildStepsFunction(options); + const { context: newStepsCtx, manifest: newManifest } = + await this.buildStepsFunction(options); if (!newStepsCtx) { throw new Error( 'Invariant: expected steps build context after rebuild' ); } stepsCtx = newStepsCtx; + currentManifest = newManifest; await workflowsCtx.interimBundleCtx.dispose(); const newWorkflowsCtx = await this.buildWorkflowsFunction(options); @@ -166,6 +179,21 @@ export async function getNextBuilder() { ); } workflowsCtx = newWorkflowsCtx; + + // Rebuild unified manifest + try { + const workflowBundlePath = join( + workflowGeneratedDir, + 'flow/route.js' + ); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + manifest: currentManifest, + }); + } catch (error) { + console.error('Failed to rebuild manifest:', error); + } }; const logBuildMessages = ( @@ -220,6 +248,21 @@ export async function getNextBuilder() { 'Rebuilt workflow bundle', `${Date.now() - rebuiltWorkflowStart}ms` ); + + // Rebuild unified manifest + try { + const workflowBundlePath = join( + workflowGeneratedDir, + 'flow/route.js' + ); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + manifest: currentManifest, + }); + } catch (error) { + console.error('Failed to rebuild manifest:', error); + } }; const isWatchableFile = (path: string) => diff --git a/packages/nitro/src/builders.ts b/packages/nitro/src/builders.ts index 8b36d24d1..d9c95409a 100644 --- a/packages/nitro/src/builders.ts +++ b/packages/nitro/src/builders.ts @@ -56,7 +56,7 @@ export class LocalBuilder extends BaseBuilder { inputFiles, }); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ outfile: join(this.#outDir, 'steps.mjs'), externalizeNonSteps: true, format: 'esm', @@ -69,5 +69,13 @@ export class LocalBuilder extends BaseBuilder { outfile: webhookRouteFile, bundle: false, }); + + // Generate manifest + const workflowBundlePath = join(this.#outDir, 'workflows.mjs'); + await this.createManifest({ + workflowBundlePath, + manifestDir: this.#outDir, + manifest, + }); } } diff --git a/packages/sveltekit/src/builder.ts b/packages/sveltekit/src/builder.ts index 52b4c8895..60678b612 100644 --- a/packages/sveltekit/src/builder.ts +++ b/packages/sveltekit/src/builder.ts @@ -57,9 +57,17 @@ export class SvelteKitBuilder extends BaseBuilder { }; // Generate the three SvelteKit route handlers - await this.buildStepsRoute(options); + const manifest = await this.buildStepsRoute(options); await this.buildWorkflowsRoute(options); await this.buildWebhookRoute({ workflowGeneratedDir }); + + // Generate unified manifest + const workflowBundlePath = join(workflowGeneratedDir, 'flow/+server.js'); + await this.createManifest({ + workflowBundlePath, + manifestDir: workflowGeneratedDir, + manifest, + }); } private async buildStepsRoute({ @@ -77,7 +85,7 @@ export class SvelteKitBuilder extends BaseBuilder { const stepsRouteDir = join(workflowGeneratedDir, 'step'); await mkdir(stepsRouteDir, { recursive: true }); - await this.createStepsBundle({ + const { manifest } = await this.createStepsBundle({ format: 'esm', inputFiles, outfile: join(stepsRouteDir, '+server.js'), @@ -101,6 +109,7 @@ export const POST = async ({request}) => { ); await writeFile(stepsRouteFile, stepsRouteContent); + return manifest; } private async buildWorkflowsRoute({