From 2acbd410d47af8f9422650ac07c868ca525edd5f Mon Sep 17 00:00:00 2001 From: David Zeng Date: Wed, 30 Mar 2022 19:43:39 +0800 Subject: [PATCH] Add workflow instrumentation support --- dist/index.cjs.js | 2 +- dist/index.d.ts | 15 ++++- dist/index.js | 2 +- package.json | 2 +- src/ReactElementWorkflowBuilder.ts | 25 +++------ src/Workflow.ts | 1 + src/WorkflowBuilder.ts | 10 +++- src/WorkflowExecutor.ts | 54 ++++++++++++++++++ test/WorkflowBuildNodeName.test.tsx | 87 +++++++++++++++++++++++++++++ test/WorkflowInst.test.tsx | 27 +++++++++ 10 files changed, 202 insertions(+), 23 deletions(-) create mode 100644 test/WorkflowBuildNodeName.test.tsx create mode 100644 test/WorkflowInst.test.tsx diff --git a/dist/index.cjs.js b/dist/index.cjs.js index 4b6b983..75143ac 100644 --- a/dist/index.cjs.js +++ b/dist/index.cjs.js @@ -1 +1 @@ -"use strict";function t(t){return null}function e(t){return null}function n(t){return null}function r(t){return null}function o(){return{run:function(t){return t}}}Object.defineProperty(exports,"__esModule",{value:!0});var i,u,s={},a=-1;function f(t,e){s[t]=e}function d(t){return t in s?-1:(s[t]=++a,a)}function p(t){return t in s?s[t]:-1}function l(t){for(;;){if("function"!=typeof t.type)throw"non functional component";if(t.type==e)break;if("object"!=typeof(t=t.type.call(null,t.props)))throw"non react element"}return t}function c(e,o){for(var i=e.props,u=e.props.children,s=o+"."+i.name,a=i.params,h=[],v=0,g=u;v0&&o[o.length-1])||6!==i[0]&&2!==i[0])){u=0;continue}if(3===i[0]&&(!o||i[1]>o[0]&&i[1]e.id?1:-1})),{inputs:t,outputs:n,nodes:r,binding:o,zeroDepNodes:e}},input:function(e){r=r.concat(e);for(var n=0,o=e;n=0&&u.splice(e,1),(i=a.call(f,n.id,t)).length&&s.push(o(i))),[2]}))}))})).catch((function(){r=exports.ExecutionStatus.Failure,f.cancel()}));s.push(p)}else{var l=a.call(f,n.id,i);t=t.concat(l)}};t.length&&"break"!==e(););p.label=1;case 1:return s.length>0?(i=s.splice(0,s.length),[4,Promise.all(i)]):[3,3];case 2:return p.sent(),[3,1];case 3:return n(!0),[2]}}))}))}))}var d,p,l,c,h;return x(this,(function(u){switch(u.label){case 0:for(n>0&&setTimeout((function(){r==exports.ExecutionStatus.Running&&f.cancel()}),n),r=exports.ExecutionStatus.Running,d=[],p=0;p=t.nodes.length)return exports.WorkflowValidationStatus.InputOutOfNodes}for(var u=0,s=t.zeroDepNodes;u=t.nodes.length)return exports.WorkflowValidationStatus.ZeroDepOutOfNodes}for(var f=0,d=Object.keys(t.outputs);f=e)return exports.WorkflowValidationStatus.OutputOutOfNodes}for(var c in t.binding){var v=parseInt(c);if(v<0||v>=e)return exports.WorkflowValidationStatus.BindingOutOfNodes;for(var g=0,x=t.binding[v];g=e)return exports.WorkflowValidationStatus.BindingOutOfNodes}}for(var b=0,y=t.nodes;b0&&o[o.length-1])||6!==i[0]&&2!==i[0])){u=0;continue}if(3===i[0]&&(!o||i[1]>o[0]&&i[1]n.id?1:-1})),{inputs:t,outputs:e,nodes:r,binding:o,zeroDepNodes:n,nodeNames:i}},input:function(n){r=r.concat(n);for(var e=0,o=n;e=0;r--)if(p[r].id==t){p[r].status=e,p[r].end=(new Date).getTime();break}}var l={state:function(){return r},inst:function(t){f=t},stats:function(){return p},workflow:function(){return t},reset:function(){r=exports.ExecutionStatus.NotStarted,o={},i={},u=[],a=[],p=[]},setTimeout:function(t){e=t},cancel:function(){if(r==exports.ExecutionStatus.Running){r=exports.ExecutionStatus.Cancelled;for(var t=0,n=u;t=0&&u.splice(n,1),(i=s.call(l,e.id,t)).length&&a.push(o(i))),[2]}))}))})).catch((function(){l.cancel(),r=exports.ExecutionStatus.Failure,d(e.id,exports.ExecutionStatus.Failure),i=void 0}));a.push(p)}else{var c=s.call(l,e.id,i);t=t.concat(c)}};t.length&&"break"!==n(););p.label=1;case 1:return a.length>0?(i=a.splice(0,a.length),[4,Promise.all(i)]):[3,3];case 2:return p.sent(),[3,1];case 3:return e(!0),[2]}}))}))}))}var f,p,c,v,g;return O(this,(function(u){switch(u.label){case 0:for(e>0&&setTimeout((function(){r==exports.ExecutionStatus.Running&&l.cancel()}),e),r=exports.ExecutionStatus.Running,f=[],p=0;p=t.nodes.length)return exports.WorkflowValidationStatus.InputOutOfNodes}for(var u=0,a=t.zeroDepNodes;u=t.nodes.length)return exports.WorkflowValidationStatus.ZeroDepOutOfNodes}for(var f=0,p=Object.keys(t.outputs);f=n)return exports.WorkflowValidationStatus.OutputOutOfNodes}for(var c in t.binding){var v=parseInt(c);if(v<0||v>=n)return exports.WorkflowValidationStatus.BindingOutOfNodes;for(var h=0,x=t.binding[v];h=n)return exports.WorkflowValidationStatus.BindingOutOfNodes}}for(var O=0,N=t.nodes;O; binding: Record; + nodeNames?: Record; } declare enum WorkflowValidationStatus { OK = 0, @@ -33,7 +34,7 @@ declare enum WorkflowValidationStatus { declare function dumpWorkflow(workflow: Workflow): string; declare function validateWorkflow(workflow: Workflow): WorkflowValidationStatus; -declare function buildJsxWorkflow(elementDefinition: React.ReactElement): Workflow; +declare function buildJsxWorkflow(elementDefinition: React.ReactElement, addNodeName?: boolean): Workflow; interface WorkflowProps { children: JSX.Element[]; @@ -66,13 +67,23 @@ declare enum ExecutionStatus { Failure = 3, Done = 4 } +interface NodeExecutionStatus { + status: ExecutionStatus; + id: number; + start: number; + end: number; + name?: string; +} interface WorkflowExecutor { cancel(): void; run(...params: any[]): Promise; setTimeout(timeout: number): void; reset(): void; state(): ExecutionStatus; + inst(inst: boolean): void; + workflow(): Workflow; + stats(): NodeExecutionStatus[]; } declare function createWorkflowExecutor(wf: Workflow): WorkflowExecutor; -export { ExecutionStatus, InputNodeComponent, InputNodeProps, NodeComponent, NodeProps, OutputNodeComponent, OutputNodeProps, Workflow, WorkflowComponent, WorkflowExecutionNode, WorkflowExecutor, WorkflowInputProps, WorkflowNode, WorkflowProps, WorkflowValidationStatus, buildJsxWorkflow, createWorkflowExecutor, dumpWorkflow, unitNodeGenerator, validateWorkflow }; +export { ExecutionStatus, InputNodeComponent, InputNodeProps, NodeComponent, NodeExecutionStatus, NodeProps, OutputNodeComponent, OutputNodeProps, Workflow, WorkflowComponent, WorkflowExecutionNode, WorkflowExecutor, WorkflowInputProps, WorkflowNode, WorkflowProps, WorkflowValidationStatus, buildJsxWorkflow, createWorkflowExecutor, dumpWorkflow, unitNodeGenerator, validateWorkflow }; diff --git a/dist/index.js b/dist/index.js index acf1412..c58a38d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1 +1 @@ -function n(n){return null}function e(n){return null}function t(n){return null}function r(n){return null}function i(){return{run:function(n){return n}}}var o,u,f={},a=-1;function s(n,e){f[n]=e}function d(n){return n in f?-1:(f[n]=++a,a)}function c(n){return n in f?f[n]:-1}function p(n){for(;;){if("function"!=typeof n.type)throw"non functional component";if(n.type==e)break;if("object"!=typeof(n=n.type.call(null,n.props)))throw"non react element"}return n}function l(e,i){for(var o=e.props,u=e.props.children,f=i+"."+o.name,a=o.params,h=[],g=0,v=u;ge.id?1:-1})),{inputs:n,outputs:t,nodes:r,binding:i,zeroDepNodes:e}},input:function(e){r=r.concat(e);for(var t=0,i=e;t=n.nodes.length)return o.InputOutOfNodes}for(var f=0,a=n.zeroDepNodes;f=n.nodes.length)return o.ZeroDepOutOfNodes}for(var d=0,c=Object.keys(n.outputs);d=e)return o.OutputOutOfNodes}for(var h in n.binding){var v=parseInt(h);if(v<0||v>=e)return o.BindingOutOfNodes;for(var b=0,y=n.binding[v];b=e)return o.BindingOutOfNodes}}for(var O=0,N=n.nodes;O0&&i[i.length-1])||6!==o[0]&&2!==o[0])){u=0;continue}if(3===o[0]&&(!i||o[1]>i[0]&&o[1]=0&&f.splice(e,1),(o=s.call(d,t.id,n)).length&&a.push(i(o))),[2]}))}))})).catch((function(){r=u.Failure,d.cancel()}));a.push(p)}else{var l=s.call(d,t.id,o);n=n.concat(l)}};n.length&&"break"!==e(););p.label=1;case 1:return a.length>0?(o=a.splice(0,a.length),[4,Promise.all(o)]):[3,3];case 2:return p.sent(),[3,1];case 3:return t(!0),[2]}}))}))}))}var c,p,l,h,g;return N(this,(function(f){switch(f.label){case 0:for(t>0&&setTimeout((function(){r==u.Running&&d.cancel()}),t),r=u.Running,c=[],p=0;pe.id?1:-1})),{inputs:n,outputs:t,nodes:r,binding:i,zeroDepNodes:e,nodeNames:o}},input:function(e){r=r.concat(e);for(var t=0,i=e;t=n.nodes.length)return o.InputOutOfNodes}for(var a=0,s=n.zeroDepNodes;a=n.nodes.length)return o.ZeroDepOutOfNodes}for(var d=0,c=Object.keys(n.outputs);d=e)return o.OutputOutOfNodes}for(var g in n.binding){var v=parseInt(g);if(v<0||v>=e)return o.BindingOutOfNodes;for(var O=0,b=n.binding[v];O=e)return o.BindingOutOfNodes}}for(var y=0,N=n.nodes;y0&&i[i.length-1])||6!==o[0]&&2!==o[0])){u=0;continue}if(3===o[0]&&(!i||o[1]>i[0]&&o[1]=0;r--)if(c[r].id==n){c[r].status=t,c[r].end=(new Date).getTime();break}}var p={state:function(){return r},inst:function(n){d=n},stats:function(){return c},workflow:function(){return n},reset:function(){r=u.NotStarted,i={},o={},a=[],s=[],c=[]},setTimeout:function(n){t=n},cancel:function(){if(r==u.Running){r=u.Cancelled;for(var n=0,e=a;n=0&&a.splice(e,1),(o=f.call(p,t.id,n)).length&&s.push(i(o))),[2]}))}))})).catch((function(){p.cancel(),r=u.Failure,l(t.id,u.Failure),o=void 0}));s.push(c)}else{var g=f.call(p,t.id,o);n=n.concat(g)}};n.length&&"break"!==e(););c.label=1;case 1:return s.length>0?(o=s.splice(0,s.length),[4,Promise.all(o)]):[3,3];case 2:return c.sent(),[3,1];case 3:return t(!0),[2]}}))}))}))}var d,c,g,v,h;return m(this,(function(a){switch(a.label){case 0:for(t>0&&setTimeout((function(){r==u.Running&&p.cancel()}),t),r=u.Running,d=[],c=0;c = {} +let workflowExecutionNodeIdToNameMap: Record = {} let workflowNodeId = -1 function setNodeId(name: string, id: number) { @@ -19,6 +20,7 @@ function genNodeId(name: string) : number { } workflowNodeNameToIdMap[name] = ++workflowNodeId + workflowExecutionNodeIdToNameMap[workflowNodeId] = name return workflowNodeId } @@ -98,7 +100,7 @@ function buildSubWorkflow(element: React.ReactElement, prefix: string) : Workflo return nodes } -export function buildJsxWorkflow(elementDefinition: React.ReactElement) : Workflow{ +export function buildJsxWorkflow(elementDefinition: React.ReactElement, addNodeName: boolean = false) : Workflow{ // eslint-disable-line const element: React.ReactElement = resolveTillWorkflowComponent(elementDefinition) const props: WorkflowProps = element.props; const children : React.ReactElement[] = props.children @@ -112,10 +114,6 @@ export function buildJsxWorkflow(elementDefinition: React.ReactElement) : Workfl const params: string[] = childProps.params for (const param of params) { const id = genNodeId(workflowName + "." + param) - if (id < 0) { - throw "duplicate node name found " + workflowName + "." + param - } - inputNodes.push({ id: id, deps: [], @@ -127,10 +125,6 @@ export function buildJsxWorkflow(elementDefinition: React.ReactElement) : Workfl const name: string = childProps.name const dep: string = childProps.dep const id = getNodeId(workflowName + "." + dep) - if (id < 0) { - throw "dependency must be defined before it's used" - } - setNodeId(workflowName + "." + name, id) outputs[id] = name } else if (child.type == NodeComponent) { @@ -141,19 +135,11 @@ export function buildJsxWorkflow(elementDefinition: React.ReactElement) : Workfl if (deps && deps.length) { for (const dep of deps) { const id = getNodeId(workflowName + "." + dep) - if (id < 0) { - throw "dependency must be defined before it's used" - } - depIds.push(id) } } const id = genNodeId(workflowName + "." + name) - if (id < 0) { - throw "duplicate node name found " + workflowName + "." + name - } - innerNodes.push({ id: id, deps: depIds, @@ -172,5 +158,10 @@ export function buildJsxWorkflow(elementDefinition: React.ReactElement) : Workfl .next(innerNodes) .output(outputs); + if (addNodeName) { + workflowBuilder.nodeNames(Object.assign({}, workflowExecutionNodeIdToNameMap)) + } + + workflowExecutionNodeIdToNameMap = {} return workflowBuilder.build() } diff --git a/src/Workflow.ts b/src/Workflow.ts index 51ccb02..b1bb690 100644 --- a/src/Workflow.ts +++ b/src/Workflow.ts @@ -6,6 +6,7 @@ export interface Workflow { nodes: WorkflowNode[]; outputs: Record; binding: Record; + nodeNames?: Record; } export enum WorkflowValidationStatus { diff --git a/src/WorkflowBuilder.ts b/src/WorkflowBuilder.ts index dc8cf7d..d840ed1 100644 --- a/src/WorkflowBuilder.ts +++ b/src/WorkflowBuilder.ts @@ -6,6 +6,7 @@ export interface WorkflowBuilder { input(nodes: WorkflowNode[]) : WorkflowBuilder; next(nodes: WorkflowNode[]) : WorkflowBuilder; output(output: Record) : WorkflowBuilder; + nodeNames(namesMap: Record) : WorkflowBuilder; } export function createWorkflowBuilder() : WorkflowBuilder { @@ -14,6 +15,7 @@ export function createWorkflowBuilder() : WorkflowBuilder { let outputs: Record = {} let nodes: WorkflowNode[] = [] const binding: Record = {} + let idToNameMap: Record | undefined = undefined const builder : WorkflowBuilder = { build: () => { nodes.sort((a: WorkflowNode, b: WorkflowNode) => { @@ -25,7 +27,8 @@ export function createWorkflowBuilder() : WorkflowBuilder { outputs: outputs, nodes: nodes, binding: binding, - zeroDepNodes: zeroDepNodes + zeroDepNodes: zeroDepNodes, + nodeNames: idToNameMap } return workflow @@ -63,6 +66,11 @@ export function createWorkflowBuilder() : WorkflowBuilder { nodes = nodes.concat(nextNodes) return builder; + }, + + nodeNames: (nodeMap: Record) => { + idToNameMap = nodeMap + return builder } } diff --git a/src/WorkflowExecutor.ts b/src/WorkflowExecutor.ts index af2229d..b0e2a54 100644 --- a/src/WorkflowExecutor.ts +++ b/src/WorkflowExecutor.ts @@ -9,12 +9,23 @@ export enum ExecutionStatus { Done = 4, } +export interface NodeExecutionStatus { + status: ExecutionStatus; + id: number; + start: number; + end: number; + name?: string; +} + export interface WorkflowExecutor { cancel() : void; run(...params: any[]) : Promise; setTimeout(timeout: number) : void; reset() : void; state() : ExecutionStatus; + inst(inst: boolean) : void; + workflow() : Workflow; + stats(): NodeExecutionStatus[]; } export function createWorkflowExecutor(wf: Workflow) { @@ -26,6 +37,7 @@ export function createWorkflowExecutor(wf: Workflow) { let runningNodes: WorkflowRunTimeNode[] = []; let pendingPromises: Promise[] = [] function update(id: number, oriResult: any) : WorkflowRunTimeNode[] { + updateNodeExecutionStatus(id, ExecutionStatus.Done) if (typeof oriResult === 'undefined') { return [] } @@ -71,17 +83,55 @@ export function createWorkflowExecutor(wf: Workflow) { return newNodesToRun } + let enableInst = false + let nodeExecutionStatus: NodeExecutionStatus[] = [] + function updateNodeExecutionStatus(id: number, status: ExecutionStatus) { + if (enableInst) { + if (status == ExecutionStatus.Running) { + nodeExecutionStatus.push({ + id: id, + start: new Date().getTime(), + end: 0, + status: status, + name: workflow.nodeNames ? ((id in workflow.nodeNames) ? workflow.nodeNames[id] : undefined) : undefined + }) + } else { + // search from last one + for (let inx = nodeExecutionStatus.length - 1; inx >= 0; inx--) { + if (nodeExecutionStatus[inx].id == id) { + nodeExecutionStatus[inx].status = status + nodeExecutionStatus[inx].end = new Date().getTime() + break + } + } + } + } + } + const executor: WorkflowExecutor = { state() : ExecutionStatus { return status }, + inst(inst: boolean) : void { + enableInst = inst + }, + + stats() : NodeExecutionStatus[] { + return nodeExecutionStatus + }, + + workflow() : Workflow { + return wf + }, + reset() : void { status = ExecutionStatus.NotStarted; intermediateResults = {} output = {} runningNodes = []; pendingPromises = [] + nodeExecutionStatus = [] }, setTimeout(timeout: number) : void { @@ -96,6 +146,7 @@ export function createWorkflowExecutor(wf: Workflow) { status = ExecutionStatus.Cancelled for (const node of runningNodes) { if (node.node.cancel) { + updateNodeExecutionStatus(node.id, ExecutionStatus.Cancelled) node.node.cancel() } } @@ -134,11 +185,13 @@ export function createWorkflowExecutor(wf: Workflow) { const node: WorkflowRunTimeNode = nodesToRun.pop()! let result = undefined try{ + updateNodeExecutionStatus(node.id, ExecutionStatus.Running) result = node.node.run(...node.inputs) } catch(err) { executor.cancel() status = ExecutionStatus.Failure + updateNodeExecutionStatus(node.id, ExecutionStatus.Failure) result = undefined break } @@ -163,6 +216,7 @@ export function createWorkflowExecutor(wf: Workflow) { .catch(() => { executor.cancel() status = ExecutionStatus.Failure + updateNodeExecutionStatus(node.id, ExecutionStatus.Failure) result = undefined }) diff --git a/test/WorkflowBuildNodeName.test.tsx b/test/WorkflowBuildNodeName.test.tsx new file mode 100644 index 0000000..788ae80 --- /dev/null +++ b/test/WorkflowBuildNodeName.test.tsx @@ -0,0 +1,87 @@ +import { buildJsxWorkflow } from "../src/ReactElementWorkflowBuilder"; +import { validateWorkflow, WorkflowValidationStatus } from "../src/Workflow"; +import { InputNodeComponent, NodeComponent, OutputNodeComponent, WorkflowComponent, WorkflowInputProps } from "../src/WorkflowComponent"; +import { unitNodeGenerator } from "../src/WorkflowNode"; +import { add, AddWithDoubleWorkflow, double, DoubleWorkflow } from "./NodeWorkflowExample"; +import React from "react"; +import { createWorkflowExecutor } from "../src/WorkflowExecutor"; + +test("workflow node name test", async () => { + const workflowJsx = ( + + + + + + ) + const workflow = buildJsxWorkflow(workflowJsx, true) + expect(validateWorkflow(workflow)).toBe(WorkflowValidationStatus.OK) + expect(workflow.nodes.length).toBe(6) + const ndoeNameMap = workflow.nodeNames + expect(Object.keys(ndoeNameMap).length).toBe(6) + expect(ndoeNameMap[0]).toBe(".num1") + expect(ndoeNameMap[1]).toBe(".num2") + expect(ndoeNameMap[2]).toBe(".num3") + expect(ndoeNameMap[3]).toBe(".add") + expect(ndoeNameMap[4]).toBe(".double") + expect(ndoeNameMap[5]).toBe(".finalAdd") +}) + +function ComputationWorkflow(props: WorkflowInputProps) { + return ( + + + + + + ) +} + +test("nested workflow node name test", async () => { + const workflowJsx = ( + + + + + + + + ) + const workflow = buildJsxWorkflow(workflowJsx, true) + expect(validateWorkflow(workflow)).toBe(WorkflowValidationStatus.OK) + expect(workflow.nodes.length).toBe(10) + const ndoeNameMap = workflow.nodeNames + expect(Object.keys(ndoeNameMap).length).toBe(10) + expect(ndoeNameMap[0]).toBe(".num1") + expect(ndoeNameMap[1]).toBe(".num2") + expect(ndoeNameMap[2]).toBe(".num3") + expect(ndoeNameMap[3]).toBe(".add") + expect(ndoeNameMap[4]).toBe(".double") + expect(ndoeNameMap[5]).toBe(".add1") + expect(ndoeNameMap[6]).toBe(".comp.add") + expect(ndoeNameMap[7]).toBe(".comp.double") + expect(ndoeNameMap[8]).toBe(".comp.finalAdd") + expect(ndoeNameMap[9]).toBe(".finalAdd") +}) + +test("nested nested workflow node name test", async () => { + let workflowJsx = ( + + + + + + ) + let workflow = buildJsxWorkflow(workflowJsx, true) + expect(validateWorkflow(workflow)).toBe(WorkflowValidationStatus.OK) + expect(workflow.nodes.length).toBe(7) + const ndoeNameMap = workflow.nodeNames + expect(Object.keys(ndoeNameMap).length).toBe(7) + expect(ndoeNameMap[0]).toBe(".num1") + expect(ndoeNameMap[1]).toBe(".num2") + expect(ndoeNameMap[2]).toBe(".num3") + expect(ndoeNameMap[3]).toBe(".add.double.double") + expect(ndoeNameMap[4]).toBe(".add.add") + expect(ndoeNameMap[5]).toBe(".double.double") + expect(ndoeNameMap[6]).toBe(".finalAdd") +}) diff --git a/test/WorkflowInst.test.tsx b/test/WorkflowInst.test.tsx new file mode 100644 index 0000000..dc08d99 --- /dev/null +++ b/test/WorkflowInst.test.tsx @@ -0,0 +1,27 @@ +import React from "react"; +import { buildJsxWorkflow } from "../src/ReactElementWorkflowBuilder"; +import { validateWorkflow, WorkflowValidationStatus } from "../src/Workflow"; +import { InputNodeComponent, NodeComponent, OutputNodeComponent, WorkflowComponent } from "../src/WorkflowComponent"; +import { createWorkflowExecutor, ExecutionStatus } from "../src/WorkflowExecutor"; +import { asyncAdd, asyncDouble } from "./NodeWorkflowExample"; + +test("workflow inst test", async () => { + let workflowJsx = ( + + + + + + ) + let workflow = buildJsxWorkflow(workflowJsx) + expect(validateWorkflow(workflow)).toBe(WorkflowValidationStatus.OK) + let workflowExecutor = createWorkflowExecutor(workflow) + workflowExecutor.inst(true) + let result = await workflowExecutor.run(1, 2, 3) + expect(result["res"]).toBe(9) + let stats = workflowExecutor.stats() + expect(stats.length).toBe(workflow.nodes.length) + for (const nodeStat of stats) { + expect(nodeStat.status).toBe(ExecutionStatus.Done) + } +})