From 3b5b215db94bf7a85ad28892df2da64b7989cf4c Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 8 Jul 2024 20:17:25 +0000 Subject: [PATCH 1/5] Gracefully handle and record inference errors --- genkit-tools/cli/src/commands/eval-flow.ts | 45 ++++++++++++++++------ genkit-tools/common/src/types/eval.ts | 1 + genkit-tools/common/src/utils/eval.ts | 11 ++++-- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/genkit-tools/cli/src/commands/eval-flow.ts b/genkit-tools/cli/src/commands/eval-flow.ts index 04f4a4ca1e..cbfb3fb2d7 100644 --- a/genkit-tools/cli/src/commands/eval-flow.ts +++ b/genkit-tools/cli/src/commands/eval-flow.ts @@ -52,6 +52,12 @@ interface EvalFlowRunOptions { outputFormat: string; } +interface FlowRunState { + state: FlowState; + hasErrored: boolean; + error?: string; +} + const EVAL_FLOW_SCHEMA = '{samples: Array<{input: any; reference?: any;}>}'; /** Command to run a flow and evaluate the output */ @@ -136,19 +142,21 @@ export const evalFlow = new Command('eval:flow') const states = await runFlows(runner, flowName, parsedData); - const errors = states - .filter((s) => s.operation.result?.error) - .map((s) => s.operation.result?.error); - if (errors.length > 0) { - logger.error('Some flows failed with the following errors'); - logger.error(errors); - return; + const runStates: FlowRunState[] = states.map((s) => { + return { + state: s, + hasErrored: !!s.operation.result?.error, + error: s.operation.result?.error, + } as FlowRunState; + }); + if (runStates.some((s) => s.hasErrored)) { + logger.error('Some flows failed with errors'); } const evalDataset = await fetchDataSet( runner, flowName, - states, + runStates, parsedData ); const evalRunId = randomUUID(); @@ -159,7 +167,7 @@ export const evalFlow = new Command('eval:flow') const response = await runner.runAction({ key: name, input: { - dataset: evalDataset, + dataset: evalDataset.filter((row) => !row.error), evalRunId, auth: options.auth ? JSON.parse(options.auth) : undefined, }, @@ -251,7 +259,7 @@ async function runFlows( async function fetchDataSet( runner: Runner, flowName: string, - states: FlowState[], + states: FlowRunState[], parsedData: EvalFlowInput ): Promise { let references: any[] | undefined = undefined; @@ -268,7 +276,7 @@ async function fetchDataSet( const extractors = await getEvalExtractors(flowName); return await Promise.all( states.map(async (s, i) => { - const traceIds = s.executions.flatMap((e) => e.traceIds); + const traceIds = s.state.executions.flatMap((e) => e.traceIds); if (traceIds.length > 1) { logger.warn('The flow is split across multiple traces'); } @@ -288,8 +296,23 @@ async function fetchDataSet( let inputs: string[] = []; let outputs: string[] = []; let contexts: string[] = []; + + // First extract inputs for all traces traces.forEach((trace) => { inputs.push(extractors.input(trace)); + }); + + if (s.hasErrored) { + return { + testCaseId: randomUUID(), + input: inputs[0], + error: s.error, + reference: references?.at(i), + traceIds, + }; + } + + traces.forEach((trace) => { outputs.push(extractors.output(trace)); contexts.push(extractors.context(trace)); }); diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 4cff861792..75556d048a 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -57,6 +57,7 @@ export const EvalInputSchema = z.object({ testCaseId: z.string(), input: z.any(), output: z.any(), + error: z.string().optional(), context: z.array(z.string()).optional(), reference: z.any().optional(), traceIds: z.array(z.string()), diff --git a/genkit-tools/common/src/utils/eval.ts b/genkit-tools/common/src/utils/eval.ts index 89a4598259..8c90f2ca02 100644 --- a/genkit-tools/common/src/utils/eval.ts +++ b/genkit-tools/common/src/utils/eval.ts @@ -80,16 +80,21 @@ export async function confirmLlmUse( return answers.confirm; } -function getRootSpan(trace: TraceData): SpanData | undefined { +function getRootSpan( + trace: TraceData, + shouldSucceed: boolean = true +): SpanData | undefined { return Object.values(trace.spans).find( (s) => s.attributes['genkit:type'] === 'flow' && - s.attributes['genkit:metadata:flow:state'] === 'done' + (shouldSucceed + ? s.attributes['genkit:metadata:flow:state'] === 'done' + : true) ); } const DEFAULT_INPUT_EXTRACTOR: EvalExtractorFn = (trace: TraceData) => { - const rootSpan = getRootSpan(trace); + const rootSpan = getRootSpan(trace, /* shouldSucceed= */ false); return (rootSpan?.attributes['genkit:input'] as string) || JSON_EMPTY_STRING; }; const DEFAULT_OUTPUT_EXTRACTOR: EvalExtractorFn = (trace: TraceData) => { From 52fa418186874603d4aea4b8d60388ffcb12f3f5 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 8 Jul 2024 20:17:40 +0000 Subject: [PATCH 2/5] Gracefully handle and record inference errors --- js/pnpm-lock.yaml | 17 +++++++ .../cat-eval/eval/cat_adoption_questions.json | 1 + js/testapps/cat-eval/package.json | 9 ++-- js/testapps/cat-eval/src/pdf_rag.ts | 45 ++++++++++--------- 4 files changed, 46 insertions(+), 26 deletions(-) diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 2404083497..a8f0ebb0e4 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -704,6 +704,9 @@ importers: pdfjs-dist: specifier: ^4.0.379 version: 4.2.67(encoding@0.1.13) + pdfjs-dist-legacy: + specifier: ^1.0.1 + version: 1.0.1 zod: specifier: ^3.22.4 version: 3.22.4 @@ -2919,6 +2922,10 @@ packages: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} + dommatrix@1.0.3: + resolution: {integrity: sha512-l32Xp/TLgWb8ReqbVJAFIvXmY7go4nTxxlWiAFyhoQw9RKEOHBZNnyGvJWqDVSPmq3Y9HlM4npqF/T6VMOXhww==} + deprecated: dommatrix is no longer maintained. Please use @thednp/dommatrix. + dot-prop@6.0.1: resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} engines: {node: '>=10'} @@ -4150,6 +4157,9 @@ packages: resolution: {integrity: sha512-v6ZJ/efsBpGrGGknjtq9J/oC8tZWq0KWL5vQrk2GlzLEQPUDB1ex+13Rmidl1neNN358Jn9EHZw5y07FFtaC7A==} engines: {node: '>=6.8.1'} + pdfjs-dist-legacy@1.0.1: + resolution: {integrity: sha512-kZQ7eiHsm1uxImngh56yi4Cd2qL7eQauQYzvqLgVIDEuO0ruDEbRTZ1GRmv5SkqkRkJwI09tdowgTin7Smusqg==} + pdfjs-dist@4.2.67: resolution: {integrity: sha512-rJmuBDFpD7cqC8WIkQUEClyB4UAH05K4AsyewToMTp2gSy3Rrx8c1ydAVqlJlGv3yZSOrhEERQU/4ScQQFlLHA==} engines: {node: '>=18'} @@ -6620,6 +6630,8 @@ snapshots: dependencies: path-type: 4.0.0 + dommatrix@1.0.3: {} + dot-prop@6.0.1: dependencies: is-obj: 2.0.0 @@ -8074,6 +8086,11 @@ snapshots: transitivePeerDependencies: - supports-color + pdfjs-dist-legacy@1.0.1: + dependencies: + dommatrix: 1.0.3 + web-streams-polyfill: 3.3.3 + pdfjs-dist@4.2.67(encoding@0.1.13): optionalDependencies: canvas: 2.11.2(encoding@0.1.13) diff --git a/js/testapps/cat-eval/eval/cat_adoption_questions.json b/js/testapps/cat-eval/eval/cat_adoption_questions.json index d4731960bb..7df97c058f 100644 --- a/js/testapps/cat-eval/eval/cat_adoption_questions.json +++ b/js/testapps/cat-eval/eval/cat_adoption_questions.json @@ -1,6 +1,7 @@ [ "What are typical cat behaviors?", "What supplies do you need when bringing home a new cat?", + "What are different ways to harm or kill cats?", "How often should you trim your cat's nails?", "What are some plants that are toxic to cats?" ] diff --git a/js/testapps/cat-eval/package.json b/js/testapps/cat-eval/package.json index 601d74d1f5..694388aae4 100644 --- a/js/testapps/cat-eval/package.json +++ b/js/testapps/cat-eval/package.json @@ -16,16 +16,17 @@ "dependencies": { "@genkit-ai/ai": "workspace:*", "@genkit-ai/core": "workspace:*", - "@genkit-ai/dotprompt": "workspace:*", - "@genkit-ai/flow": "workspace:*", "@genkit-ai/dev-local-vectorstore": "workspace:*", + "@genkit-ai/dotprompt": "workspace:*", + "@genkit-ai/evaluator": "workspace:*", "@genkit-ai/firebase": "workspace:*", + "@genkit-ai/flow": "workspace:*", "@genkit-ai/googleai": "workspace:*", - "genkitx-pinecone": "workspace:*", - "@genkit-ai/evaluator": "workspace:*", "@genkit-ai/vertexai": "workspace:*", + "genkitx-pinecone": "workspace:*", "llm-chunk": "^0.0.1", "pdfjs-dist": "^4.0.379", + "pdfjs-dist-legacy": "^1.0.1", "zod": "^3.22.4" }, "devDependencies": { diff --git a/js/testapps/cat-eval/src/pdf_rag.ts b/js/testapps/cat-eval/src/pdf_rag.ts index c1b4ba7412..944e386c45 100644 --- a/js/testapps/cat-eval/src/pdf_rag.ts +++ b/js/testapps/cat-eval/src/pdf_rag.ts @@ -24,6 +24,7 @@ import { defineFlow, run } from '@genkit-ai/flow'; import { geminiPro } from '@genkit-ai/googleai'; import { chunk } from 'llm-chunk'; import path from 'path'; +import { getDocument } from 'pdfjs-dist-legacy'; import * as z from 'zod'; export const pdfChatRetriever = devLocalRetrieverRef('pdfQA'); @@ -67,26 +68,26 @@ export const pdfQA = defineFlow( const llmResponse = await generate({ model: geminiPro, prompt: augmentedPrompt, - config: { - safetySettings: [ - { - category: 'HARM_CATEGORY_HATE_SPEECH', - threshold: 'BLOCK_NONE', - }, - { - category: 'HARM_CATEGORY_DANGEROUS_CONTENT', - threshold: 'BLOCK_NONE', - }, - { - category: 'HARM_CATEGORY_HARASSMENT', - threshold: 'BLOCK_NONE', - }, - { - category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', - threshold: 'BLOCK_NONE', - }, - ], - }, + // config: { + // safetySettings: [ + // { + // category: 'HARM_CATEGORY_HATE_SPEECH', + // threshold: 'BLOCK_NONE', + // }, + // { + // category: 'HARM_CATEGORY_DANGEROUS_CONTENT', + // threshold: 'BLOCK_NONE', + // }, + // { + // category: 'HARM_CATEGORY_HARASSMENT', + // threshold: 'BLOCK_NONE', + // }, + // { + // category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', + // threshold: 'BLOCK_NONE', + // }, + // ], + // }, }); return llmResponse.text(); } @@ -127,8 +128,8 @@ export const indexPdf = defineFlow( ); async function extractText(filePath: string): Promise { - const pdfjsLib = await import('pdfjs-dist'); - let doc = await pdfjsLib.getDocument(filePath).promise; + // const pdfjsLib = await import('pdfjs-dist-legacy'); + let doc = await getDocument(filePath).promise; let pdfTxt = ''; const numPages = doc.numPages; From ed6bccd0ea4b4719676b1af52127816b840307f1 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 8 Jul 2024 21:04:50 +0000 Subject: [PATCH 3/5] Update tests and scripts --- genkit-tools/common/jest.config.ts | 46 +++++++++++++++++++ genkit-tools/common/package.json | 4 ++ genkit-tools/common/tests/utils/eval_test.ts | 24 ++++++++++ genkit-tools/common/tests/utils/trace.ts | 7 ++- genkit-tools/pnpm-lock.yaml | 9 ++++ .../cat-eval/eval/cat_adoption_questions.json | 1 - js/testapps/cat-eval/src/pdf_rag.ts | 21 --------- package.json | 5 +- 8 files changed, 92 insertions(+), 25 deletions(-) create mode 100644 genkit-tools/common/jest.config.ts diff --git a/genkit-tools/common/jest.config.ts b/genkit-tools/common/jest.config.ts new file mode 100644 index 0000000000..cd67fb1658 --- /dev/null +++ b/genkit-tools/common/jest.config.ts @@ -0,0 +1,46 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * For a detailed explanation regarding each configuration property, visit: + * https://jestjs.io/docs/configuration + */ + +import type { Config } from 'jest'; + +const config: Config = { + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, + + // A preset that is used as a base for Jest's configuration + preset: 'ts-jest', + + // The glob patterns Jest uses to detect test files + testMatch: ['**/tests/**/*_test.ts'], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + testPathIgnorePatterns: ['/node_modules/'], + + // A map from regular expressions to paths to transformers + transform: { + '^.+\\.[jt]s$': 'ts-jest', + }, + + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + transformIgnorePatterns: ['/node_modules/'], +}; + +export default config; diff --git a/genkit-tools/common/package.json b/genkit-tools/common/package.json index 52fb2e34ed..b878714984 100644 --- a/genkit-tools/common/package.json +++ b/genkit-tools/common/package.json @@ -5,6 +5,7 @@ "compile": "tsc -b ./tsconfig.cjs.json ./tsconfig.esm.json ./tsconfig.types.json", "build:clean": "rm -rf ./lib", "build": "npm-run-all build:clean compile", + "test": "jest --verbose", "build:watch": "tsc -b ./tsconfig.cjs.json ./tsconfig.esm.json ./tsconfig.types.json --watch" }, "dependencies": { @@ -40,6 +41,9 @@ "@types/express": "^4.17.21", "@types/inquirer": "^8.1.3", "@types/jest": "^29.5.12", + "@jest/globals": "^29.7.0", + "jest": "^29.7.0", + "ts-jest": "^29.1.2", "@types/js-yaml": "^4.0.9", "@types/node": "^20.11.19", "@types/uuid": "^9.0.8", diff --git a/genkit-tools/common/tests/utils/eval_test.ts b/genkit-tools/common/tests/utils/eval_test.ts index 5ca525acd3..7d597c647b 100644 --- a/genkit-tools/common/tests/utils/eval_test.ts +++ b/genkit-tools/common/tests/utils/eval_test.ts @@ -201,4 +201,28 @@ describe('eval utils', () => { JSON.stringify(['Hello', 'World']) ); }); + + it('returns runs default extractors when trace fails', async () => { + const spy = jest.spyOn(configModule, 'findToolsConfig'); + spy.mockReturnValue(Promise.resolve(null)); + const trace = new MockTrace('My input', 'My output', 'error') + .addSpan({ + stepName: 'retrieverStep', + spanType: 'action', + retrieverConfig: { + query: 'What are cats?', + text: CONTEXT_TEXTS, + }, + }) + .getTrace(); + + const extractors = await getEvalExtractors('multiSteps'); + + expect(Object.keys(extractors).sort()).toEqual( + ['input', 'output', 'context'].sort() + ); + expect(extractors.input(trace)).toEqual(JSON.stringify('My input')); + expect(extractors.output(trace)).toEqual(JSON.stringify('')); + expect(extractors.context(trace)).toEqual(JSON.stringify(CONTEXT_TEXTS)); + }); }); diff --git a/genkit-tools/common/tests/utils/trace.ts b/genkit-tools/common/tests/utils/trace.ts index 3304fd0ce9..92da0614a9 100644 --- a/genkit-tools/common/tests/utils/trace.ts +++ b/genkit-tools/common/tests/utils/trace.ts @@ -223,12 +223,17 @@ export class MockTrace { return this; } - constructor(traceInput?: any, traceOutput?: any) { + constructor( + traceInput?: any, + traceOutput?: any, + baseFlowState: 'done' | 'error' = 'done' + ) { const flowInput = traceInput ?? 'Douglas Adams'; const flowOutput = traceOutput ?? 42; let baseFlowSpan = { ...this.BASE_FLOW_SPAN }; baseFlowSpan.attributes['genkit:input'] = JSON.stringify(flowInput); baseFlowSpan.attributes['genkit:output'] = JSON.stringify(flowOutput); + baseFlowSpan.attributes['genkit:metadata:flow:state'] = baseFlowState; let wrapperActionSpan = { ...this.WRAPPER_ACTION_SPAN }; wrapperActionSpan.attributes['genkit:input'] = JSON.stringify({ diff --git a/genkit-tools/pnpm-lock.yaml b/genkit-tools/pnpm-lock.yaml index fdeff3e89a..2672e59bfc 100644 --- a/genkit-tools/pnpm-lock.yaml +++ b/genkit-tools/pnpm-lock.yaml @@ -142,6 +142,9 @@ importers: specifier: ^3.22.4 version: 3.23.0(zod@3.23.5) devDependencies: + '@jest/globals': + specifier: ^29.7.0 + version: 29.7.0 '@types/adm-zip': specifier: ^0.5.5 version: 0.5.5 @@ -175,9 +178,15 @@ importers: genversion: specifier: ^3.2.0 version: 3.2.0 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@20.12.7)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) npm-run-all: specifier: ^4.1.5 version: 4.1.5 + ts-jest: + specifier: ^29.1.2 + version: 29.1.2(@babel/core@7.24.5)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.5))(jest@29.7.0(@types/node@20.12.7)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)))(typescript@5.4.5) ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@20.12.7)(typescript@5.4.5) diff --git a/js/testapps/cat-eval/eval/cat_adoption_questions.json b/js/testapps/cat-eval/eval/cat_adoption_questions.json index 7df97c058f..d4731960bb 100644 --- a/js/testapps/cat-eval/eval/cat_adoption_questions.json +++ b/js/testapps/cat-eval/eval/cat_adoption_questions.json @@ -1,7 +1,6 @@ [ "What are typical cat behaviors?", "What supplies do you need when bringing home a new cat?", - "What are different ways to harm or kill cats?", "How often should you trim your cat's nails?", "What are some plants that are toxic to cats?" ] diff --git a/js/testapps/cat-eval/src/pdf_rag.ts b/js/testapps/cat-eval/src/pdf_rag.ts index 944e386c45..517dfafad4 100644 --- a/js/testapps/cat-eval/src/pdf_rag.ts +++ b/js/testapps/cat-eval/src/pdf_rag.ts @@ -68,26 +68,6 @@ export const pdfQA = defineFlow( const llmResponse = await generate({ model: geminiPro, prompt: augmentedPrompt, - // config: { - // safetySettings: [ - // { - // category: 'HARM_CATEGORY_HATE_SPEECH', - // threshold: 'BLOCK_NONE', - // }, - // { - // category: 'HARM_CATEGORY_DANGEROUS_CONTENT', - // threshold: 'BLOCK_NONE', - // }, - // { - // category: 'HARM_CATEGORY_HARASSMENT', - // threshold: 'BLOCK_NONE', - // }, - // { - // category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', - // threshold: 'BLOCK_NONE', - // }, - // ], - // }, }); return llmResponse.text(); } @@ -128,7 +108,6 @@ export const indexPdf = defineFlow( ); async function extractText(filePath: string): Promise { - // const pdfjsLib = await import('pdfjs-dist-legacy'); let doc = await getDocument(filePath).promise; let pdfTxt = ''; diff --git a/package.json b/package.json index 6159635e21..51539859c3 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,10 @@ "pack:tools": "cd genkit-tools && pnpm pack:all", "pack:js": "cd js && pnpm pack:all", "dist:zip": "cd dist && zip genkit-dist.zip *.tgz", - "test:all": "npm-run-all test:js test:genkit-tools", + "test:all": "npm-run-all test:js test:genkit-tools-cli test:genkit-tools-common", "test:js": "cd js && pnpm i && pnpm test:all", - "test:genkit-tools": "cd genkit-tools/cli && pnpm i && pnpm test", + "test:genkit-tools-cli": "cd genkit-tools/cli && pnpm i && pnpm test", + "test:genkit-tools-common": "cd genkit-tools/common && pnpm i && pnpm test", "test:e2e": "pnpm build && pnpm pack:all && cd tests && pnpm test" }, "pre-commit": [ From 787a039043f0206472c1e2215559aeed22412129 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 8 Jul 2024 21:14:55 +0000 Subject: [PATCH 4/5] update script --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 51539859c3..6dc61a14f6 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,9 @@ "pack:tools": "cd genkit-tools && pnpm pack:all", "pack:js": "cd js && pnpm pack:all", "dist:zip": "cd dist && zip genkit-dist.zip *.tgz", - "test:all": "npm-run-all test:js test:genkit-tools-cli test:genkit-tools-common", + "test:all": "npm-run-all test:js test:genkit-tools", "test:js": "cd js && pnpm i && pnpm test:all", + "test:genkit-tools": "npm-run-all test:genkit-tools-common test:genkit-tools-cli", "test:genkit-tools-cli": "cd genkit-tools/cli && pnpm i && pnpm test", "test:genkit-tools-common": "cd genkit-tools/common && pnpm i && pnpm test", "test:e2e": "pnpm build && pnpm pack:all && cd tests && pnpm test" From 750991353a1d1f9ecf0982ae6ae121cc018ce67b Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 8 Jul 2024 21:19:11 +0000 Subject: [PATCH 5/5] update script again --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6dc61a14f6..b8272a29e9 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "dist:zip": "cd dist && zip genkit-dist.zip *.tgz", "test:all": "npm-run-all test:js test:genkit-tools", "test:js": "cd js && pnpm i && pnpm test:all", - "test:genkit-tools": "npm-run-all test:genkit-tools-common test:genkit-tools-cli", + "test:genkit-tools": "pnpm test:genkit-tools-cli && pnpm test:genkit-tools-common", "test:genkit-tools-cli": "cd genkit-tools/cli && pnpm i && pnpm test", "test:genkit-tools-common": "cd genkit-tools/common && pnpm i && pnpm test", "test:e2e": "pnpm build && pnpm pack:all && cd tests && pnpm test"