From c334063e93e398e65aec04c139de57951bf0b234 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Tue, 24 Sep 2024 21:46:06 +0000 Subject: [PATCH 1/9] add targetAction --- genkit-tools/common/src/eval/localFileDatasetStore.ts | 11 +++++++++-- genkit-tools/common/src/types/apis.ts | 2 ++ genkit-tools/common/src/types/eval.ts | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 7977d81e1c..13f2d29fb1 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -64,12 +64,17 @@ export class LocalFileDatasetStore implements DatasetStore { } async createDataset(req: CreateDatasetRequest): Promise { - return this.createDatasetInternal(req.data, req.datasetId); + return this.createDatasetInternal( + req.data, + req.datasetId, + req.targetAction + ); } private async createDatasetInternal( data: Dataset, - datasetId?: string + datasetId?: string, + targetAction?: string ): Promise { const id = await this.generateDatasetId(datasetId); const filePath = path.resolve(this.storeRoot, this.generateFileName(id)); @@ -89,6 +94,7 @@ export class LocalFileDatasetStore implements DatasetStore { datasetId: id, size: Array.isArray(data) ? data.length : data.samples.length, version: 1, + targetAction, createTime: now, updateTime: now, }; @@ -131,6 +137,7 @@ export class LocalFileDatasetStore implements DatasetStore { datasetId: datasetId, size: Array.isArray(req.data) ? req.data.length : req.data.samples.length, version: prevMetadata.version + 1, + targetAction: req.targetAction ?? prevMetadata.targetAction, createTime: prevMetadata.createTime, updateTime: now, }; diff --git a/genkit-tools/common/src/types/apis.ts b/genkit-tools/common/src/types/apis.ts index cfb151bf77..ff544053a0 100644 --- a/genkit-tools/common/src/types/apis.ts +++ b/genkit-tools/common/src/types/apis.ts @@ -135,6 +135,7 @@ export type GetEvalRunRequest = z.infer; export const CreateDatasetRequestSchema = z.object({ data: EvalFlowInputSchema, datasetId: z.string().optional(), + targetAction: z.string().optional(), }); export type CreateDatasetRequest = z.infer; @@ -142,6 +143,7 @@ export type CreateDatasetRequest = z.infer; export const UpdateDatasetRequestSchema = z.object({ /** Only supports replacement, throws if data is absent */ data: EvalFlowInputSchema.optional(), + targetAction: z.string().optional(), datasetId: z.string(), }); export type UpdateDatasetRequest = z.infer; diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 8304b75abb..26de7cffd6 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -156,6 +156,7 @@ export const DatasetMetadataSchema = z.object({ size: z.number(), /** 1 for v1, 2 for v2, etc */ version: z.number(), + targetAction: z.string().optional(), createTime: z.string(), updateTime: z.string(), }); From fe825a40db73795c7ebffd8152612e452f42cf3b Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Tue, 24 Sep 2024 22:26:55 +0000 Subject: [PATCH 2/9] update test --- genkit-tools/common/tests/eval/localFileDatasetStore_test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts index 953d43f3b9..99096ff6f4 100644 --- a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts +++ b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts @@ -60,7 +60,6 @@ const SAMPLE_DATASET_1_V2 = { ], }; -const SAMPLE_DATASET_NAME_1 = 'dataset-1'; const SAMPLE_DATASET_ID_1 = 'dataset-1-123456'; const SAMPLE_DATASET_METADATA_1_V1 = { @@ -74,6 +73,7 @@ const SAMPLE_DATASET_METADATA_1_V2 = { datasetId: SAMPLE_DATASET_ID_1, size: 3, version: 2, + targetAction: '/flow/myFlow', createTime: FAKE_TIME.toString(), updateTime: FAKE_TIME.toString(), }; @@ -85,10 +85,10 @@ const CREATE_DATASET_REQUEST = CreateDatasetRequestSchema.parse({ const UPDATE_DATASET_REQUEST = UpdateDatasetRequestSchema.parse({ data: SAMPLE_DATASET_1_V2, datasetId: SAMPLE_DATASET_ID_1, + targetAction: '/flow/myFlow', }); const SAMPLE_DATASET_ID_2 = 'dataset-2-123456'; -const SAMPLE_DATASET_NAME_2 = 'dataset-2'; const SAMPLE_DATASET_METADATA_2 = { datasetId: SAMPLE_DATASET_ID_2, From 880a5adee81b1220fc11cff4c46f86187296c152 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Thu, 26 Sep 2024 16:52:58 +0000 Subject: [PATCH 3/9] WIP, target action + testCaseId --- .../cli/src/commands/eval-extract-data.ts | 9 +- genkit-tools/cli/src/commands/eval-run.ts | 9 +- genkit-tools/common/src/eval/evaluate.ts | 58 +- .../common/src/eval/localFileDatasetStore.ts | 62 +- genkit-tools/common/src/types/apis.ts | 1 - genkit-tools/common/src/types/eval.ts | 14 +- genkit-tools/common/src/utils/eval.ts | 6 + js/ai/src/evaluator.ts | 23 +- js/plugins/evaluators/src/index.ts | 10 +- .../src/metrics/answer_relevancy.ts | 4 +- .../evaluators/src/metrics/faithfulness.ts | 4 +- .../evaluators/src/metrics/maliciousness.ts | 4 +- js/plugins/langchain/src/evaluators.ts | 5 +- js/plugins/vertexai/src/evaluator_factory.ts | 6 +- js/pnpm-lock.yaml | 750 ++++++++++++++++-- .../src/deliciousness/deliciousness.ts | 4 +- .../deliciousness/deliciousness_evaluator.ts | 4 +- .../byo-evaluator/src/funniness/funniness.ts | 4 +- .../src/funniness/funniness_evaluator.ts | 4 +- .../byo-evaluator/src/pii/pii_detection.ts | 4 +- .../byo-evaluator/src/pii/pii_evaluator.ts | 4 +- .../src/regex/regex_evaluator.ts | 8 +- 22 files changed, 871 insertions(+), 126 deletions(-) diff --git a/genkit-tools/cli/src/commands/eval-extract-data.ts b/genkit-tools/cli/src/commands/eval-extract-data.ts index 80ec9dd23f..eee23efbed 100644 --- a/genkit-tools/cli/src/commands/eval-extract-data.ts +++ b/genkit-tools/cli/src/commands/eval-extract-data.ts @@ -15,9 +15,12 @@ */ import { EnvTypes, EvalInput, TraceData } from '@genkit-ai/tools-common'; -import { getEvalExtractors, logger } from '@genkit-ai/tools-common/utils'; +import { + generateTestCaseId, + getEvalExtractors, + logger, +} from '@genkit-ai/tools-common/utils'; import { Command } from 'commander'; -import { randomUUID } from 'crypto'; import { writeFile } from 'fs/promises'; import { runInRunnerThenStop } from '../utils/runner-utils'; @@ -75,7 +78,7 @@ export const evalExtractData = new Command('eval:extractData') .filter((t): t is TraceData => !!t) .map((trace) => { return { - testCaseId: randomUUID(), + testCaseId: generateTestCaseId(), input: extractors.input(trace), output: extractors.output(trace), context: JSON.parse(extractors.context(trace)) as string[], diff --git a/genkit-tools/cli/src/commands/eval-run.ts b/genkit-tools/cli/src/commands/eval-run.ts index e8e877d660..4656447358 100644 --- a/genkit-tools/cli/src/commands/eval-run.ts +++ b/genkit-tools/cli/src/commands/eval-run.ts @@ -23,9 +23,12 @@ import { getMatchingEvaluatorActions, runEvaluation, } from '@genkit-ai/tools-common/eval'; -import { confirmLlmUse, logger } from '@genkit-ai/tools-common/utils'; +import { + confirmLlmUse, + generateTestCaseId, + logger, +} from '@genkit-ai/tools-common/utils'; import { Command } from 'commander'; -import { randomUUID } from 'crypto'; import { readFile } from 'fs/promises'; import { runInRunnerThenStop } from '../utils/runner-utils'; @@ -91,7 +94,7 @@ export const evalRun = new Command('eval:run') (await readFile(dataset)).toString('utf-8') ).map((testCase: any) => ({ ...testCase, - testCaseId: testCase.testCaseId || randomUUID(), + testCaseId: testCase.testCaseId || generateTestCaseId(), traceIds: testCase.traceIds || [], })); const evalRun = await runEvaluation({ diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index c0c7756108..b9642d4c2c 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -27,6 +27,7 @@ import { } from '../types'; import { evaluatorName, + generateTestCaseId, getEvalExtractors, isEvaluator, logger, @@ -36,6 +37,7 @@ import { enrichResultsWithScoring, extractMetricsMetadata } from './parser'; interface BulkRunResponse { traceId?: string; + testCaseId: string; response?: any; } @@ -50,6 +52,8 @@ export async function runNewEvaluation( const datasetStore = await getDatasetStore(); logger.info(`Fetching dataset ${datasetId}...`); const dataset = await datasetStore.getDataset(datasetId); + const datasetMetadatas = await datasetStore.listDatasets(); + const targetDataset = datasetMetadatas.find((d) => d.datasetId === datasetId); logger.info('Running inference...'); const evalDataset = await runInference({ @@ -58,15 +62,19 @@ export async function runNewEvaluation( evalFlowInput: dataset, auth: request.options?.auth, }); - const evaluatorAction = await getMatchingEvaluatorActions(runner, evaluators); + const evaluatorActions = await getMatchingEvaluatorActions( + runner, + evaluators + ); logger.info('Running evaluation...'); const evalRun = await runEvaluation({ runner, - evaluatorActions: evaluatorAction, + evaluatorActions, evalDataset, actionRef, datasetId, + datasetVersion: targetDataset!.version, }); logger.info('Finished evaluation, returning key...'); const evalStore = getEvalStore(); @@ -87,14 +95,11 @@ export async function runInference(params: { // TODO(ssbushi): Support model inference throw new Error('Inference is only supported on flows'); } - let inputs: any[] = Array.isArray(evalFlowInput) - ? (evalFlowInput as any[]) - : evalFlowInput.samples.map((c) => c.input); const runResponses: BulkRunResponse[] = await bulkRunAction({ runner, actionRef, - inputs, + evalFlowInput, auth, }); @@ -115,9 +120,16 @@ export async function runEvaluation(params: { evalDataset: EvalInput[]; actionRef?: string; datasetId?: string; + datasetVersion?: number; }): Promise { - const { runner, evaluatorActions, evalDataset, actionRef, datasetId } = - params; + const { + runner, + evaluatorActions, + evalDataset, + actionRef, + datasetVersion, + datasetId, + } = params; const evalRunId = randomUUID(); const scores: Record = {}; for (const action of evaluatorActions) { @@ -140,6 +152,7 @@ export async function runEvaluation(params: { actionRef, evalRunId, datasetId, + datasetVersion, createdAt: new Date().toISOString(), }, results: scoredResults, @@ -183,10 +196,22 @@ export async function getMatchingEvaluatorActions( async function bulkRunAction(params: { runner: Runner; actionRef: string; - inputs: any[]; + evalFlowInput: EvalFlowInput; auth?: string; }): Promise { - const { runner, actionRef, inputs, auth } = params; + const { runner, actionRef, evalFlowInput, auth } = params; + let inputs: { input?: any; testCaseId: string }[] = Array.isArray( + evalFlowInput + ) + ? (evalFlowInput as any[]).map((i) => ({ + input: i, + testCaseId: generateTestCaseId(), + })) + : evalFlowInput.samples.map((c) => ({ + ...c, + testCaseId: c.testCaseId ?? generateTestCaseId(), + })); + let responses: BulkRunResponse[] = []; for (const d of inputs) { logger.info(`Running '${actionRef}' ...`); @@ -196,18 +221,19 @@ async function bulkRunAction(params: { key: actionRef, input: { start: { - input: d, + input: d.input, }, auth: auth ? JSON.parse(auth) : undefined, }, }); response = { + testCaseId: d.testCaseId, traceId: runActionResponse.telemetry?.traceId, response: runActionResponse.result, }; } catch (e: any) { const traceId = e?.data?.details?.traceId; - response = { traceId }; + response = { testCaseId: d.testCaseId, traceId }; } responses.push(response); } @@ -243,7 +269,7 @@ async function fetchEvalInput(params: { logger.warn('No traceId available...'); return { // TODO Replace this with unified trace class - testCaseId: randomUUID(), + testCaseId: s.testCaseId, traceIds: [], }; } @@ -265,7 +291,7 @@ async function fetchEvalInput(params: { const nestedSpan = stackTraceSpans(trace); if (!nestedSpan) { return { - testCaseId: randomUUID(), + testCaseId: s.testCaseId, input: inputs[0], error: `Unable to extract any spans from trace ${traceId}`, reference: references?.at(i), @@ -275,7 +301,7 @@ async function fetchEvalInput(params: { if (nestedSpan.attributes['genkit:state'] === 'error') { return { - testCaseId: randomUUID(), + testCaseId: s.testCaseId, input: inputs[0], error: getSpanErrorMessage(nestedSpan) ?? @@ -290,7 +316,7 @@ async function fetchEvalInput(params: { return { // TODO Replace this with unified trace class - testCaseId: randomUUID(), + testCaseId: s.testCaseId, input: inputs[0], output: outputs[0], context: JSON.parse(contexts[0]) as string[], diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 13f2d29fb1..974823ad80 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import crypto from 'crypto'; +import crypto, { randomUUID } from 'crypto'; import fs from 'fs'; import { readFile, rm, writeFile } from 'fs/promises'; import path from 'path'; @@ -23,8 +23,9 @@ import { CreateDatasetRequest, UpdateDatasetRequest } from '../types/apis'; import { Dataset, DatasetMetadata, + DatasetSchema, DatasetStore, - EvalFlowInputSchema, + EvalFlowInput, } from '../types/eval'; import { logger } from '../utils/logger'; @@ -72,7 +73,7 @@ export class LocalFileDatasetStore implements DatasetStore { } private async createDatasetInternal( - data: Dataset, + data: EvalFlowInput, datasetId?: string, targetAction?: string ): Promise { @@ -85,14 +86,14 @@ export class LocalFileDatasetStore implements DatasetStore { `Create dataset failed: file already exists at {$filePath}` ); } - + const dataset = this.getDatasetFromEvalFlowInput(data); logger.info(`Saving Dataset to ` + filePath); - await writeFile(filePath, JSON.stringify(data)); + await writeFile(filePath, JSON.stringify(dataset)); const now = new Date().toString(); const metadata = { datasetId: id, - size: Array.isArray(data) ? data.length : data.samples.length, + size: dataset.length, version: 1, targetAction, createTime: now, @@ -112,9 +113,6 @@ export class LocalFileDatasetStore implements DatasetStore { async updateDataset(req: UpdateDatasetRequest): Promise { const datasetId = req.datasetId; - if (!req.data) { - throw new Error('Error: `data` is required for updateDataset'); - } const filePath = path.resolve( this.storeRoot, this.generateFileName(datasetId) @@ -128,14 +126,17 @@ export class LocalFileDatasetStore implements DatasetStore { if (!prevMetadata) { throw new Error(`Update dataset failed: dataset metadata not found`); } - - logger.info(`Updating Dataset at ` + filePath); - await writeFile(filePath, JSON.stringify(req.data)); + const patch = this.getDatasetFromEvalFlowInput(req.data ?? []); + let newSize = prevMetadata.size; + if (patch.length > 0) { + logger.info(`Updating Dataset at ` + filePath); + newSize = await this.patchDataset(datasetId, patch, filePath); + } const now = new Date().toString(); const newMetadata = { datasetId: datasetId, - size: Array.isArray(req.data) ? req.data.length : req.data.samples.length, + size: newSize, version: prevMetadata.version + 1, targetAction: req.targetAction ?? prevMetadata.targetAction, createTime: prevMetadata.createTime, @@ -162,7 +163,7 @@ export class LocalFileDatasetStore implements DatasetStore { throw new Error(`Dataset not found for dataset ID {$id}`); } return await readFile(filePath, 'utf8').then((data) => - EvalFlowInputSchema.parse(JSON.parse(data)) + DatasetSchema.parse(JSON.parse(data)) ); } @@ -243,4 +244,37 @@ export class LocalFileDatasetStore implements DatasetStore { JSON.parse(data) ); } + + private getDatasetFromEvalFlowInput(data: EvalFlowInput): Dataset { + if (Array.isArray(data)) { + return data.map((d) => ({ + testCaseId: d.testCaseId ?? randomUUID(), + input: d, + })); + } else if (!!data.samples) { + return data.samples.map((d) => ({ + testCaseId: d.testCaseId ?? randomUUID(), + ...d, + })); + } + return []; + } + + private async patchDataset( + datasetId: string, + patch: Dataset, + filePath: string + ): Promise { + const existingDataset = await this.getDataset(datasetId); + const datasetMap = new Map(existingDataset.map((d) => [d.testCaseId, d])); + const patchMap = new Map(patch.map((d) => [d.testCaseId, d])); + + patchMap.forEach((value, key) => { + datasetMap.set(key, value); + }); + + const newDataset = Object.values(patchMap) as Dataset; + await writeFile(filePath, JSON.stringify(newDataset)); + return newDataset.length; + } } diff --git a/genkit-tools/common/src/types/apis.ts b/genkit-tools/common/src/types/apis.ts index ff544053a0..254ec8bfb4 100644 --- a/genkit-tools/common/src/types/apis.ts +++ b/genkit-tools/common/src/types/apis.ts @@ -141,7 +141,6 @@ export const CreateDatasetRequestSchema = z.object({ export type CreateDatasetRequest = z.infer; export const UpdateDatasetRequestSchema = z.object({ - /** Only supports replacement, throws if data is absent */ data: EvalFlowInputSchema.optional(), targetAction: z.string().optional(), datasetId: z.string(), diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 26de7cffd6..880f573c19 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -32,6 +32,7 @@ import { export const EvalFlowStructuredInputSchema = z.object({ samples: z.array( z.object({ + testCaseId: z.string().optional(), input: z.any(), reference: z.any().optional(), }) @@ -54,10 +55,16 @@ export const EvalFlowInputSchema = z.union([ export type EvalFlowInput = z.infer; /** - * Alias for EvalFlowInput to be used in the DatasetStore related APIs. - * We may deprecate EvalFlowInput in favor of this in the future. + * Represents a Dataset, to be used for bulk-inference / evaluation */ -export type Dataset = z.infer; +export const DatasetSchema = z.array( + z.object({ + testCaseId: z.string(), + input: z.any(), + reference: z.any().optional(), + }) +); +export type Dataset = z.infer; /** * A record that is ready for evaluation. @@ -101,6 +108,7 @@ export type EvalResult = z.infer; export const EvalRunKeySchema = z.object({ actionRef: z.string().optional(), datasetId: z.string().optional(), + datasetVersion: z.number().optional(), evalRunId: z.string(), createdAt: z.string(), }); diff --git a/genkit-tools/common/src/utils/eval.ts b/genkit-tools/common/src/utils/eval.ts index 6439a7ceaa..96492ccacb 100644 --- a/genkit-tools/common/src/utils/eval.ts +++ b/genkit-tools/common/src/utils/eval.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { randomUUID } from 'crypto'; import * as inquirer from 'inquirer'; import { EvalField, @@ -204,3 +205,8 @@ export async function getEvalExtractors( } return Promise.resolve(composedExtractors); } + +/**Global function to generate testCaseId */ +export function generateTestCaseId() { + return randomUUID(); +} diff --git a/js/ai/src/evaluator.ts b/js/ai/src/evaluator.ts index 5f4f8ed3fa..52eb574d80 100644 --- a/js/ai/src/evaluator.ts +++ b/js/ai/src/evaluator.ts @@ -18,6 +18,7 @@ import { Action, defineAction, z } from '@genkit-ai/core'; import { logger } from '@genkit-ai/core/logging'; import { lookupAction } from '@genkit-ai/core/registry'; import { SPAN_TYPE_ATTR, runInNewSpan } from '@genkit-ai/core/tracing'; +import { randomUUID } from 'crypto'; export const ATTR_PREFIX = 'genkit'; export const SPAN_STATE_ATTR = ATTR_PREFIX + ':state'; @@ -31,6 +32,12 @@ export const BaseDataPointSchema = z.object({ traceIds: z.array(z.string()).optional(), }); +// DataPoint that is to be used for actions. This needs testCaseId to be present. +export const BaseEvalDataPointSchema = BaseDataPointSchema.extend({ + testCaseId: z.string(), +}); +export type BaseEvalDataPoint = z.infer; + export const ScoreSchema = z.object({ score: z.union([z.number(), z.string(), z.boolean()]).optional(), // TODO: use StatusSchema @@ -56,7 +63,7 @@ export type Dataset< export const EvalResponseSchema = z.object({ sampleIndex: z.number().optional(), - testCaseId: z.string().optional(), + testCaseId: z.string(), traceId: z.string().optional(), spanId: z.string().optional(), evaluation: ScoreSchema, @@ -67,10 +74,11 @@ export const EvalResponsesSchema = z.array(EvalResponseSchema); export type EvalResponses = z.infer; type EvaluatorFn< - DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema, + EvalDataPoint extends + typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema, CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, > = ( - input: z.infer, + input: z.infer, evaluatorOptions?: z.infer ) => Promise; @@ -106,6 +114,8 @@ const EvalRequestSchema = z.object({ */ export function defineEvaluator< DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema, + EvalDataPoint extends + typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema, EvaluatorOptions extends z.ZodTypeAny = z.ZodTypeAny, >( options: { @@ -116,7 +126,7 @@ export function defineEvaluator< configSchema?: EvaluatorOptions; isBilled?: boolean; }, - runner: EvaluatorFn + runner: EvaluatorFn ) { const metadata = {}; metadata[EVALUATOR_METADATA_KEY_IS_BILLED] = @@ -140,7 +150,10 @@ export function defineEvaluator< async (i) => { let evalResponses: EvalResponses = []; for (let index = 0; index < i.dataset.length; index++) { - const datapoint = i.dataset[index]; + const datapoint: BaseEvalDataPoint = { + ...i.dataset[index], + testCaseId: i.dataset[index].testCaseId ?? randomUUID(), + }; try { await runInNewSpan( { diff --git a/js/plugins/evaluators/src/index.ts b/js/plugins/evaluators/src/index.ts index cca9e728ba..072d0e459d 100644 --- a/js/plugins/evaluators/src/index.ts +++ b/js/plugins/evaluators/src/index.ts @@ -22,7 +22,7 @@ import { z, } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, defineEvaluator, EvalResponse, evaluatorRef, @@ -88,7 +88,7 @@ function hasMetric(arr: GenkitMetric[] | undefined, metric: GenkitMetric) { return arr?.some((m) => m === metric); } -function fillScores(dataPoint: BaseDataPoint, score: Score): EvalResponse { +function fillScores(dataPoint: BaseEvalDataPoint, score: Score): EvalResponse { return { testCaseId: dataPoint.testCaseId, evaluation: score, @@ -118,7 +118,7 @@ export function genkitEvaluators< definition: 'Assesses how pertinent the generated answer is to the given prompt', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const answerRelevancy = await answerRelevancyScore( judge, datapoint, @@ -138,7 +138,7 @@ export function genkitEvaluators< definition: 'Measures the factual consistency of the generated answer against the given context', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const faithfulness = await faithfulnessScore( judge, datapoint, @@ -156,7 +156,7 @@ export function genkitEvaluators< definition: 'Measures whether the generated output intends to deceive, harm, or exploit', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const maliciousness = await maliciousnessScore( judge, datapoint, diff --git a/js/plugins/evaluators/src/metrics/answer_relevancy.ts b/js/plugins/evaluators/src/metrics/answer_relevancy.ts index 70e1797c14..ab5e02b79c 100644 --- a/js/plugins/evaluators/src/metrics/answer_relevancy.ts +++ b/js/plugins/evaluators/src/metrics/answer_relevancy.ts @@ -17,7 +17,7 @@ import similarity from 'compute-cosine-similarity'; import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; import { embed, EmbedderArgument } from 'genkit/embedder'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -32,7 +32,7 @@ export async function answerRelevancyScore< CustomEmbedderOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, embedder: EmbedderArgument, judgeConfig?: CustomModelOptions, embedderOptions?: z.infer diff --git a/js/plugins/evaluators/src/metrics/faithfulness.ts b/js/plugins/evaluators/src/metrics/faithfulness.ts index b8816164b4..fa80c6f4b7 100644 --- a/js/plugins/evaluators/src/metrics/faithfulness.ts +++ b/js/plugins/evaluators/src/metrics/faithfulness.ts @@ -15,7 +15,7 @@ */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -40,7 +40,7 @@ export async function faithfulnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { try { diff --git a/js/plugins/evaluators/src/metrics/maliciousness.ts b/js/plugins/evaluators/src/metrics/maliciousness.ts index 316e1eeda5..ad166d2b2c 100644 --- a/js/plugins/evaluators/src/metrics/maliciousness.ts +++ b/js/plugins/evaluators/src/metrics/maliciousness.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -27,7 +27,7 @@ export async function maliciousnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/plugins/langchain/src/evaluators.ts b/js/plugins/langchain/src/evaluators.ts index dab2679136..b112149d20 100644 --- a/js/plugins/langchain/src/evaluators.ts +++ b/js/plugins/langchain/src/evaluators.ts @@ -15,7 +15,7 @@ */ import { ModelArgument } from 'genkit'; -import { BaseDataPointSchema, defineEvaluator } from 'genkit/evaluator'; +import { BaseEvalDataPoint, defineEvaluator } from 'genkit/evaluator'; import { Criteria, loadEvaluator } from 'langchain/evaluation'; import { genkitModel } from './model.js'; import { GenkitTracer } from './tracing.js'; @@ -31,9 +31,8 @@ export function langchainEvaluator( name: `langchain/${criteria}`, displayName: `${criteria}`, definition: `${criteria}: refer to https://js.langchain.com/docs/guides/evaluation`, - dataPointType: BaseDataPointSchema, }, - async (datapoint) => { + async (datapoint: BaseEvalDataPoint) => { try { switch (type) { case 'labeled_criteria': diff --git a/js/plugins/vertexai/src/evaluator_factory.ts b/js/plugins/vertexai/src/evaluator_factory.ts index 3cc0405a04..7fbb4f2991 100644 --- a/js/plugins/vertexai/src/evaluator_factory.ts +++ b/js/plugins/vertexai/src/evaluator_factory.ts @@ -15,7 +15,7 @@ */ import { Action, GENKIT_CLIENT_HEADER, z } from 'genkit'; -import { BaseDataPoint, Score, defineEvaluator } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score, defineEvaluator } from 'genkit/evaluator'; import { runInNewSpan } from 'genkit/tracing'; import { GoogleAuth } from 'google-auth-library'; import { VertexAIEvaluationMetricType } from './evaluation.js'; @@ -34,7 +34,7 @@ export class EvaluatorFactory { definition: string; responseSchema: ResponseType; }, - toRequest: (datapoint: BaseDataPoint) => any, + toRequest: (datapoint: BaseEvalDataPoint) => any, responseHandler: (response: z.infer) => Score ): Action { return defineEvaluator( @@ -43,7 +43,7 @@ export class EvaluatorFactory { displayName: config.displayName, definition: config.definition, }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const responseSchema = config.responseSchema; const response = await this.evaluateInstances( toRequest(datapoint), diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index dbda0165f0..38aefc9392 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -28,7 +28,7 @@ importers: version: 1.9.0 '@types/node': specifier: ^20.11.19 - version: 20.11.30 + version: 20.16.9 colorette: specifier: ^2.0.20 version: 2.0.20 @@ -47,10 +47,10 @@ importers: version: 4.1.5 tsup: specifier: ^8.0.2 - version: 8.0.2(postcss@8.4.47)(typescript@4.9.5) + version: 8.3.0(postcss@8.4.47)(tsx@4.19.1)(typescript@4.9.5) tsx: specifier: ^4.7.1 - version: 4.7.1 + version: 4.19.1 typescript: specifier: ^4.9.0 version: 4.9.5 @@ -339,10 +339,10 @@ importers: version: 0.19.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) '@google-cloud/opentelemetry-cloud-trace-exporter': specifier: ^2.1.0 - version: 2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.26.0)(encoding@0.1.13) + version: 2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) '@google-cloud/opentelemetry-resource-util': specifier: ^2.1.0 - version: 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.26.0)(encoding@0.1.13) + version: 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 @@ -1257,138 +1257,282 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.23.1': + resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.19.12': resolution: {integrity: sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==} engines: {node: '>=12'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.23.1': + resolution: {integrity: sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.19.12': resolution: {integrity: sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==} engines: {node: '>=12'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.23.1': + resolution: {integrity: sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.19.12': resolution: {integrity: sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==} engines: {node: '>=12'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.23.1': + resolution: {integrity: sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.19.12': resolution: {integrity: sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==} engines: {node: '>=12'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.23.1': + resolution: {integrity: sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.19.12': resolution: {integrity: sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==} engines: {node: '>=12'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.23.1': + resolution: {integrity: sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.19.12': resolution: {integrity: sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==} engines: {node: '>=12'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.23.1': + resolution: {integrity: sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.19.12': resolution: {integrity: sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==} engines: {node: '>=12'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.23.1': + resolution: {integrity: sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.19.12': resolution: {integrity: sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==} engines: {node: '>=12'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.23.1': + resolution: {integrity: sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.19.12': resolution: {integrity: sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==} engines: {node: '>=12'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.23.1': + resolution: {integrity: sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.19.12': resolution: {integrity: sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==} engines: {node: '>=12'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.23.1': + resolution: {integrity: sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.19.12': resolution: {integrity: sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==} engines: {node: '>=12'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.23.1': + resolution: {integrity: sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.19.12': resolution: {integrity: sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==} engines: {node: '>=12'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.23.1': + resolution: {integrity: sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.19.12': resolution: {integrity: sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==} engines: {node: '>=12'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.23.1': + resolution: {integrity: sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.19.12': resolution: {integrity: sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==} engines: {node: '>=12'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.23.1': + resolution: {integrity: sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.19.12': resolution: {integrity: sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==} engines: {node: '>=12'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.23.1': + resolution: {integrity: sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.19.12': resolution: {integrity: sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==} engines: {node: '>=12'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.23.1': + resolution: {integrity: sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-x64@0.19.12': resolution: {integrity: sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==} engines: {node: '>=12'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.23.1': + resolution: {integrity: sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.23.1': + resolution: {integrity: sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.19.12': resolution: {integrity: sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==} engines: {node: '>=12'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.23.1': + resolution: {integrity: sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/sunos-x64@0.19.12': resolution: {integrity: sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==} engines: {node: '>=12'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.23.1': + resolution: {integrity: sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.19.12': resolution: {integrity: sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==} engines: {node: '>=12'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.23.1': + resolution: {integrity: sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.19.12': resolution: {integrity: sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==} engines: {node: '>=12'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.23.1': + resolution: {integrity: sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.19.12': resolution: {integrity: sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==} engines: {node: '>=12'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.23.1': + resolution: {integrity: sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} engines: {node: '>=14'} @@ -1903,12 +2047,24 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/context-async-hooks@1.26.0': + resolution: {integrity: sha512-HedpXXYzzbaoutw6DFLWLDket2FwLkLpil4hGCZ1xYEIMTcivdfwEOISgdbLEWyG3HW52gTq2V9mOVJrONgiwg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@1.25.1': resolution: {integrity: sha512-GeT/l6rBYWVQ4XArluLVB6WWQ8flHbdb6r2FCHC3smtdOAbrJBIv35tpV/yp9bmYUJf+xmZpu9DRTIeJVhFbEQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@1.26.0': + resolution: {integrity: sha512-1iKxXXE8415Cdv0yjG3G6hQnB5eVEsJce3QaawX8SjDn0mAS0ZM8fAbZZJD4ajvhC15cePvosSCut404KrIIvQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/exporter-trace-otlp-grpc@0.52.1': resolution: {integrity: sha512-pVkSH20crBwMTqB3nIN4jpQKUEoB0Z94drIHpYyEqs7UBr+I0cpYyOR3bqjA/UasQUMROb3GX8ZX4/9cVRqGBQ==} engines: {node: '>=14'} @@ -2255,6 +2411,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/resources@1.26.0': + resolution: {integrity: sha512-CPNYchBE7MBecCSVy0HKpUISEeJOniWqcHaAHpmasZ3j9o6V3AyBzhRc90jdmemq0HOxDr6ylhUbDhBqqPpeNw==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-logs@0.52.1': resolution: {integrity: sha512-MBYh+WcPPsN8YpRHRmK1Hsca9pVlyyKd4BxOC4SsgHACnl/bPp4Cri9hWhVm5+2tiQ9Zf4qSc1Jshw9tOLGWQA==} engines: {node: '>=14'} @@ -2267,6 +2429,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-metrics@1.26.0': + resolution: {integrity: sha512-0SvDXmou/JjzSDOjUmetAAvcKQW6ZrvosU0rkbDGpXvvZN+pQF6JbK/Kd4hNdK4q/22yeruqvukXEJyySTzyTQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-node@0.52.1': resolution: {integrity: sha512-uEG+gtEr6eKd8CVWeKMhH2olcCHM9dEK68pe0qE0be32BcCRsvYURhHaD1Srngh1SQcnQzZ4TP324euxqtBOJA==} engines: {node: '>=14'} @@ -2279,6 +2447,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-trace-base@1.26.0': + resolution: {integrity: sha512-olWQldtvbK4v22ymrKLbIcBi9L2SpMO84sCPY54IVsJhP9fRsxJT194C/AVaAuJzLE30EdhhM1VmvVYR7az+cw==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-trace-node@1.25.1': resolution: {integrity: sha512-nMcjFIKxnFqoez4gUmihdBrbpsEnAX/Xj16sGvZm+guceYE0NE00vLhpDVK6f3q8Q4VFI5xG8JjlXKMB/SkTTQ==} engines: {node: '>=14'} @@ -2297,6 +2471,10 @@ packages: resolution: {integrity: sha512-U9PJlOswJPSgQVPI+XEuNLElyFWkb0hAiMg+DExD9V0St03X2lPHGMdxMY/LrVmoukuIpXJ12oyrOtEZ4uXFkw==} engines: {node: '>=14'} + '@opentelemetry/semantic-conventions@1.27.0': + resolution: {integrity: sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==} + engines: {node: '>=14'} + '@opentelemetry/sql-common@0.40.1': resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==} engines: {node: '>=14'} @@ -2346,76 +2524,156 @@ packages: cpu: [arm] os: [android] + '@rollup/rollup-android-arm-eabi@4.22.4': + resolution: {integrity: sha512-Fxamp4aEZnfPOcGA8KSNEohV8hX7zVHOemC8jVBoBUHu5zpJK/Eu3uJwt6BMgy9fkvzxDaurgj96F/NiLukF2w==} + cpu: [arm] + os: [android] + '@rollup/rollup-android-arm64@4.13.2': resolution: {integrity: sha512-GdxxXbAuM7Y/YQM9/TwwP+L0omeE/lJAR1J+olu36c3LqqZEBdsIWeQ91KBe6nxwOnb06Xh7JS2U5ooWU5/LgQ==} cpu: [arm64] os: [android] + '@rollup/rollup-android-arm64@4.22.4': + resolution: {integrity: sha512-VXoK5UMrgECLYaMuGuVTOx5kcuap1Jm8g/M83RnCHBKOqvPPmROFJGQaZhGccnsFtfXQ3XYa4/jMCJvZnbJBdA==} + cpu: [arm64] + os: [android] + '@rollup/rollup-darwin-arm64@4.13.2': resolution: {integrity: sha512-mCMlpzlBgOTdaFs83I4XRr8wNPveJiJX1RLfv4hggyIVhfB5mJfN4P8Z6yKh+oE4Luz+qq1P3kVdWrCKcMYrrA==} cpu: [arm64] os: [darwin] + '@rollup/rollup-darwin-arm64@4.22.4': + resolution: {integrity: sha512-xMM9ORBqu81jyMKCDP+SZDhnX2QEVQzTcC6G18KlTQEzWK8r/oNZtKuZaCcHhnsa6fEeOBionoyl5JsAbE/36Q==} + cpu: [arm64] + os: [darwin] + '@rollup/rollup-darwin-x64@4.13.2': resolution: {integrity: sha512-yUoEvnH0FBef/NbB1u6d3HNGyruAKnN74LrPAfDQL3O32e3k3OSfLrPgSJmgb3PJrBZWfPyt6m4ZhAFa2nZp2A==} cpu: [x64] os: [darwin] + '@rollup/rollup-darwin-x64@4.22.4': + resolution: {integrity: sha512-aJJyYKQwbHuhTUrjWjxEvGnNNBCnmpHDvrb8JFDbeSH3m2XdHcxDd3jthAzvmoI8w/kSjd2y0udT+4okADsZIw==} + cpu: [x64] + os: [darwin] + '@rollup/rollup-linux-arm-gnueabihf@4.13.2': resolution: {integrity: sha512-GYbLs5ErswU/Xs7aGXqzc3RrdEjKdmoCrgzhJWyFL0r5fL3qd1NPcDKDowDnmcoSiGJeU68/Vy+OMUluRxPiLQ==} cpu: [arm] os: [linux] + '@rollup/rollup-linux-arm-gnueabihf@4.22.4': + resolution: {integrity: sha512-j63YtCIRAzbO+gC2L9dWXRh5BFetsv0j0va0Wi9epXDgU/XUi5dJKo4USTttVyK7fGw2nPWK0PbAvyliz50SCQ==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.22.4': + resolution: {integrity: sha512-dJnWUgwWBX1YBRsuKKMOlXCzh2Wu1mlHzv20TpqEsfdZLb3WoJW2kIEsGwLkroYf24IrPAvOT/ZQ2OYMV6vlrg==} + cpu: [arm] + os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.13.2': resolution: {integrity: sha512-L1+D8/wqGnKQIlh4Zre9i4R4b4noxzH5DDciyahX4oOz62CphY7WDWqJoQ66zNR4oScLNOqQJfNSIAe/6TPUmQ==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.22.4': + resolution: {integrity: sha512-AdPRoNi3NKVLolCN/Sp4F4N1d98c4SBnHMKoLuiG6RXgoZ4sllseuGioszumnPGmPM2O7qaAX/IJdeDU8f26Aw==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-arm64-musl@4.13.2': resolution: {integrity: sha512-tK5eoKFkXdz6vjfkSTCupUzCo40xueTOiOO6PeEIadlNBkadH1wNOH8ILCPIl8by/Gmb5AGAeQOFeLev7iZDOA==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-musl@4.22.4': + resolution: {integrity: sha512-Gl0AxBtDg8uoAn5CCqQDMqAx22Wx22pjDOjBdmG0VIWX3qUBHzYmOKh8KXHL4UpogfJ14G4wk16EQogF+v8hmA==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-powerpc64le-gnu@4.13.2': resolution: {integrity: sha512-zvXvAUGGEYi6tYhcDmb9wlOckVbuD+7z3mzInCSTACJ4DQrdSLPNUeDIcAQW39M3q6PDquqLWu7pnO39uSMRzQ==} cpu: [ppc64le] os: [linux] + '@rollup/rollup-linux-powerpc64le-gnu@4.22.4': + resolution: {integrity: sha512-3aVCK9xfWW1oGQpTsYJJPF6bfpWfhbRnhdlyhak2ZiyFLDaayz0EP5j9V1RVLAAxlmWKTDfS9wyRyY3hvhPoOg==} + cpu: [ppc64] + os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.13.2': resolution: {integrity: sha512-C3GSKvMtdudHCN5HdmAMSRYR2kkhgdOfye4w0xzyii7lebVr4riCgmM6lRiSCnJn2w1Xz7ZZzHKuLrjx5620kw==} cpu: [riscv64] os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.22.4': + resolution: {integrity: sha512-ePYIir6VYnhgv2C5Xe9u+ico4t8sZWXschR6fMgoPUK31yQu7hTEJb7bCqivHECwIClJfKgE7zYsh1qTP3WHUA==} + cpu: [riscv64] + os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.13.2': resolution: {integrity: sha512-l4U0KDFwzD36j7HdfJ5/TveEQ1fUTjFFQP5qIt9gBqBgu1G8/kCaq5Ok05kd5TG9F8Lltf3MoYsUMw3rNlJ0Yg==} cpu: [s390x] os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.22.4': + resolution: {integrity: sha512-GqFJ9wLlbB9daxhVlrTe61vJtEY99/xB3C8e4ULVsVfflcpmR6c8UZXjtkMA6FhNONhj2eA5Tk9uAVw5orEs4Q==} + cpu: [s390x] + os: [linux] + '@rollup/rollup-linux-x64-gnu@4.13.2': resolution: {integrity: sha512-xXMLUAMzrtsvh3cZ448vbXqlUa7ZL8z0MwHp63K2IIID2+DeP5iWIT6g1SN7hg1VxPzqx0xZdiDM9l4n9LRU1A==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-gnu@4.22.4': + resolution: {integrity: sha512-87v0ol2sH9GE3cLQLNEy0K/R0pz1nvg76o8M5nhMR0+Q+BBGLnb35P0fVz4CQxHYXaAOhE8HhlkaZfsdUOlHwg==} + cpu: [x64] + os: [linux] + '@rollup/rollup-linux-x64-musl@4.13.2': resolution: {integrity: sha512-M/JYAWickafUijWPai4ehrjzVPKRCyDb1SLuO+ZyPfoXgeCEAlgPkNXewFZx0zcnoIe3ay4UjXIMdXQXOZXWqA==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-musl@4.22.4': + resolution: {integrity: sha512-UV6FZMUgePDZrFjrNGIWzDo/vABebuXBhJEqrHxrGiU6HikPy0Z3LfdtciIttEUQfuDdCn8fqh7wiFJjCNwO+g==} + cpu: [x64] + os: [linux] + '@rollup/rollup-win32-arm64-msvc@4.13.2': resolution: {integrity: sha512-2YWwoVg9KRkIKaXSh0mz3NmfurpmYoBBTAXA9qt7VXk0Xy12PoOP40EFuau+ajgALbbhi4uTj3tSG3tVseCjuA==} cpu: [arm64] os: [win32] + '@rollup/rollup-win32-arm64-msvc@4.22.4': + resolution: {integrity: sha512-BjI+NVVEGAXjGWYHz/vv0pBqfGoUH0IGZ0cICTn7kB9PyjrATSkX+8WkguNjWoj2qSr1im/+tTGRaY+4/PdcQw==} + cpu: [arm64] + os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.13.2': resolution: {integrity: sha512-2FSsE9aQ6OWD20E498NYKEQLneShWes0NGMPQwxWOdws35qQXH+FplabOSP5zEe1pVjurSDOGEVCE2agFwSEsw==} cpu: [ia32] os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.22.4': + resolution: {integrity: sha512-SiWG/1TuUdPvYmzmYnmd3IEifzR61Tragkbx9D3+R8mzQqDBz8v+BvZNDlkiTtI9T15KYZhP0ehn3Dld4n9J5g==} + cpu: [ia32] + os: [win32] + '@rollup/rollup-win32-x64-msvc@4.13.2': resolution: {integrity: sha512-7h7J2nokcdPePdKykd8wtc8QqqkqxIrUz7MHj6aNr8waBRU//NLDVnNjQnqQO6fqtjrtCdftpbTuOKAyrAQETQ==} cpu: [x64] os: [win32] + '@rollup/rollup-win32-x64-msvc@4.22.4': + resolution: {integrity: sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q==} + cpu: [x64] + os: [win32] + '@sinclair/typebox@0.29.6': resolution: {integrity: sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==} @@ -2480,14 +2738,17 @@ packages: '@types/node-fetch@2.6.11': resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} - '@types/node@18.19.31': - resolution: {integrity: sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==} + '@types/node@18.19.53': + resolution: {integrity: sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==} '@types/node@20.11.30': resolution: {integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw==} - '@types/node@22.4.1': - resolution: {integrity: sha512-1tbpb9325+gPnKK0dMm+/LMriX0vKxf6RnB0SZUqfyVkQ4fMgUSySqhxE/y8Jvs4NyF1yHzTfG9KlnkIODxPKg==} + '@types/node@20.16.9': + resolution: {integrity: sha512-rkvIVJxsOfBejxK7I0FO5sa2WxFmJCzoDwcd88+fq/CUfynNywTo/1/T6hyFz22CyztsnLS9nVlHOnTI36RH5w==} + + '@types/node@22.7.2': + resolution: {integrity: sha512-866lXSrpGpgyHBZUa2m9YNWqHDjjM0aBTJlNtYaGEw4rqY/dcD7deRVTbBBAJelfA7oaGDbNftXF/TL/A6RgoA==} '@types/pdf-parse@1.1.4': resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==} @@ -2575,6 +2836,9 @@ packages: ajv@8.12.0: resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -2706,6 +2970,12 @@ packages: peerDependencies: esbuild: '>=0.17' + bundle-require@5.0.0: + resolution: {integrity: sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + peerDependencies: + esbuild: '>=0.18' + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -2827,6 +3097,10 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + consola@3.2.3: + resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==} + engines: {node: ^14.18.0 || >=16.10.0} + console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} @@ -2904,6 +3178,15 @@ packages: supports-color: optional: true + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decamelize@1.2.0: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} @@ -3037,6 +3320,11 @@ packages: engines: {node: '>=12'} hasBin: true + esbuild@0.23.1: + resolution: {integrity: sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -3104,6 +3392,9 @@ packages: fast-text-encoding@1.0.6: resolution: {integrity: sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==} + fast-uri@3.0.1: + resolution: {integrity: sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==} + fast-xml-parser@4.3.6: resolution: {integrity: sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==} hasBin: true @@ -3115,6 +3406,14 @@ packages: resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} engines: {node: '>=0.8.0'} + fdir@6.3.0: + resolution: {integrity: sha512-QOnuT+BOtivR77wYvCWHfGt9s4Pz1VIMbD463vegT5MLqNXy8rYFT/lPVEqf/bhYeT6qmqrNHhsX+rWwe3rOCQ==} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} @@ -3290,6 +3589,9 @@ packages: get-tsconfig@4.7.3: resolution: {integrity: sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==} + get-tsconfig@4.8.1: + resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -3842,6 +4144,10 @@ packages: resolution: {integrity: sha512-O18pf7nyvHTckunPWCV1XUNXU1piu01y2b7ATJ0ppkUkk8ocqVWBrYjJBCwHDjD/ZWcfyrA0P4gKhzWGi5EINQ==} engines: {node: '>=14'} + lilconfig@3.1.2: + resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==} + engines: {node: '>=14'} + limiter@1.1.5: resolution: {integrity: sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==} @@ -4284,6 +4590,10 @@ packages: resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} engines: {node: '>=8.6'} + picomatch@4.0.2: + resolution: {integrity: sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==} + engines: {node: '>=12'} + pidtree@0.3.1: resolution: {integrity: sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==} engines: {node: '>=0.10'} @@ -4313,6 +4623,24 @@ packages: ts-node: optional: true + postcss-load-config@6.0.1: + resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} + engines: {node: '>= 18'} + peerDependencies: + jiti: '>=1.21.0' + postcss: '>=8.0.9' + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + jiti: + optional: true + postcss: + optional: true + tsx: + optional: true + yaml: + optional: true + postcss@8.4.47: resolution: {integrity: sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==} engines: {node: ^10 || ^12 || >=14} @@ -4480,6 +4808,11 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rollup@4.22.4: + resolution: {integrity: sha512-vD8HJ5raRcWOyymsR6Z3o6+RzfEPCnVLMFJ6vRslO1jt4LO6dUo5Qnpg7y4RkZFM2DMe3WUirkI5c16onjrc6A==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -4719,6 +5052,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + tinyglobby@0.2.6: + resolution: {integrity: sha512-NbBoFBpqfcgd1tCiO8Lkfdk+xrA7mlLR9zgvZcZWQQwU63XAfUePyd6wZBaU93Hqw347lHnwFzttAkemHzzz4g==} + engines: {node: '>=12.0.0'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -4770,6 +5107,30 @@ packages: typescript: optional: true + tsup@8.3.0: + resolution: {integrity: sha512-ALscEeyS03IomcuNdFdc0YWGVIkwH1Ws7nfTbAPuoILvEV2hpGQAY72LIOjglGo4ShWpZfpBqP/jpQVCzqYQag==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + + tsx@4.19.1: + resolution: {integrity: sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==} + engines: {node: '>=18.0.0'} + hasBin: true + tsx@4.7.1: resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==} engines: {node: '>=18.0.0'} @@ -4988,6 +5349,11 @@ packages: peerDependencies: zod: ^3.22.4 + zod-to-json-schema@3.23.3: + resolution: {integrity: sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==} + peerDependencies: + zod: ^3.23.3 + zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} @@ -4998,7 +5364,7 @@ snapshots: '@anthropic-ai/sdk@0.24.3(encoding@0.1.13)': dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -5011,7 +5377,7 @@ snapshots: '@anthropic-ai/sdk@0.9.1(encoding@0.1.13)': dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -5042,72 +5408,144 @@ snapshots: '@esbuild/aix-ppc64@0.19.12': optional: true + '@esbuild/aix-ppc64@0.23.1': + optional: true + '@esbuild/android-arm64@0.19.12': optional: true + '@esbuild/android-arm64@0.23.1': + optional: true + '@esbuild/android-arm@0.19.12': optional: true + '@esbuild/android-arm@0.23.1': + optional: true + '@esbuild/android-x64@0.19.12': optional: true + '@esbuild/android-x64@0.23.1': + optional: true + '@esbuild/darwin-arm64@0.19.12': optional: true + '@esbuild/darwin-arm64@0.23.1': + optional: true + '@esbuild/darwin-x64@0.19.12': optional: true + '@esbuild/darwin-x64@0.23.1': + optional: true + '@esbuild/freebsd-arm64@0.19.12': optional: true + '@esbuild/freebsd-arm64@0.23.1': + optional: true + '@esbuild/freebsd-x64@0.19.12': optional: true + '@esbuild/freebsd-x64@0.23.1': + optional: true + '@esbuild/linux-arm64@0.19.12': optional: true + '@esbuild/linux-arm64@0.23.1': + optional: true + '@esbuild/linux-arm@0.19.12': optional: true + '@esbuild/linux-arm@0.23.1': + optional: true + '@esbuild/linux-ia32@0.19.12': optional: true + '@esbuild/linux-ia32@0.23.1': + optional: true + '@esbuild/linux-loong64@0.19.12': optional: true + '@esbuild/linux-loong64@0.23.1': + optional: true + '@esbuild/linux-mips64el@0.19.12': optional: true + '@esbuild/linux-mips64el@0.23.1': + optional: true + '@esbuild/linux-ppc64@0.19.12': optional: true + '@esbuild/linux-ppc64@0.23.1': + optional: true + '@esbuild/linux-riscv64@0.19.12': optional: true + '@esbuild/linux-riscv64@0.23.1': + optional: true + '@esbuild/linux-s390x@0.19.12': optional: true + '@esbuild/linux-s390x@0.23.1': + optional: true + '@esbuild/linux-x64@0.19.12': optional: true + '@esbuild/linux-x64@0.23.1': + optional: true + '@esbuild/netbsd-x64@0.19.12': optional: true + '@esbuild/netbsd-x64@0.23.1': + optional: true + + '@esbuild/openbsd-arm64@0.23.1': + optional: true + '@esbuild/openbsd-x64@0.19.12': optional: true + '@esbuild/openbsd-x64@0.23.1': + optional: true + '@esbuild/sunos-x64@0.19.12': optional: true + '@esbuild/sunos-x64@0.23.1': + optional: true + '@esbuild/win32-arm64@0.19.12': optional: true + '@esbuild/win32-arm64@0.23.1': + optional: true + '@esbuild/win32-ia32@0.19.12': optional: true + '@esbuild/win32-ia32@0.23.1': + optional: true + '@esbuild/win32-x64@0.19.12': optional: true + '@esbuild/win32-x64@0.23.1': + optional: true + '@fastify/busboy@2.1.1': {} '@fastify/busboy@3.0.0': {} @@ -5159,7 +5597,7 @@ snapshots: dependencies: '@genkit-ai/core': 0.6.0-dev.1 '@opentelemetry/api': 1.9.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 colorette: 2.0.20 json5: 2.2.3 node-fetch: 3.3.2 @@ -5171,18 +5609,18 @@ snapshots: '@genkit-ai/core@0.6.0-dev.1': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) + '@opentelemetry/context-async-hooks': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.26.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': 0.52.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - ajv: 8.12.0 - ajv-formats: 3.0.1(ajv@8.12.0) + '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) async-mutex: 0.5.0 express: 4.19.2 json-schema: 0.4.0 zod: 3.23.8 - zod-to-json-schema: 3.22.5(zod@3.23.8) + zod-to-json-schema: 3.23.3(zod@3.23.8) transitivePeerDependencies: - supports-color @@ -5303,9 +5741,9 @@ snapshots: - encoding - supports-color - '@google-cloud/opentelemetry-cloud-trace-exporter@2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.26.0)(encoding@0.1.13)': + '@google-cloud/opentelemetry-cloud-trace-exporter@2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13)': dependencies: - '@google-cloud/opentelemetry-resource-util': 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.26.0)(encoding@0.1.13) + '@google-cloud/opentelemetry-resource-util': 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) '@grpc/grpc-js': 1.10.4 '@grpc/proto-loader': 0.7.12 '@opentelemetry/api': 1.9.0 @@ -5319,10 +5757,10 @@ snapshots: - encoding - supports-color - '@google-cloud/opentelemetry-resource-util@2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.26.0)(encoding@0.1.13)': + '@google-cloud/opentelemetry-resource-util@2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13)': dependencies: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.26.0 + '@opentelemetry/semantic-conventions': 1.27.0 gcp-metadata: 5.3.0(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -5579,11 +6017,20 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 + '@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/exporter-trace-otlp-grpc@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.10.10 @@ -6057,6 +6504,12 @@ snapshots: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/resources@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/sdk-logs@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6071,6 +6524,12 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) lodash.merge: 4.6.2 + '@opentelemetry/sdk-metrics@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-node@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6097,6 +6556,13 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/sdk-trace-node@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6113,6 +6579,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.26.0': {} + '@opentelemetry/semantic-conventions@1.27.0': {} + '@opentelemetry/sql-common@0.40.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6154,48 +6622,96 @@ snapshots: '@rollup/rollup-android-arm-eabi@4.13.2': optional: true + '@rollup/rollup-android-arm-eabi@4.22.4': + optional: true + '@rollup/rollup-android-arm64@4.13.2': optional: true + '@rollup/rollup-android-arm64@4.22.4': + optional: true + '@rollup/rollup-darwin-arm64@4.13.2': optional: true + '@rollup/rollup-darwin-arm64@4.22.4': + optional: true + '@rollup/rollup-darwin-x64@4.13.2': optional: true + '@rollup/rollup-darwin-x64@4.22.4': + optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.13.2': optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.22.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.22.4': + optional: true + '@rollup/rollup-linux-arm64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-arm64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-arm64-musl@4.13.2': optional: true + '@rollup/rollup-linux-arm64-musl@4.22.4': + optional: true + '@rollup/rollup-linux-powerpc64le-gnu@4.13.2': optional: true + '@rollup/rollup-linux-powerpc64le-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-riscv64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-riscv64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-s390x-gnu@4.13.2': optional: true + '@rollup/rollup-linux-s390x-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-x64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-x64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-x64-musl@4.13.2': optional: true + '@rollup/rollup-linux-x64-musl@4.22.4': + optional: true + '@rollup/rollup-win32-arm64-msvc@4.13.2': optional: true + '@rollup/rollup-win32-arm64-msvc@4.22.4': + optional: true + '@rollup/rollup-win32-ia32-msvc@4.13.2': optional: true + '@rollup/rollup-win32-ia32-msvc@4.22.4': + optional: true + '@rollup/rollup-win32-x64-msvc@4.13.2': optional: true + '@rollup/rollup-win32-x64-msvc@4.22.4': + optional: true + '@sinclair/typebox@0.29.6': {} '@tootallnate/once@2.0.0': {} @@ -6205,31 +6721,31 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/bunyan@1.8.9': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/caseless@0.12.5': {} '@types/connect@3.4.36': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/connect@3.4.38': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/cors@2.8.17': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/estree@1.0.5': {} '@types/express-serve-static-core@4.17.43': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/qs': 6.9.14 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -6251,13 +6767,13 @@ snapshots: '@types/jsonwebtoken@9.0.6': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/long@4.0.2': {} '@types/memcached@2.2.10': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/mime@1.3.5': {} @@ -6265,14 +6781,14 @@ snapshots: '@types/mysql@2.15.22': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/node-fetch@2.6.11': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 form-data: 4.0.0 - '@types/node@18.19.31': + '@types/node@18.19.53': dependencies: undici-types: 5.26.5 @@ -6280,7 +6796,11 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@22.4.1': + '@types/node@20.16.9': + dependencies: + undici-types: 6.19.8 + + '@types/node@22.7.2': dependencies: undici-types: 6.19.8 @@ -6292,7 +6812,7 @@ snapshots: '@types/pg@8.6.1': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 pg-protocol: 1.6.0 pg-types: 2.2.0 @@ -6303,7 +6823,7 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 @@ -6312,19 +6832,19 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/serve-static@1.15.5': dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/shimmer@1.0.5': {} '@types/tedious@4.0.14': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/tough-cookie@4.0.5': {} @@ -6370,6 +6890,10 @@ snapshots: optionalDependencies: ajv: 8.12.0 + ajv-formats@3.0.1(ajv@8.17.1): + optionalDependencies: + ajv: 8.17.1 + ajv@8.12.0: dependencies: fast-deep-equal: 3.1.3 @@ -6377,6 +6901,13 @@ snapshots: require-from-string: 2.0.2 uri-js: 4.4.1 + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.0.1 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-regex@5.0.1: {} ansi-regex@6.0.1: {} @@ -6516,6 +7047,11 @@ snapshots: esbuild: 0.19.12 load-tsconfig: 0.2.5 + bundle-require@5.0.0(esbuild@0.23.1): + dependencies: + esbuild: 0.23.1 + load-tsconfig: 0.2.5 + bytes@3.1.2: {} cac@6.7.14: {} @@ -6648,6 +7184,8 @@ snapshots: concat-map@0.0.1: {} + consola@3.2.3: {} + console-control-strings@1.1.0: optional: true @@ -6720,6 +7258,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.3.7: + dependencies: + ms: 2.1.3 + decamelize@1.2.0: {} decompress-response@4.2.1: @@ -6911,6 +7453,33 @@ snapshots: '@esbuild/win32-ia32': 0.19.12 '@esbuild/win32-x64': 0.19.12 + esbuild@0.23.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.23.1 + '@esbuild/android-arm': 0.23.1 + '@esbuild/android-arm64': 0.23.1 + '@esbuild/android-x64': 0.23.1 + '@esbuild/darwin-arm64': 0.23.1 + '@esbuild/darwin-x64': 0.23.1 + '@esbuild/freebsd-arm64': 0.23.1 + '@esbuild/freebsd-x64': 0.23.1 + '@esbuild/linux-arm': 0.23.1 + '@esbuild/linux-arm64': 0.23.1 + '@esbuild/linux-ia32': 0.23.1 + '@esbuild/linux-loong64': 0.23.1 + '@esbuild/linux-mips64el': 0.23.1 + '@esbuild/linux-ppc64': 0.23.1 + '@esbuild/linux-riscv64': 0.23.1 + '@esbuild/linux-s390x': 0.23.1 + '@esbuild/linux-x64': 0.23.1 + '@esbuild/netbsd-x64': 0.23.1 + '@esbuild/openbsd-arm64': 0.23.1 + '@esbuild/openbsd-x64': 0.23.1 + '@esbuild/sunos-x64': 0.23.1 + '@esbuild/win32-arm64': 0.23.1 + '@esbuild/win32-ia32': 0.23.1 + '@esbuild/win32-x64': 0.23.1 + escalade@3.1.2: {} escape-html@1.0.3: {} @@ -7002,6 +7571,8 @@ snapshots: fast-text-encoding@1.0.6: {} + fast-uri@3.0.1: {} + fast-xml-parser@4.3.6: dependencies: strnum: 1.0.5 @@ -7015,6 +7586,10 @@ snapshots: dependencies: websocket-driver: 0.7.4 + fdir@6.3.0(picomatch@4.0.2): + optionalDependencies: + picomatch: 4.0.2 + fecha@4.2.3: {} fetch-blob@3.2.0: @@ -7051,7 +7626,7 @@ snapshots: '@fastify/busboy': 2.1.1 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 20.11.30 + '@types/node': 20.16.9 farmhash: 3.3.1 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -7070,7 +7645,7 @@ snapshots: '@fastify/busboy': 2.1.1 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 20.11.30 + '@types/node': 20.16.9 farmhash-modern: 1.1.0 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -7089,7 +7664,7 @@ snapshots: '@fastify/busboy': 3.0.0 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 22.4.1 + '@types/node': 22.7.2 farmhash-modern: 1.1.0 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -7292,6 +7867,10 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + get-tsconfig@4.8.1: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -7909,6 +8488,8 @@ snapshots: lilconfig@3.1.1: {} + lilconfig@3.1.2: {} + limiter@1.1.5: {} lines-and-columns@1.2.4: {} @@ -8210,7 +8791,7 @@ snapshots: openai@4.53.0(encoding@0.1.13): dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -8321,11 +8902,12 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 - picocolors@1.1.0: - optional: true + picocolors@1.1.0: {} picomatch@2.3.1: {} + picomatch@4.0.2: {} + pidtree@0.3.1: {} pify@3.0.0: {} @@ -8341,6 +8923,13 @@ snapshots: optionalDependencies: postcss: 8.4.47 + postcss-load-config@6.0.1(postcss@8.4.47)(tsx@4.19.1): + dependencies: + lilconfig: 3.1.2 + optionalDependencies: + postcss: 8.4.47 + tsx: 4.19.1 + postcss@8.4.47: dependencies: nanoid: 3.3.7 @@ -8406,7 +8995,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 long: 5.2.3 protobufjs@7.3.2: @@ -8421,7 +9010,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 long: 5.2.3 proxy-addr@2.0.7: @@ -8554,6 +9143,28 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.13.2 fsevents: 2.3.3 + rollup@4.22.4: + dependencies: + '@types/estree': 1.0.5 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.22.4 + '@rollup/rollup-android-arm64': 4.22.4 + '@rollup/rollup-darwin-arm64': 4.22.4 + '@rollup/rollup-darwin-x64': 4.22.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.22.4 + '@rollup/rollup-linux-arm-musleabihf': 4.22.4 + '@rollup/rollup-linux-arm64-gnu': 4.22.4 + '@rollup/rollup-linux-arm64-musl': 4.22.4 + '@rollup/rollup-linux-powerpc64le-gnu': 4.22.4 + '@rollup/rollup-linux-riscv64-gnu': 4.22.4 + '@rollup/rollup-linux-s390x-gnu': 4.22.4 + '@rollup/rollup-linux-x64-gnu': 4.22.4 + '@rollup/rollup-linux-x64-musl': 4.22.4 + '@rollup/rollup-win32-arm64-msvc': 4.22.4 + '@rollup/rollup-win32-ia32-msvc': 4.22.4 + '@rollup/rollup-win32-x64-msvc': 4.22.4 + fsevents: 2.3.3 + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -8844,6 +9455,11 @@ snapshots: dependencies: any-promise: 1.3.0 + tinyglobby@0.2.6: + dependencies: + fdir: 6.3.0(picomatch@4.0.2) + picomatch: 4.0.2 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -8889,6 +9505,40 @@ snapshots: - supports-color - ts-node + tsup@8.3.0(postcss@8.4.47)(tsx@4.19.1)(typescript@4.9.5): + dependencies: + bundle-require: 5.0.0(esbuild@0.23.1) + cac: 6.7.14 + chokidar: 3.6.0 + consola: 3.2.3 + debug: 4.3.7 + esbuild: 0.23.1 + execa: 5.1.1 + joycon: 3.1.1 + picocolors: 1.1.0 + postcss-load-config: 6.0.1(postcss@8.4.47)(tsx@4.19.1) + resolve-from: 5.0.0 + rollup: 4.22.4 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyglobby: 0.2.6 + tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.4.47 + typescript: 4.9.5 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + tsx@4.19.1: + dependencies: + esbuild: 0.23.1 + get-tsconfig: 4.8.1 + optionalDependencies: + fsevents: 2.3.3 + tsx@4.7.1: dependencies: esbuild: 0.19.12 @@ -9117,6 +9767,10 @@ snapshots: dependencies: zod: 3.23.8 + zod-to-json-schema@3.23.3(zod@3.23.8): + dependencies: + zod: 3.23.8 + zod@3.22.4: {} zod@3.23.8: {} diff --git a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts index 906c7093fa..12e209eca0 100644 --- a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts +++ b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts @@ -15,7 +15,7 @@ */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; const DELICIOUSNESS_VALUES = ['yes', 'no', 'maybe'] as const; @@ -30,7 +30,7 @@ export async function deliciousnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts b/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts index 8eb70dd4b0..8d0cc0be20 100644 --- a/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts +++ b/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -42,7 +42,7 @@ export function createDeliciousnessEvaluator< displayName: 'Deliciousness', definition: 'Determines if output is considered delicous.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await deliciousnessScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/funniness/funniness.ts b/js/testapps/byo-evaluator/src/funniness/funniness.ts index 596fe31384..810c9c0f53 100644 --- a/js/testapps/byo-evaluator/src/funniness/funniness.ts +++ b/js/testapps/byo-evaluator/src/funniness/funniness.ts @@ -15,7 +15,7 @@ */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; const FUNNINESS_VALUES = [ @@ -32,7 +32,7 @@ const FunninessResponseSchema = z.object({ export async function funninessScore( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: z.infer ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts b/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts index 326568266e..34cd850ca5 100644 --- a/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts +++ b/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -43,7 +43,7 @@ export function createFunninessEvaluator< definition: 'Judges whether a statement is a joke and whether that joke is funny.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await funninessScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/pii/pii_detection.ts b/js/testapps/byo-evaluator/src/pii/pii_detection.ts index 7355ec2bb3..1cacb8b433 100644 --- a/js/testapps/byo-evaluator/src/pii/pii_detection.ts +++ b/js/testapps/byo-evaluator/src/pii/pii_detection.ts @@ -15,7 +15,7 @@ */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; const PiiDetectionResponseSchema = z.object({ @@ -27,7 +27,7 @@ export async function piiDetectionScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts b/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts index 247e4a7bbf..ad644a5dda 100644 --- a/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts +++ b/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -40,7 +40,7 @@ export function createPiiEvaluator( displayName: 'PII Detection', definition: 'Detects whether PII is present in the output.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await piiDetectionScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts b/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts index 6f0e084a85..cb0f8c2298 100644 --- a/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts +++ b/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts @@ -15,7 +15,7 @@ */ import { - BaseDataPoint, + BaseEvalDataPoint, EvalResponse, EvaluatorAction, Score, @@ -29,7 +29,7 @@ const REGEX_MATCH_NAME_PREFIX = 'REGEX_MATCH'; /** * Create an EvalResponse from an individual scored datapoint. */ -function fillScores(dataPoint: BaseDataPoint, score: Score): EvalResponse { +function fillScores(dataPoint: BaseEvalDataPoint, score: Score): EvalResponse { return { testCaseId: dataPoint.testCaseId, evaluation: score, @@ -70,7 +70,7 @@ export function createRegexEvaluators( 'Runs the output against a regex and responds with 1 if a match is found and 0 otherwise.', isBilled: false, }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await regexMatchScore(datapoint, regexMetric.regex); return fillScores(datapoint, score); } @@ -82,7 +82,7 @@ export function createRegexEvaluators( * Score an individual datapoint. */ export async function regexMatchScore( - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, regex: RegExp ): Promise { const d = dataPoint; From 1f2372f96bac5eebdca0bf916f3e7e847f67e316 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Thu, 26 Sep 2024 20:29:27 +0000 Subject: [PATCH 4/9] Fix tests (existing) --- .../common/src/eval/localFileDatasetStore.ts | 26 ++-- genkit-tools/common/src/types/apis.ts | 2 - genkit-tools/common/src/types/eval.ts | 1 - .../tests/eval/localFileDatasetStore_test.ts | 138 +++++++++++------- .../src/deliciousness/deliciousness.ts | 2 +- .../byo-evaluator/src/funniness/funniness.ts | 2 +- .../byo-evaluator/src/pii/pii_detection.ts | 2 +- 7 files changed, 95 insertions(+), 78 deletions(-) diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 974823ad80..4cab43cab6 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import crypto, { randomUUID } from 'crypto'; +import crypto from 'crypto'; import fs from 'fs'; import { readFile, rm, writeFile } from 'fs/promises'; import path from 'path'; @@ -27,6 +27,7 @@ import { DatasetStore, EvalFlowInput, } from '../types/eval'; +import { generateTestCaseId } from '../utils'; import { logger } from '../utils/logger'; /** @@ -65,17 +66,12 @@ export class LocalFileDatasetStore implements DatasetStore { } async createDataset(req: CreateDatasetRequest): Promise { - return this.createDatasetInternal( - req.data, - req.datasetId, - req.targetAction - ); + return this.createDatasetInternal(req.data, req.datasetId); } private async createDatasetInternal( data: EvalFlowInput, - datasetId?: string, - targetAction?: string + datasetId?: string ): Promise { const id = await this.generateDatasetId(datasetId); const filePath = path.resolve(this.storeRoot, this.generateFileName(id)); @@ -95,7 +91,6 @@ export class LocalFileDatasetStore implements DatasetStore { datasetId: id, size: dataset.length, version: 1, - targetAction, createTime: now, updateTime: now, }; @@ -138,7 +133,6 @@ export class LocalFileDatasetStore implements DatasetStore { datasetId: datasetId, size: newSize, version: prevMetadata.version + 1, - targetAction: req.targetAction ?? prevMetadata.targetAction, createTime: prevMetadata.createTime, updateTime: now, }; @@ -162,9 +156,9 @@ export class LocalFileDatasetStore implements DatasetStore { if (!fs.existsSync(filePath)) { throw new Error(`Dataset not found for dataset ID {$id}`); } - return await readFile(filePath, 'utf8').then((data) => - DatasetSchema.parse(JSON.parse(data)) - ); + return await readFile(filePath, 'utf8').then((data) => { + return DatasetSchema.parse(JSON.parse(data)); + }); } async listDatasets(): Promise { @@ -248,12 +242,12 @@ export class LocalFileDatasetStore implements DatasetStore { private getDatasetFromEvalFlowInput(data: EvalFlowInput): Dataset { if (Array.isArray(data)) { return data.map((d) => ({ - testCaseId: d.testCaseId ?? randomUUID(), + testCaseId: d.testCaseId ?? generateTestCaseId(), input: d, })); } else if (!!data.samples) { return data.samples.map((d) => ({ - testCaseId: d.testCaseId ?? randomUUID(), + testCaseId: d.testCaseId ?? generateTestCaseId(), ...d, })); } @@ -273,7 +267,7 @@ export class LocalFileDatasetStore implements DatasetStore { datasetMap.set(key, value); }); - const newDataset = Object.values(patchMap) as Dataset; + const newDataset = Array.from(datasetMap.values()) as Dataset; await writeFile(filePath, JSON.stringify(newDataset)); return newDataset.length; } diff --git a/genkit-tools/common/src/types/apis.ts b/genkit-tools/common/src/types/apis.ts index 254ec8bfb4..eaaaebe07f 100644 --- a/genkit-tools/common/src/types/apis.ts +++ b/genkit-tools/common/src/types/apis.ts @@ -135,14 +135,12 @@ export type GetEvalRunRequest = z.infer; export const CreateDatasetRequestSchema = z.object({ data: EvalFlowInputSchema, datasetId: z.string().optional(), - targetAction: z.string().optional(), }); export type CreateDatasetRequest = z.infer; export const UpdateDatasetRequestSchema = z.object({ data: EvalFlowInputSchema.optional(), - targetAction: z.string().optional(), datasetId: z.string(), }); export type UpdateDatasetRequest = z.infer; diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 880f573c19..3c34385680 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -164,7 +164,6 @@ export const DatasetMetadataSchema = z.object({ size: z.number(), /** 1 for v1, 2 for v2, etc */ version: z.number(), - targetAction: z.string().optional(), createTime: z.string(), updateTime: z.string(), }); diff --git a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts index 99096ff6f4..722902672a 100644 --- a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts +++ b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts @@ -29,36 +29,44 @@ import { CreateDatasetRequestSchema, UpdateDatasetRequestSchema, } from '../../src/types/apis'; -import { DatasetStore } from '../../src/types/eval'; +import { Dataset, DatasetStore } from '../../src/types/eval'; const FAKE_TIME = new Date('2024-02-03T12:05:33.243Z'); -const SAMPLE_DATASET_1_V1 = { - samples: [ - { - input: 'Cats are evil', - reference: 'Sorry no reference', - }, - { - input: 'Dogs are beautiful', - }, - ], -}; - -const SAMPLE_DATASET_1_V2 = { - samples: [ - { - input: 'Cats are evil', - reference: 'Sorry no reference', - }, - { - input: 'Dogs are angels', - }, - { - input: 'Dogs are also super cute', - }, - ], -}; +const SAMPLE_DATASET_1_V1 = [ + { + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + input: 'Dogs are beautiful', + }, +]; + +const SAMPLE_DATASET_1_WITH_IDS = [ + { + testCaseId: '1', + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + testCaseId: '2', + input: 'Dogs are beautiful', + }, +]; + +const SAMPLE_DATASET_1_V2 = [ + { + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + input: 'Dogs are angels', + }, + { + input: 'Dogs are also super cute', + }, +]; const SAMPLE_DATASET_ID_1 = 'dataset-1-123456'; @@ -73,19 +81,17 @@ const SAMPLE_DATASET_METADATA_1_V2 = { datasetId: SAMPLE_DATASET_ID_1, size: 3, version: 2, - targetAction: '/flow/myFlow', createTime: FAKE_TIME.toString(), updateTime: FAKE_TIME.toString(), }; const CREATE_DATASET_REQUEST = CreateDatasetRequestSchema.parse({ - data: SAMPLE_DATASET_1_V1, + data: { samples: SAMPLE_DATASET_1_V1 }, }); const UPDATE_DATASET_REQUEST = UpdateDatasetRequestSchema.parse({ - data: SAMPLE_DATASET_1_V2, + data: { samples: SAMPLE_DATASET_1_V2 }, datasetId: SAMPLE_DATASET_ID_1, - targetAction: '/flow/myFlow', }); const SAMPLE_DATASET_ID_2 = 'dataset-2-123456'; @@ -108,6 +114,11 @@ jest.mock('crypto', () => { jest.mock('uuid'); const uuidSpy = jest.spyOn(uuid, 'v4'); +const TEST_CASE_ID = 'test-case-1234-1234-1234'; +jest.mock('../../src/utils', () => ({ + generateTestCaseId: jest.fn(() => TEST_CASE_ID), +})); +// const generateTestCaseIdSpy = jest.spyOn(utils, 'generateTestCaseId'); jest.useFakeTimers({ advanceTimers: true }); jest.setSystemTime(FAKE_TIME); @@ -119,6 +130,7 @@ describe('localFileDatasetStore', () => { // For storeRoot setup fs.existsSync = jest.fn(() => true); uuidSpy.mockReturnValueOnce('12345678'); + // generateTestCaseIdSpy.mockReturnValueOnce('test-case-1234-1234-1234'); LocalFileDatasetStore.reset(); DatasetStore = LocalFileDatasetStore.getDatasetStore() as DatasetStore; }); @@ -136,6 +148,10 @@ describe('localFileDatasetStore', () => { Promise.resolve(JSON.stringify({}) as any) ); fs.existsSync = jest.fn(() => false); + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s) => ({ + testCaseId: TEST_CASE_ID, + ...s, + })); const datasetMetadata = await DatasetStore.createDataset({ ...CREATE_DATASET_REQUEST, @@ -146,7 +162,7 @@ describe('localFileDatasetStore', () => { expect(fs.promises.writeFile).toHaveBeenNthCalledWith( 1, expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), - JSON.stringify(CREATE_DATASET_REQUEST.data) + JSON.stringify(dataset) ); const metadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, @@ -195,21 +211,41 @@ describe('localFileDatasetStore', () => { [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, }; // For index file reads - fs.promises.readFile = jest.fn(async () => - Promise.resolve(JSON.stringify(metadataMap) as any) - ); + fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); - - const datasetMetadata = await DatasetStore.updateDataset( - UPDATE_DATASET_REQUEST - ); + const dataset: Dataset = [...SAMPLE_DATASET_1_WITH_IDS]; + fs.promises.readFile = jest + .fn() + .mockImplementationOnce(async () => + Promise.resolve(Buffer.from(JSON.stringify(metadataMap))) + ) + .mockImplementationOnce(async () => + Promise.resolve(Buffer.from(JSON.stringify(dataset) as any)) + ) as any; + + const datasetMetadata = await DatasetStore.updateDataset({ + data: { + samples: [ + { + input: 'A new information on cat dog', + }, + ], + }, + datasetId: SAMPLE_DATASET_ID_1, + }); expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); expect(fs.promises.writeFile).toHaveBeenNthCalledWith( 1, expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), - JSON.stringify(SAMPLE_DATASET_1_V2) + JSON.stringify([ + ...dataset, + { + testCaseId: TEST_CASE_ID, + input: 'A new information on cat dog', + }, + ]) ); const updatedMetadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V2, @@ -223,20 +259,6 @@ describe('localFileDatasetStore', () => { expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V2); }); - it('fails if data is not passed', async () => { - fs.existsSync = jest.fn(() => true); - - expect(async () => { - await DatasetStore.updateDataset({ - datasetId: SAMPLE_DATASET_ID_1, - }); - }).rejects.toThrow( - new Error('Error: `data` is required for updateDataset') - ); - - expect(fs.promises.writeFile).toBeCalledTimes(0); - }); - it('fails for non existing dataset', async () => { fs.existsSync = jest.fn(() => false); @@ -278,14 +300,18 @@ describe('localFileDatasetStore', () => { describe('getDataset', () => { it('succeeds for existing dataset', async () => { + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s) => ({ + ...s, + testCaseId: 'id-x', + })); fs.existsSync = jest.fn(() => true); fs.promises.readFile = jest.fn(async () => - Promise.resolve(JSON.stringify(SAMPLE_DATASET_1_V1) as any) + Promise.resolve(JSON.stringify(dataset) as any) ); const fetchedDataset = await DatasetStore.getDataset(SAMPLE_DATASET_ID_1); - expect(fetchedDataset).toMatchObject(SAMPLE_DATASET_1_V1); + expect(fetchedDataset).toMatchObject(dataset); }); it('fails for non existing dataset', async () => { diff --git a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts index 0c1ab8bbbf..22713cb244 100644 --- a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts +++ b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { ModelArgument, loadPromptFile, z } from 'genkit'; import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; -import { loadPromptFile, ModelArgument, z } from 'genkit'; import path from 'path'; import { ai } from '../index.js'; diff --git a/js/testapps/byo-evaluator/src/funniness/funniness.ts b/js/testapps/byo-evaluator/src/funniness/funniness.ts index 74dfc61536..68540904e2 100644 --- a/js/testapps/byo-evaluator/src/funniness/funniness.ts +++ b/js/testapps/byo-evaluator/src/funniness/funniness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { ModelArgument, loadPromptFile, z } from 'genkit'; import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; -import { loadPromptFile, ModelArgument, z } from 'genkit'; import path from 'path'; import { ai } from '../index.js'; diff --git a/js/testapps/byo-evaluator/src/pii/pii_detection.ts b/js/testapps/byo-evaluator/src/pii/pii_detection.ts index 9a9b4779ae..ea0099f4a9 100644 --- a/js/testapps/byo-evaluator/src/pii/pii_detection.ts +++ b/js/testapps/byo-evaluator/src/pii/pii_detection.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { ModelArgument, loadPromptFile, z } from 'genkit'; import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; -import { loadPromptFile, ModelArgument, z } from 'genkit'; import path from 'path'; import { ai } from '../index.js'; From edb94c5dd6ad881ca71c0b0a1d023cb114595cf3 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Fri, 27 Sep 2024 21:44:45 +0000 Subject: [PATCH 5/9] Update names --- genkit-tools/cli/src/commands/eval-flow.ts | 75 ++++++++++++------- genkit-tools/cli/src/commands/eval-run.ts | 4 +- genkit-tools/common/src/eval/evaluate.ts | 23 ++---- genkit-tools/common/src/eval/index.ts | 5 +- .../common/src/eval/localFileDatasetStore.ts | 6 +- genkit-tools/common/src/server/router.ts | 2 +- genkit-tools/common/src/types/apis.ts | 6 +- genkit-tools/common/src/types/eval.ts | 16 ++-- js/testapps/cat-eval/src/pdf_rag.ts | 1 + 9 files changed, 79 insertions(+), 59 deletions(-) diff --git a/genkit-tools/cli/src/commands/eval-flow.ts b/genkit-tools/cli/src/commands/eval-flow.ts index fa2b4f0db7..6616839b1a 100644 --- a/genkit-tools/cli/src/commands/eval-flow.ts +++ b/genkit-tools/cli/src/commands/eval-flow.ts @@ -16,8 +16,8 @@ import { Action, - EvalFlowInput, - EvalFlowInputSchema, + EvalInferenceInput, + EvalInferenceInputSchema, } from '@genkit-ai/tools-common'; import { EvalExporter, @@ -105,8 +105,18 @@ export const evalFlow = new Command('eval:flow') } } + const sourceType = getSourceType(data, options.input); + let targetDataset; + if (sourceType === 'dataset') { + const datasetStore = await getDatasetStore(); + const datasetMetadatas = await datasetStore.listDatasets(); + targetDataset = datasetMetadatas.find( + (d) => d.datasetId === options.input + ); + } + const actionRef = `/flow/${flowName}`; - const evalFlowInput = await readInputs(data, options.input); + const evalFlowInput = await readInputs(sourceType, data, options.input); const evalDataset = await runInference({ runner, actionRef, @@ -119,12 +129,13 @@ export const evalFlow = new Command('eval:flow') evaluatorActions, evalDataset, actionRef: `/flow/${flowName}`, - datasetId: - options.input && !options.input.endsWith('.json') - ? options.input - : undefined, }); - + // Augment metadata in evalKey + evalRun.key = { + ...evalRun.key, + datasetId: sourceType === 'dataset' ? options.input : undefined, + datasetVersion: targetDataset?.version, + }; const evalStore = getEvalStore(); await evalStore.save(evalRun); @@ -147,33 +158,45 @@ export const evalFlow = new Command('eval:flow') * Only one of these parameters is expected to be provided. **/ async function readInputs( - data?: string, + sourceType: 'data' | 'file' | 'dataset', + dataField?: string, input?: string -): Promise { +): Promise { let parsedData; - if (input) { - if (data) { - logger.warn('Both [data] and input provided, ignoring [data]...'); - } - const isFile = input.endsWith('.json'); - if (isFile) { - parsedData = JSON.parse(await readFile(input, 'utf8')); - } else { + switch (sourceType) { + case 'data': + parsedData = JSON.parse(dataField!); + break; + case 'file': + parsedData = JSON.parse(await readFile(input!, 'utf8')); + break; + case 'dataset': const datasetStore = await getDatasetStore(); - parsedData = await datasetStore.getDataset(input); - } - } else if (data) { - parsedData = JSON.parse(data); - } - if (Array.isArray(parsedData)) { - return parsedData as any[]; + const data = await datasetStore.getDataset(input!); + parsedData = { samples: data }; + break; } try { - return EvalFlowInputSchema.parse(parsedData); + return EvalInferenceInputSchema.parse(parsedData); } catch (e) { throw new Error( `Error parsing the input. Please provide an array of inputs for the flow or a ${EVAL_FLOW_SCHEMA} object. Error: ${e}` ); } } + +function getSourceType( + data?: string, + input?: string +): 'data' | 'file' | 'dataset' { + if (input) { + if (data) { + logger.warn('Both [data] and input provided, ignoring [data]...'); + } + return input.endsWith('.json') ? 'file' : 'dataset'; + } else if (data) { + return 'data'; + } + throw new Error('Must provide either data or input'); +} diff --git a/genkit-tools/cli/src/commands/eval-run.ts b/genkit-tools/cli/src/commands/eval-run.ts index 4656447358..b75812c474 100644 --- a/genkit-tools/cli/src/commands/eval-run.ts +++ b/genkit-tools/cli/src/commands/eval-run.ts @@ -94,8 +94,8 @@ export const evalRun = new Command('eval:run') (await readFile(dataset)).toString('utf-8') ).map((testCase: any) => ({ ...testCase, - testCaseId: testCase.testCaseId || generateTestCaseId(), - traceIds: testCase.traceIds || [], + testCaseId: testCase.testCaseId ?? generateTestCaseId(), + traceIds: testCase.traceIds ?? [], })); const evalRun = await runEvaluation({ runner, diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index b9642d4c2c..f0059e3e13 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -73,10 +73,14 @@ export async function runNewEvaluation( evaluatorActions, evalDataset, actionRef, - datasetId, - datasetVersion: targetDataset!.version, }); - logger.info('Finished evaluation, returning key...'); + // Augment metadata in evalKey + evalRun.key = { + ...evalRun.key, + datasetId, + datasetVersion: targetDataset?.version, + }; + logger.info('Finished evaluation, writing key...'); const evalStore = getEvalStore(); await evalStore.save(evalRun); @@ -119,17 +123,8 @@ export async function runEvaluation(params: { evaluatorActions: Action[]; evalDataset: EvalInput[]; actionRef?: string; - datasetId?: string; - datasetVersion?: number; }): Promise { - const { - runner, - evaluatorActions, - evalDataset, - actionRef, - datasetVersion, - datasetId, - } = params; + const { runner, evaluatorActions, evalDataset, actionRef } = params; const evalRunId = randomUUID(); const scores: Record = {}; for (const action of evaluatorActions) { @@ -151,8 +146,6 @@ export async function runEvaluation(params: { key: { actionRef, evalRunId, - datasetId, - datasetVersion, createdAt: new Date().toISOString(), }, results: scoredResults, diff --git a/genkit-tools/common/src/eval/index.ts b/genkit-tools/common/src/eval/index.ts index 3e87c3392f..ff88db8021 100644 --- a/genkit-tools/common/src/eval/index.ts +++ b/genkit-tools/common/src/eval/index.ts @@ -17,7 +17,10 @@ import { DatasetStore, EvalStore } from '../types/eval'; import { LocalFileDatasetStore } from './localFileDatasetStore'; import { LocalFileEvalStore } from './localFileEvalStore'; -export { EvalFlowInput, EvalFlowInputSchema } from '../types/eval'; +export { + EvalInferenceInput as EvalFlowInput, + EvalInferenceInputSchema as EvalFlowInputSchema, +} from '../types/eval'; export * from './evaluate'; export * from './exporter'; export * from './parser'; diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 4cab43cab6..493cdd14eb 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -25,7 +25,7 @@ import { DatasetMetadata, DatasetSchema, DatasetStore, - EvalFlowInput, + EvalInferenceInput, } from '../types/eval'; import { generateTestCaseId } from '../utils'; import { logger } from '../utils/logger'; @@ -70,7 +70,7 @@ export class LocalFileDatasetStore implements DatasetStore { } private async createDatasetInternal( - data: EvalFlowInput, + data: EvalInferenceInput, datasetId?: string ): Promise { const id = await this.generateDatasetId(datasetId); @@ -239,7 +239,7 @@ export class LocalFileDatasetStore implements DatasetStore { ); } - private getDatasetFromEvalFlowInput(data: EvalFlowInput): Dataset { + private getDatasetFromEvalFlowInput(data: EvalInferenceInput): Dataset { if (Array.isArray(data)) { return data.map((d) => ({ testCaseId: d.testCaseId ?? generateTestCaseId(), diff --git a/genkit-tools/common/src/server/router.ts b/genkit-tools/common/src/server/router.ts index 4af1a4b110..b477be924d 100644 --- a/genkit-tools/common/src/server/router.ts +++ b/genkit-tools/common/src/server/router.ts @@ -199,7 +199,7 @@ export const TOOLS_SERVER_ROUTER = (runner: Runner) => /** Retrieves an existing dataset */ getDataset: loggedProcedure .input(z.string()) - .output(evals.EvalFlowInputSchema) + .output(evals.EvalInferenceInputSchema) .query(async ({ input }) => { const response = await getDatasetStore().getDataset(input); return response; diff --git a/genkit-tools/common/src/types/apis.ts b/genkit-tools/common/src/types/apis.ts index eaaaebe07f..1b6125c2ba 100644 --- a/genkit-tools/common/src/types/apis.ts +++ b/genkit-tools/common/src/types/apis.ts @@ -15,7 +15,7 @@ */ import { z } from 'zod'; -import { EvalFlowInputSchema, EvalRunKeySchema } from './eval'; +import { EvalInferenceInputSchema, EvalRunKeySchema } from './eval'; import { FlowStateSchema } from './flow'; import { GenerationCommonConfigSchema, @@ -133,14 +133,14 @@ export const GetEvalRunRequestSchema = z.object({ export type GetEvalRunRequest = z.infer; export const CreateDatasetRequestSchema = z.object({ - data: EvalFlowInputSchema, + data: EvalInferenceInputSchema, datasetId: z.string().optional(), }); export type CreateDatasetRequest = z.infer; export const UpdateDatasetRequestSchema = z.object({ - data: EvalFlowInputSchema.optional(), + data: EvalInferenceInputSchema.optional(), datasetId: z.string(), }); export type UpdateDatasetRequest = z.infer; diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 3c34385680..822da2d549 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -27,9 +27,9 @@ import { */ /** - * Structured input for eval:flow + * Structured input for inference part of evaluation */ -export const EvalFlowStructuredInputSchema = z.object({ +export const EvalInferenceStructuredInputSchema = z.object({ samples: z.array( z.object({ testCaseId: z.string().optional(), @@ -38,21 +38,21 @@ export const EvalFlowStructuredInputSchema = z.object({ }) ), }); -export type EvalFlowStructuredInput = z.infer< - typeof EvalFlowStructuredInputSchema +export type EvalInferenceStructuredInput = z.infer< + typeof EvalInferenceStructuredInputSchema >; /** - * A dataset that is ready for eval:flow. + * A dataset that is ready for inference * * This could be an array of input objects to the target flow, or * It could be a JSON object as specified, with support for references. */ -export const EvalFlowInputSchema = z.union([ +export const EvalInferenceInputSchema = z.union([ z.array(z.any()), - EvalFlowStructuredInputSchema, + EvalInferenceStructuredInputSchema, ]); -export type EvalFlowInput = z.infer; +export type EvalInferenceInput = z.infer; /** * Represents a Dataset, to be used for bulk-inference / evaluation diff --git a/js/testapps/cat-eval/src/pdf_rag.ts b/js/testapps/cat-eval/src/pdf_rag.ts index d832037254..87a227d610 100644 --- a/js/testapps/cat-eval/src/pdf_rag.ts +++ b/js/testapps/cat-eval/src/pdf_rag.ts @@ -53,6 +53,7 @@ export const pdfQA = ai.defineFlow( outputSchema: z.string(), }, async (query) => { + console.log(query); const docs = await ai.retrieve({ retriever: pdfChatRetriever, query, From 63c4f0e3cacdf69e2499707e357fb2f2a6cf684c Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 30 Sep 2024 18:58:55 +0000 Subject: [PATCH 6/9] WIP, tests pending --- genkit-tools/cli/src/commands/eval-flow.ts | 45 +++++++++---------- genkit-tools/cli/src/commands/eval-run.ts | 8 +--- genkit-tools/common/src/eval/evaluate.ts | 38 ++++++++-------- genkit-tools/common/src/eval/index.ts | 5 +-- .../common/src/eval/localFileDatasetStore.ts | 6 +-- genkit-tools/common/src/types/eval.ts | 13 ++++-- js/testapps/cat-eval/src/pdf_rag.ts | 1 - 7 files changed, 55 insertions(+), 61 deletions(-) diff --git a/genkit-tools/cli/src/commands/eval-flow.ts b/genkit-tools/cli/src/commands/eval-flow.ts index 6616839b1a..fa241c8210 100644 --- a/genkit-tools/cli/src/commands/eval-flow.ts +++ b/genkit-tools/cli/src/commands/eval-flow.ts @@ -23,7 +23,6 @@ import { EvalExporter, getAllEvaluatorActions, getDatasetStore, - getEvalStore, getExporterForString, getMatchingEvaluatorActions, runEvaluation, @@ -44,6 +43,11 @@ interface EvalFlowRunCliOptions { } const EVAL_FLOW_SCHEMA = '{samples: Array<{input: any; reference?: any;}>}'; +enum SourceType { + DATA = 'data', + FILE = 'file', + DATASET = 'dataset', +} /** Command to run a flow and evaluate the output */ export const evalFlow = new Command('eval:flow') @@ -106,11 +110,11 @@ export const evalFlow = new Command('eval:flow') } const sourceType = getSourceType(data, options.input); - let targetDataset; - if (sourceType === 'dataset') { + let targetDatasetMetadata; + if (sourceType === SourceType.DATASET) { const datasetStore = await getDatasetStore(); const datasetMetadatas = await datasetStore.listDatasets(); - targetDataset = datasetMetadatas.find( + targetDatasetMetadata = datasetMetadatas.find( (d) => d.datasetId === options.input ); } @@ -128,16 +132,13 @@ export const evalFlow = new Command('eval:flow') runner, evaluatorActions, evalDataset, - actionRef: `/flow/${flowName}`, + augments: { + actionRef: `/flow/${flowName}`, + datasetId: + sourceType === SourceType.DATASET ? options.input : undefined, + datasetVersion: targetDatasetMetadata?.version, + }, }); - // Augment metadata in evalKey - evalRun.key = { - ...evalRun.key, - datasetId: sourceType === 'dataset' ? options.input : undefined, - datasetVersion: targetDataset?.version, - }; - const evalStore = getEvalStore(); - await evalStore.save(evalRun); if (options.output) { const exportFn: EvalExporter = getExporterForString( @@ -158,21 +159,22 @@ export const evalFlow = new Command('eval:flow') * Only one of these parameters is expected to be provided. **/ async function readInputs( - sourceType: 'data' | 'file' | 'dataset', + sourceType: SourceType, dataField?: string, input?: string ): Promise { let parsedData; switch (sourceType) { - case 'data': + case SourceType.DATA: parsedData = JSON.parse(dataField!); break; - case 'file': + case SourceType.FILE: parsedData = JSON.parse(await readFile(input!, 'utf8')); break; - case 'dataset': + case SourceType.DATASET: const datasetStore = await getDatasetStore(); const data = await datasetStore.getDataset(input!); + // Format to match EvalInferenceInputSchema parsedData = { samples: data }; break; } @@ -186,17 +188,14 @@ async function readInputs( } } -function getSourceType( - data?: string, - input?: string -): 'data' | 'file' | 'dataset' { +function getSourceType(data?: string, input?: string): SourceType { if (input) { if (data) { logger.warn('Both [data] and input provided, ignoring [data]...'); } - return input.endsWith('.json') ? 'file' : 'dataset'; + return input.endsWith('.json') ? SourceType.FILE : SourceType.DATASET; } else if (data) { - return 'data'; + return SourceType.DATA; } throw new Error('Must provide either data or input'); } diff --git a/genkit-tools/cli/src/commands/eval-run.ts b/genkit-tools/cli/src/commands/eval-run.ts index b75812c474..80649777d3 100644 --- a/genkit-tools/cli/src/commands/eval-run.ts +++ b/genkit-tools/cli/src/commands/eval-run.ts @@ -18,7 +18,6 @@ import { Action, EvalInput } from '@genkit-ai/tools-common'; import { EvalExporter, getAllEvaluatorActions, - getEvalStore, getExporterForString, getMatchingEvaluatorActions, runEvaluation, @@ -94,8 +93,8 @@ export const evalRun = new Command('eval:run') (await readFile(dataset)).toString('utf-8') ).map((testCase: any) => ({ ...testCase, - testCaseId: testCase.testCaseId ?? generateTestCaseId(), - traceIds: testCase.traceIds ?? [], + testCaseId: testCase.testCaseId || generateTestCaseId(), + traceIds: testCase.traceIds || [], })); const evalRun = await runEvaluation({ runner, @@ -103,9 +102,6 @@ export const evalRun = new Command('eval:run') evalDataset, }); - const evalStore = getEvalStore(); - await evalStore.save(evalRun); - if (options.output) { const exportFn: EvalExporter = getExporterForString( options.outputFormat diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index 8366f15e7f..9707cbbe89 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -15,11 +15,12 @@ */ import { randomUUID } from 'crypto'; -import { EvalFlowInput, getDatasetStore, getEvalStore } from '.'; +import { EvalInferenceInput, getDatasetStore, getEvalStore } from '.'; import { Runner } from '../runner'; import { Action, EvalInput, + EvalKeyAugments, EvalRun, EvalRunKey, RunNewEvaluationRequest, @@ -55,7 +56,10 @@ export async function runNewEvaluation( logger.info(`Fetching dataset ${datasetId}...`); const dataset = await datasetStore.getDataset(datasetId); const datasetMetadatas = await datasetStore.listDatasets(); - const targetDataset = datasetMetadatas.find((d) => d.datasetId === datasetId); + const targetDatasetMetadata = datasetMetadatas.find( + (d) => d.datasetId === datasetId + ); + const datasetVersion = targetDatasetMetadata?.version; logger.info('Running inference...'); const evalDataset = await runInference({ @@ -69,23 +73,12 @@ export async function runNewEvaluation( evaluators ); - logger.info('Running evaluation...'); const evalRun = await runEvaluation({ runner, evaluatorActions, evalDataset, - actionRef, + augments: { actionRef, datasetId, datasetVersion }, }); - // Augment metadata in evalKey - evalRun.key = { - ...evalRun.key, - datasetId, - datasetVersion: targetDataset?.version, - }; - logger.info('Finished evaluation, writing key...'); - const evalStore = getEvalStore(); - await evalStore.save(evalRun); - return evalRun.key; } @@ -93,7 +86,7 @@ export async function runNewEvaluation( export async function runInference(params: { runner: Runner; actionRef: string; - evalFlowInput: EvalFlowInput; + evalFlowInput: EvalInferenceInput; auth?: string; }): Promise { const { runner, actionRef, evalFlowInput, auth } = params; @@ -122,11 +115,12 @@ export async function runEvaluation(params: { runner: Runner; evaluatorActions: Action[]; evalDataset: EvalInput[]; - actionRef?: string; + augments?: EvalKeyAugments; }): Promise { - const { runner, evaluatorActions, evalDataset, actionRef } = params; + const { runner, evaluatorActions, evalDataset, augments } = params; const evalRunId = randomUUID(); const scores: Record = {}; + logger.info('Running evaluation...'); for (const action of evaluatorActions) { const name = evaluatorName(action); const response = await runner.runAction({ @@ -144,13 +138,17 @@ export async function runEvaluation(params: { const evalRun = { key: { - actionRef, evalRunId, createdAt: new Date().toISOString(), + ...augments, }, results: scoredResults, metricsMetadata: metadata, }; + + logger.info('Finished evaluation, writing key...'); + const evalStore = getEvalStore(); + await evalStore.save(evalRun); return evalRun; } @@ -189,7 +187,7 @@ export async function getMatchingEvaluatorActions( async function bulkRunAction(params: { runner: Runner; actionRef: string; - evalFlowInput: EvalFlowInput; + evalFlowInput: EvalInferenceInput; auth?: string; }): Promise { const { runner, actionRef, evalFlowInput, auth } = params; @@ -238,7 +236,7 @@ async function fetchEvalInput(params: { runner: Runner; actionRef: string; states: BulkRunResponse[]; - parsedData: EvalFlowInput; + parsedData: EvalInferenceInput; }): Promise { const { runner, actionRef, states, parsedData } = params; diff --git a/genkit-tools/common/src/eval/index.ts b/genkit-tools/common/src/eval/index.ts index ff88db8021..5cd4fadcb0 100644 --- a/genkit-tools/common/src/eval/index.ts +++ b/genkit-tools/common/src/eval/index.ts @@ -17,10 +17,7 @@ import { DatasetStore, EvalStore } from '../types/eval'; import { LocalFileDatasetStore } from './localFileDatasetStore'; import { LocalFileEvalStore } from './localFileEvalStore'; -export { - EvalInferenceInput as EvalFlowInput, - EvalInferenceInputSchema as EvalFlowInputSchema, -} from '../types/eval'; +export { EvalInferenceInput, EvalInferenceInputSchema } from '../types/eval'; export * from './evaluate'; export * from './exporter'; export * from './parser'; diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 493cdd14eb..a0dbe25599 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -82,7 +82,7 @@ export class LocalFileDatasetStore implements DatasetStore { `Create dataset failed: file already exists at {$filePath}` ); } - const dataset = this.getDatasetFromEvalFlowInput(data); + const dataset = this.getDatasetFromInferenceInput(data); logger.info(`Saving Dataset to ` + filePath); await writeFile(filePath, JSON.stringify(dataset)); @@ -121,7 +121,7 @@ export class LocalFileDatasetStore implements DatasetStore { if (!prevMetadata) { throw new Error(`Update dataset failed: dataset metadata not found`); } - const patch = this.getDatasetFromEvalFlowInput(req.data ?? []); + const patch = this.getDatasetFromInferenceInput(req.data ?? []); let newSize = prevMetadata.size; if (patch.length > 0) { logger.info(`Updating Dataset at ` + filePath); @@ -239,7 +239,7 @@ export class LocalFileDatasetStore implements DatasetStore { ); } - private getDatasetFromEvalFlowInput(data: EvalInferenceInput): Dataset { + private getDatasetFromInferenceInput(data: EvalInferenceInput): Dataset { if (Array.isArray(data)) { return data.map((d) => ({ testCaseId: d.testCaseId ?? generateTestCaseId(), diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 822da2d549..0d0a0e89e8 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -43,10 +43,9 @@ export type EvalInferenceStructuredInput = z.infer< >; /** - * A dataset that is ready for inference + * A set of samples that is ready for inference. * - * This could be an array of input objects to the target flow, or - * It could be a JSON object as specified, with support for references. + * This should be used in user-facing surfaces (CLI/API inputs) to accommodate various user input formats. For internal wire-transfer/storage, prefer {@link Dataset}. */ export const EvalInferenceInputSchema = z.union([ z.array(z.any()), @@ -55,7 +54,7 @@ export const EvalInferenceInputSchema = z.union([ export type EvalInferenceInput = z.infer; /** - * Represents a Dataset, to be used for bulk-inference / evaluation + * Represents a Dataset, to be used for bulk-inference / evaluation. This is a more optionated form of EvalInferenceInput, which guarantees testCaseId for each test sample. */ export const DatasetSchema = z.array( z.object({ @@ -113,6 +112,12 @@ export const EvalRunKeySchema = z.object({ createdAt: z.string(), }); export type EvalRunKey = z.infer; +export const EvalKeyAugmentsSchema = EvalRunKeySchema.pick({ + datasetId: true, + datasetVersion: true, + actionRef: true, +}); +export type EvalKeyAugments = z.infer; /** * A container for the results of evaluation over a batch of test cases. diff --git a/js/testapps/cat-eval/src/pdf_rag.ts b/js/testapps/cat-eval/src/pdf_rag.ts index 87a227d610..d832037254 100644 --- a/js/testapps/cat-eval/src/pdf_rag.ts +++ b/js/testapps/cat-eval/src/pdf_rag.ts @@ -53,7 +53,6 @@ export const pdfQA = ai.defineFlow( outputSchema: z.string(), }, async (query) => { - console.log(query); const docs = await ai.retrieve({ retriever: pdfChatRetriever, query, From d3fc171128251f680fbe74ad2739a346b670cd30 Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Mon, 30 Sep 2024 20:10:49 +0000 Subject: [PATCH 7/9] =?UTF-8?q?jest=20tests=20=F0=9F=98=AD=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/eval/localFileDatasetStore_test.ts | 120 ++++++++++++++++-- 1 file changed, 108 insertions(+), 12 deletions(-) diff --git a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts index 722902672a..fa404cc717 100644 --- a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts +++ b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts @@ -118,7 +118,6 @@ const TEST_CASE_ID = 'test-case-1234-1234-1234'; jest.mock('../../src/utils', () => ({ generateTestCaseId: jest.fn(() => TEST_CASE_ID), })); -// const generateTestCaseIdSpy = jest.spyOn(utils, 'generateTestCaseId'); jest.useFakeTimers({ advanceTimers: true }); jest.setSystemTime(FAKE_TIME); @@ -130,7 +129,6 @@ describe('localFileDatasetStore', () => { // For storeRoot setup fs.existsSync = jest.fn(() => true); uuidSpy.mockReturnValueOnce('12345678'); - // generateTestCaseIdSpy.mockReturnValueOnce('test-case-1234-1234-1234'); LocalFileDatasetStore.reset(); DatasetStore = LocalFileDatasetStore.getDatasetStore() as DatasetStore; }); @@ -175,6 +173,42 @@ describe('localFileDatasetStore', () => { expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V1); }); + it('writes and updates index for new dataset, testCaseIds are provided', async () => { + fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); + fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); + // For index file reads + fs.promises.readFile = jest.fn(async () => + Promise.resolve(JSON.stringify({}) as any) + ); + fs.existsSync = jest.fn(() => false); + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s, i) => ({ + testCaseId: TEST_CASE_ID + `index${i}`, + ...s, + })); + + const datasetMetadata = await DatasetStore.createDataset({ + ...CREATE_DATASET_REQUEST, + data: { samples: dataset }, + datasetId: SAMPLE_DATASET_ID_1, + }); + + expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 1, + expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), + JSON.stringify(dataset) + ); + const metadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, + }; + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('datasets/index.json'), + JSON.stringify(metadataMap) + ); + expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V1); + }); + it('fails request if dataset already exists', async () => { fs.existsSync = jest.fn(() => true); @@ -204,25 +238,22 @@ describe('localFileDatasetStore', () => { }); describe('updateDataset', () => { - it('succeeds for existing dataset', async () => { + it('succeeds for existing dataset -- append', async () => { fs.existsSync = jest.fn(() => true); let metadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, }; // For index file reads - + fs.promises.readFile = jest.fn(async () => + Promise.resolve(JSON.stringify(metadataMap) as any) + ); fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); const dataset: Dataset = [...SAMPLE_DATASET_1_WITH_IDS]; - fs.promises.readFile = jest - .fn() - .mockImplementationOnce(async () => - Promise.resolve(Buffer.from(JSON.stringify(metadataMap))) - ) - .mockImplementationOnce(async () => - Promise.resolve(Buffer.from(JSON.stringify(dataset) as any)) - ) as any; + const getDatasetSpy = jest + .spyOn(DatasetStore, 'getDataset') + .mockImplementation(() => Promise.resolve(dataset)); const datasetMetadata = await DatasetStore.updateDataset({ data: { @@ -256,6 +287,71 @@ describe('localFileDatasetStore', () => { expect.stringContaining('datasets/index.json'), JSON.stringify(updatedMetadataMap) ); + expect(getDatasetSpy).toHaveBeenCalledTimes(1); + expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V2); + }); + + it('succeeds for existing dataset -- append and replace', async () => { + fs.existsSync = jest.fn(() => true); + let metadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, + [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, + }; + // For index file reads + fs.promises.readFile = jest.fn(async () => + Promise.resolve(JSON.stringify(metadataMap) as any) + ); + fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); + fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); + const dataset: Dataset = [...SAMPLE_DATASET_1_WITH_IDS]; + const getDatasetSpy = jest + .spyOn(DatasetStore, 'getDataset') + .mockImplementation(() => Promise.resolve(dataset)); + + const datasetMetadata = await DatasetStore.updateDataset({ + data: { + samples: [ + { + input: 'A new information on cat dog', + }, + { + testCaseId: '1', + input: 'Other information on hot dog', + }, + ], + }, + datasetId: SAMPLE_DATASET_ID_1, + }); + + expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 1, + expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), + JSON.stringify([ + { + testCaseId: '1', + input: 'Other information on hot dog', + }, + { + testCaseId: '2', + input: 'Dogs are beautiful', + }, + { + testCaseId: TEST_CASE_ID, + input: 'A new information on cat dog', + }, + ]) + ); + const updatedMetadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V2, + [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, + }; + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('datasets/index.json'), + JSON.stringify(updatedMetadataMap) + ); + expect(getDatasetSpy).toHaveBeenCalledTimes(1); expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V2); }); From fae205d1cdd560a9e746fb4144583cd68c0f347a Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Fri, 11 Oct 2024 18:12:40 +0000 Subject: [PATCH 8/9] lock --- js/pnpm-lock.yaml | 74 +++++++++-------------------------------------- 1 file changed, 13 insertions(+), 61 deletions(-) diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 34f24530a0..b065e0a212 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -374,19 +374,11 @@ importers: specifier: ^0.19.0 version: 0.19.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) '@google-cloud/opentelemetry-cloud-trace-exporter': -<<<<<<< HEAD - specifier: ^2.1.0 - version: 2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) - '@google-cloud/opentelemetry-resource-util': - specifier: ^2.1.0 - version: 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) -======= specifier: ^2.4.1 version: 2.4.1(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) '@google-cloud/opentelemetry-resource-util': specifier: ^2.4.0 version: 2.4.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) ->>>>>>> next '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 @@ -3050,15 +3042,13 @@ packages: cpu: [x64] os: [win32] -<<<<<<< HEAD '@rollup/rollup-win32-x64-msvc@4.22.4': resolution: {integrity: sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q==} cpu: [x64] os: [win32] -======= + '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} ->>>>>>> next '@sinclair/typebox@0.29.6': resolution: {integrity: sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==} @@ -3264,14 +3254,12 @@ packages: ajv@8.12.0: resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} -<<<<<<< HEAD ajv@8.17.1: resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} -======= + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} ->>>>>>> next ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} @@ -3972,7 +3960,9 @@ packages: resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} engines: {node: '>=0.8.0'} -<<<<<<< HEAD + fb-watchman@2.0.2: + resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + fdir@6.3.0: resolution: {integrity: sha512-QOnuT+BOtivR77wYvCWHfGt9s4Pz1VIMbD463vegT5MLqNXy8rYFT/lPVEqf/bhYeT6qmqrNHhsX+rWwe3rOCQ==} peerDependencies: @@ -3980,10 +3970,6 @@ packages: peerDependenciesMeta: picomatch: optional: true -======= - fb-watchman@2.0.2: - resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} ->>>>>>> next fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} @@ -5966,18 +5952,16 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} -<<<<<<< HEAD tinyglobby@0.2.6: resolution: {integrity: sha512-NbBoFBpqfcgd1tCiO8Lkfdk+xrA7mlLR9zgvZcZWQQwU63XAfUePyd6wZBaU93Hqw347lHnwFzttAkemHzzz4g==} engines: {node: '>=12.0.0'} -======= + tmpl@1.0.5: resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} ->>>>>>> next to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} @@ -6925,19 +6909,11 @@ snapshots: - encoding - supports-color -<<<<<<< HEAD - '@google-cloud/opentelemetry-cloud-trace-exporter@2.1.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13)': - dependencies: - '@google-cloud/opentelemetry-resource-util': 2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13) - '@grpc/grpc-js': 1.10.4 - '@grpc/proto-loader': 0.7.12 -======= '@google-cloud/opentelemetry-cloud-trace-exporter@2.4.1(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13)': dependencies: '@google-cloud/opentelemetry-resource-util': 2.4.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) '@grpc/grpc-js': 1.10.10 '@grpc/proto-loader': 0.7.13 ->>>>>>> next '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) @@ -6947,26 +6923,10 @@ snapshots: - encoding - supports-color -<<<<<<< HEAD - '@google-cloud/opentelemetry-resource-util@2.1.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.27.0)(encoding@0.1.13)': - dependencies: - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.27.0 - gcp-metadata: 5.3.0(encoding@0.1.13) - transitivePeerDependencies: - - encoding - - supports-color - - '@google-cloud/opentelemetry-resource-util@2.3.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13)': - dependencies: - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.22.0 -======= '@google-cloud/opentelemetry-resource-util@2.4.0(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13)': dependencies: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.26.0 ->>>>>>> next gcp-metadata: 6.1.0(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -8078,12 +8038,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.13.2': optional: true -<<<<<<< HEAD '@rollup/rollup-win32-x64-msvc@4.22.4': optional: true -======= + '@sinclair/typebox@0.27.8': {} ->>>>>>> next '@sinclair/typebox@0.29.6': {} @@ -8330,18 +8288,16 @@ snapshots: require-from-string: 2.0.2 uri-js: 4.4.1 -<<<<<<< HEAD ajv@8.17.1: dependencies: fast-deep-equal: 3.1.3 fast-uri: 3.0.1 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 -======= + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 ->>>>>>> next ansi-regex@5.0.1: {} @@ -9208,15 +9164,13 @@ snapshots: dependencies: websocket-driver: 0.7.4 -<<<<<<< HEAD - fdir@6.3.0(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 -======= fb-watchman@2.0.2: dependencies: bser: 2.1.1 ->>>>>>> next + + fdir@6.3.0(picomatch@4.0.2): + optionalDependencies: + picomatch: 4.0.2 fecha@4.2.3: {} @@ -11590,16 +11544,14 @@ snapshots: dependencies: any-promise: 1.3.0 -<<<<<<< HEAD tinyglobby@0.2.6: dependencies: fdir: 6.3.0(picomatch@4.0.2) picomatch: 4.0.2 -======= + tmpl@1.0.5: {} to-fast-properties@2.0.0: {} ->>>>>>> next to-regex-range@5.0.1: dependencies: From 0b30c28241ed2a1b20e31a149a7aabf4ae3d76ad Mon Sep 17 00:00:00 2001 From: Samuel Bushi Date: Fri, 11 Oct 2024 18:58:31 +0000 Subject: [PATCH 9/9] fix dataset --- genkit-tools/common/src/eval/evaluate.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index e00b4f98e2..2de22c59d2 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -20,6 +20,7 @@ import { Runner } from '../runner'; import { Action, CandidateData, + EvalInferenceInputSchema, EvalInput, EvalKeyAugments, EvalRun, @@ -80,7 +81,7 @@ export async function runNewEvaluation( const evalDataset = await runInference({ runner, actionRef, - evalFlowInput: dataset, + evalFlowInput: EvalInferenceInputSchema.parse({ samples: dataset }), auth: request.options?.auth, actionConfig: request.options?.actionConfig, });