diff --git a/genkit-tools/cli/src/commands/eval-extract-data.ts b/genkit-tools/cli/src/commands/eval-extract-data.ts index 0482428eea..dc767974d3 100644 --- a/genkit-tools/cli/src/commands/eval-extract-data.ts +++ b/genkit-tools/cli/src/commands/eval-extract-data.ts @@ -15,9 +15,12 @@ */ import { EnvTypes, EvalInput, TraceData } from '@genkit-ai/tools-common'; -import { getEvalExtractors, logger } from '@genkit-ai/tools-common/utils'; +import { + generateTestCaseId, + getEvalExtractors, + logger, +} from '@genkit-ai/tools-common/utils'; import { Command } from 'commander'; -import { randomUUID } from 'crypto'; import { writeFile } from 'fs/promises'; import { runInRunnerThenStop } from '../utils/runner-utils'; @@ -73,7 +76,7 @@ export const evalExtractData = new Command('eval:extractData') .filter((t): t is TraceData => !!t) .map((trace) => { return { - testCaseId: randomUUID(), + testCaseId: generateTestCaseId(), input: extractors.input(trace), output: extractors.output(trace), context: JSON.parse(extractors.context(trace)) as string[], diff --git a/genkit-tools/cli/src/commands/eval-flow.ts b/genkit-tools/cli/src/commands/eval-flow.ts index fa2b4f0db7..fa241c8210 100644 --- a/genkit-tools/cli/src/commands/eval-flow.ts +++ b/genkit-tools/cli/src/commands/eval-flow.ts @@ -16,14 +16,13 @@ import { Action, - EvalFlowInput, - EvalFlowInputSchema, + EvalInferenceInput, + EvalInferenceInputSchema, } from '@genkit-ai/tools-common'; import { EvalExporter, getAllEvaluatorActions, getDatasetStore, - getEvalStore, getExporterForString, getMatchingEvaluatorActions, runEvaluation, @@ -44,6 +43,11 @@ interface EvalFlowRunCliOptions { } const EVAL_FLOW_SCHEMA = '{samples: Array<{input: any; reference?: any;}>}'; +enum SourceType { + DATA = 'data', + FILE = 'file', + DATASET = 'dataset', +} /** Command to run a flow and evaluate the output */ export const evalFlow = new Command('eval:flow') @@ -105,8 +109,18 @@ export const evalFlow = new Command('eval:flow') } } + const sourceType = getSourceType(data, options.input); + let targetDatasetMetadata; + if (sourceType === SourceType.DATASET) { + const datasetStore = await getDatasetStore(); + const datasetMetadatas = await datasetStore.listDatasets(); + targetDatasetMetadata = datasetMetadatas.find( + (d) => d.datasetId === options.input + ); + } + const actionRef = `/flow/${flowName}`; - const evalFlowInput = await readInputs(data, options.input); + const evalFlowInput = await readInputs(sourceType, data, options.input); const evalDataset = await runInference({ runner, actionRef, @@ -118,16 +132,14 @@ export const evalFlow = new Command('eval:flow') runner, evaluatorActions, evalDataset, - actionRef: `/flow/${flowName}`, - datasetId: - options.input && !options.input.endsWith('.json') - ? options.input - : undefined, + augments: { + actionRef: `/flow/${flowName}`, + datasetId: + sourceType === SourceType.DATASET ? options.input : undefined, + datasetVersion: targetDatasetMetadata?.version, + }, }); - const evalStore = getEvalStore(); - await evalStore.save(evalRun); - if (options.output) { const exportFn: EvalExporter = getExporterForString( options.outputFormat @@ -147,33 +159,43 @@ export const evalFlow = new Command('eval:flow') * Only one of these parameters is expected to be provided. **/ async function readInputs( - data?: string, + sourceType: SourceType, + dataField?: string, input?: string -): Promise { +): Promise { let parsedData; - if (input) { - if (data) { - logger.warn('Both [data] and input provided, ignoring [data]...'); - } - const isFile = input.endsWith('.json'); - if (isFile) { - parsedData = JSON.parse(await readFile(input, 'utf8')); - } else { + switch (sourceType) { + case SourceType.DATA: + parsedData = JSON.parse(dataField!); + break; + case SourceType.FILE: + parsedData = JSON.parse(await readFile(input!, 'utf8')); + break; + case SourceType.DATASET: const datasetStore = await getDatasetStore(); - parsedData = await datasetStore.getDataset(input); - } - } else if (data) { - parsedData = JSON.parse(data); - } - if (Array.isArray(parsedData)) { - return parsedData as any[]; + const data = await datasetStore.getDataset(input!); + // Format to match EvalInferenceInputSchema + parsedData = { samples: data }; + break; } try { - return EvalFlowInputSchema.parse(parsedData); + return EvalInferenceInputSchema.parse(parsedData); } catch (e) { throw new Error( `Error parsing the input. Please provide an array of inputs for the flow or a ${EVAL_FLOW_SCHEMA} object. Error: ${e}` ); } } + +function getSourceType(data?: string, input?: string): SourceType { + if (input) { + if (data) { + logger.warn('Both [data] and input provided, ignoring [data]...'); + } + return input.endsWith('.json') ? SourceType.FILE : SourceType.DATASET; + } else if (data) { + return SourceType.DATA; + } + throw new Error('Must provide either data or input'); +} diff --git a/genkit-tools/cli/src/commands/eval-run.ts b/genkit-tools/cli/src/commands/eval-run.ts index e8e877d660..80649777d3 100644 --- a/genkit-tools/cli/src/commands/eval-run.ts +++ b/genkit-tools/cli/src/commands/eval-run.ts @@ -18,14 +18,16 @@ import { Action, EvalInput } from '@genkit-ai/tools-common'; import { EvalExporter, getAllEvaluatorActions, - getEvalStore, getExporterForString, getMatchingEvaluatorActions, runEvaluation, } from '@genkit-ai/tools-common/eval'; -import { confirmLlmUse, logger } from '@genkit-ai/tools-common/utils'; +import { + confirmLlmUse, + generateTestCaseId, + logger, +} from '@genkit-ai/tools-common/utils'; import { Command } from 'commander'; -import { randomUUID } from 'crypto'; import { readFile } from 'fs/promises'; import { runInRunnerThenStop } from '../utils/runner-utils'; @@ -91,7 +93,7 @@ export const evalRun = new Command('eval:run') (await readFile(dataset)).toString('utf-8') ).map((testCase: any) => ({ ...testCase, - testCaseId: testCase.testCaseId || randomUUID(), + testCaseId: testCase.testCaseId || generateTestCaseId(), traceIds: testCase.traceIds || [], })); const evalRun = await runEvaluation({ @@ -100,9 +102,6 @@ export const evalRun = new Command('eval:run') evalDataset, }); - const evalStore = getEvalStore(); - await evalStore.save(evalRun); - if (options.output) { const exportFn: EvalExporter = getExporterForString( options.outputFormat diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index e322a0b74a..2de22c59d2 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -15,12 +15,14 @@ */ import { randomUUID } from 'crypto'; -import { EvalFlowInput, getDatasetStore, getEvalStore } from '.'; +import { EvalInferenceInput, getDatasetStore, getEvalStore } from '.'; import { Runner } from '../runner'; import { Action, CandidateData, + EvalInferenceInputSchema, EvalInput, + EvalKeyAugments, EvalRun, EvalRunKey, FlowActionInputSchema, @@ -33,6 +35,7 @@ import { } from '../types'; import { evaluatorName, + generateTestCaseId, getEvalExtractors, isEvaluator, logger, @@ -41,6 +44,7 @@ import { import { enrichResultsWithScoring, extractMetricsMetadata } from './parser'; interface InferenceRunState { + testCaseId: string; input: any; reference?: any; traceId?: string; @@ -48,6 +52,12 @@ interface InferenceRunState { evalError?: string; } +interface TestCase { + testCaseId: string; + input: any; + reference?: any; +} + const SUPPORTED_ACTION_TYPES = ['flow', 'model'] as const; /** @@ -61,12 +71,17 @@ export async function runNewEvaluation( const datasetStore = await getDatasetStore(); logger.info(`Fetching dataset ${datasetId}...`); const dataset = await datasetStore.getDataset(datasetId); + const datasetMetadatas = await datasetStore.listDatasets(); + const targetDatasetMetadata = datasetMetadatas.find( + (d) => d.datasetId === datasetId + ); + const datasetVersion = targetDatasetMetadata?.version; logger.info('Running inference...'); const evalDataset = await runInference({ runner, actionRef, - evalFlowInput: dataset, + evalFlowInput: EvalInferenceInputSchema.parse({ samples: dataset }), auth: request.options?.auth, actionConfig: request.options?.actionConfig, }); @@ -79,13 +94,8 @@ export async function runNewEvaluation( runner, evaluatorActions, evalDataset, - actionRef, - datasetId, + augments: { actionRef, datasetId, datasetVersion }, }); - logger.info('Finished evaluation, returning key...'); - const evalStore = getEvalStore(); - await evalStore.save(evalRun); - return evalRun.key; } @@ -93,7 +103,7 @@ export async function runNewEvaluation( export async function runInference(params: { runner: Runner; actionRef: string; - evalFlowInput: EvalFlowInput; + evalFlowInput: EvalInferenceInput; auth?: string; actionConfig?: any; }): Promise { @@ -117,13 +127,12 @@ export async function runEvaluation(params: { runner: Runner; evaluatorActions: Action[]; evalDataset: EvalInput[]; - actionRef?: string; - datasetId?: string; + augments?: EvalKeyAugments; }): Promise { - const { runner, evaluatorActions, evalDataset, actionRef, datasetId } = - params; + const { runner, evaluatorActions, evalDataset, augments } = params; const evalRunId = randomUUID(); const scores: Record = {}; + logger.info('Running evaluation...'); for (const action of evaluatorActions) { const name = evaluatorName(action); const response = await runner.runAction({ @@ -141,14 +150,17 @@ export async function runEvaluation(params: { const evalRun = { key: { - actionRef, evalRunId, - datasetId, createdAt: new Date().toISOString(), + ...augments, }, results: scoredResults, metricsMetadata: metadata, }; + + logger.info('Finished evaluation, writing key...'); + const evalStore = getEvalStore(); + await evalStore.save(evalRun); return evalRun; } @@ -187,31 +199,32 @@ export async function getMatchingEvaluatorActions( async function bulkRunAction(params: { runner: Runner; actionRef: string; - evalFlowInput: EvalFlowInput; + evalFlowInput: EvalInferenceInput; auth?: string; actionConfig?: any; }): Promise { const { runner, actionRef, evalFlowInput, auth, actionConfig } = params; const isModelAction = actionRef.startsWith('/model'); - let inputs: { input: string; reference?: string }[] = Array.isArray( - evalFlowInput - ) - ? evalFlowInput.map((i) => ({ input: i })) + let testCases: TestCase[] = Array.isArray(evalFlowInput) + ? (evalFlowInput as any[]).map((i) => ({ + input: i, + testCaseId: generateTestCaseId(), + })) : evalFlowInput.samples.map((c) => ({ input: c.input, reference: c.reference, + testCaseId: c.testCaseId ?? generateTestCaseId(), })); let evalInputs: EvalInput[] = []; - for (const sample of inputs) { + for (const testCase of testCases) { logger.info(`Running '${actionRef}' ...`); if (isModelAction) { evalInputs.push( await runModelAction({ runner, actionRef, - input: sample.input, - reference: sample.reference, + testCase, modelConfig: actionConfig, }) ); @@ -220,8 +233,7 @@ async function bulkRunAction(params: { await runFlowAction({ runner, actionRef, - input: sample.input, - reference: sample.reference, + testCase, auth, }) ); @@ -233,16 +245,15 @@ async function bulkRunAction(params: { async function runFlowAction(params: { runner: Runner; actionRef: string; - input: any; - reference?: any; + testCase: TestCase; auth?: any; }): Promise { - const { runner, actionRef, input, auth, reference } = { ...params }; + const { runner, actionRef, testCase, auth } = { ...params }; let state: InferenceRunState; try { const flowInput = FlowActionInputSchema.parse({ start: { - input, + input: testCase.input, }, auth: auth ? JSON.parse(auth) : undefined, }); @@ -251,16 +262,14 @@ async function runFlowAction(params: { input: flowInput, }); state = { - reference, - input, + ...testCase, traceId: runActionResponse.telemetry?.traceId, response: runActionResponse.result, }; } catch (e: any) { const traceId = e?.data?.details?.traceId; state = { - input, - reference, + ...testCase, traceId, evalError: `Error when running inference. Details: ${e?.message ?? e}`, }; @@ -271,29 +280,26 @@ async function runFlowAction(params: { async function runModelAction(params: { runner: Runner; actionRef: string; - input: any; - reference?: any; + testCase: TestCase; modelConfig?: any; }): Promise { - const { runner, actionRef, input, modelConfig, reference } = { ...params }; + const { runner, actionRef, modelConfig, testCase } = { ...params }; let state: InferenceRunState; try { - const modelInput = getModelInput(input, modelConfig); + const modelInput = getModelInput(testCase.input, modelConfig); const runActionResponse = await runner.runAction({ key: actionRef, input: modelInput, }); state = { - input, - reference, + ...testCase, traceId: runActionResponse.telemetry?.traceId, response: runActionResponse.result, }; } catch (e: any) { const traceId = e?.data?.details?.traceId; state = { - input, - reference, + ...testCase, traceId, evalError: `Error when running inference. Details: ${e?.message ?? e}`, }; @@ -335,7 +341,7 @@ async function gatherEvalInput(params: { const nestedSpan = stackTraceSpans(trace); if (!nestedSpan) { return { - testCaseId: randomUUID(), + testCaseId: state.testCaseId, input, error: `Unable to extract any spans from trace ${traceId}`, reference: state.reference, @@ -345,7 +351,7 @@ async function gatherEvalInput(params: { if (nestedSpan.attributes['genkit:state'] === 'error') { return { - testCaseId: randomUUID(), + testCaseId: state.testCaseId, input, error: getSpanErrorMessage(nestedSpan) ?? `Unknown error in trace ${traceId}`, @@ -360,7 +366,7 @@ async function gatherEvalInput(params: { return { // TODO Replace this with unified trace class - testCaseId: randomUUID(), + testCaseId: state.testCaseId, input, output, error, diff --git a/genkit-tools/common/src/eval/index.ts b/genkit-tools/common/src/eval/index.ts index 3e87c3392f..5cd4fadcb0 100644 --- a/genkit-tools/common/src/eval/index.ts +++ b/genkit-tools/common/src/eval/index.ts @@ -17,7 +17,7 @@ import { DatasetStore, EvalStore } from '../types/eval'; import { LocalFileDatasetStore } from './localFileDatasetStore'; import { LocalFileEvalStore } from './localFileEvalStore'; -export { EvalFlowInput, EvalFlowInputSchema } from '../types/eval'; +export { EvalInferenceInput, EvalInferenceInputSchema } from '../types/eval'; export * from './evaluate'; export * from './exporter'; export * from './parser'; diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts index 7977d81e1c..a0dbe25599 100644 --- a/genkit-tools/common/src/eval/localFileDatasetStore.ts +++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts @@ -23,9 +23,11 @@ import { CreateDatasetRequest, UpdateDatasetRequest } from '../types/apis'; import { Dataset, DatasetMetadata, + DatasetSchema, DatasetStore, - EvalFlowInputSchema, + EvalInferenceInput, } from '../types/eval'; +import { generateTestCaseId } from '../utils'; import { logger } from '../utils/logger'; /** @@ -68,7 +70,7 @@ export class LocalFileDatasetStore implements DatasetStore { } private async createDatasetInternal( - data: Dataset, + data: EvalInferenceInput, datasetId?: string ): Promise { const id = await this.generateDatasetId(datasetId); @@ -80,14 +82,14 @@ export class LocalFileDatasetStore implements DatasetStore { `Create dataset failed: file already exists at {$filePath}` ); } - + const dataset = this.getDatasetFromInferenceInput(data); logger.info(`Saving Dataset to ` + filePath); - await writeFile(filePath, JSON.stringify(data)); + await writeFile(filePath, JSON.stringify(dataset)); const now = new Date().toString(); const metadata = { datasetId: id, - size: Array.isArray(data) ? data.length : data.samples.length, + size: dataset.length, version: 1, createTime: now, updateTime: now, @@ -106,9 +108,6 @@ export class LocalFileDatasetStore implements DatasetStore { async updateDataset(req: UpdateDatasetRequest): Promise { const datasetId = req.datasetId; - if (!req.data) { - throw new Error('Error: `data` is required for updateDataset'); - } const filePath = path.resolve( this.storeRoot, this.generateFileName(datasetId) @@ -122,14 +121,17 @@ export class LocalFileDatasetStore implements DatasetStore { if (!prevMetadata) { throw new Error(`Update dataset failed: dataset metadata not found`); } - - logger.info(`Updating Dataset at ` + filePath); - await writeFile(filePath, JSON.stringify(req.data)); + const patch = this.getDatasetFromInferenceInput(req.data ?? []); + let newSize = prevMetadata.size; + if (patch.length > 0) { + logger.info(`Updating Dataset at ` + filePath); + newSize = await this.patchDataset(datasetId, patch, filePath); + } const now = new Date().toString(); const newMetadata = { datasetId: datasetId, - size: Array.isArray(req.data) ? req.data.length : req.data.samples.length, + size: newSize, version: prevMetadata.version + 1, createTime: prevMetadata.createTime, updateTime: now, @@ -154,9 +156,9 @@ export class LocalFileDatasetStore implements DatasetStore { if (!fs.existsSync(filePath)) { throw new Error(`Dataset not found for dataset ID {$id}`); } - return await readFile(filePath, 'utf8').then((data) => - EvalFlowInputSchema.parse(JSON.parse(data)) - ); + return await readFile(filePath, 'utf8').then((data) => { + return DatasetSchema.parse(JSON.parse(data)); + }); } async listDatasets(): Promise { @@ -236,4 +238,37 @@ export class LocalFileDatasetStore implements DatasetStore { JSON.parse(data) ); } + + private getDatasetFromInferenceInput(data: EvalInferenceInput): Dataset { + if (Array.isArray(data)) { + return data.map((d) => ({ + testCaseId: d.testCaseId ?? generateTestCaseId(), + input: d, + })); + } else if (!!data.samples) { + return data.samples.map((d) => ({ + testCaseId: d.testCaseId ?? generateTestCaseId(), + ...d, + })); + } + return []; + } + + private async patchDataset( + datasetId: string, + patch: Dataset, + filePath: string + ): Promise { + const existingDataset = await this.getDataset(datasetId); + const datasetMap = new Map(existingDataset.map((d) => [d.testCaseId, d])); + const patchMap = new Map(patch.map((d) => [d.testCaseId, d])); + + patchMap.forEach((value, key) => { + datasetMap.set(key, value); + }); + + const newDataset = Array.from(datasetMap.values()) as Dataset; + await writeFile(filePath, JSON.stringify(newDataset)); + return newDataset.length; + } } diff --git a/genkit-tools/common/src/server/router.ts b/genkit-tools/common/src/server/router.ts index 4af1a4b110..b477be924d 100644 --- a/genkit-tools/common/src/server/router.ts +++ b/genkit-tools/common/src/server/router.ts @@ -199,7 +199,7 @@ export const TOOLS_SERVER_ROUTER = (runner: Runner) => /** Retrieves an existing dataset */ getDataset: loggedProcedure .input(z.string()) - .output(evals.EvalFlowInputSchema) + .output(evals.EvalInferenceInputSchema) .query(async ({ input }) => { const response = await getDatasetStore().getDataset(input); return response; diff --git a/genkit-tools/common/src/types/apis.ts b/genkit-tools/common/src/types/apis.ts index a67444e9b5..9e280e972d 100644 --- a/genkit-tools/common/src/types/apis.ts +++ b/genkit-tools/common/src/types/apis.ts @@ -15,7 +15,7 @@ */ import { z } from 'zod'; -import { EvalFlowInputSchema, EvalRunKeySchema } from './eval'; +import { EvalInferenceInputSchema, EvalRunKeySchema } from './eval'; import { FlowStateSchema } from './flow'; import { GenerationCommonConfigSchema, @@ -133,15 +133,14 @@ export const GetEvalRunRequestSchema = z.object({ export type GetEvalRunRequest = z.infer; export const CreateDatasetRequestSchema = z.object({ - data: EvalFlowInputSchema, + data: EvalInferenceInputSchema, datasetId: z.string().optional(), }); export type CreateDatasetRequest = z.infer; export const UpdateDatasetRequestSchema = z.object({ - /** Only supports replacement, throws if data is absent */ - data: EvalFlowInputSchema.optional(), + data: EvalInferenceInputSchema.optional(), datasetId: z.string(), }); export type UpdateDatasetRequest = z.infer; diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts index 8304b75abb..0d0a0e89e8 100644 --- a/genkit-tools/common/src/types/eval.ts +++ b/genkit-tools/common/src/types/eval.ts @@ -27,37 +27,43 @@ import { */ /** - * Structured input for eval:flow + * Structured input for inference part of evaluation */ -export const EvalFlowStructuredInputSchema = z.object({ +export const EvalInferenceStructuredInputSchema = z.object({ samples: z.array( z.object({ + testCaseId: z.string().optional(), input: z.any(), reference: z.any().optional(), }) ), }); -export type EvalFlowStructuredInput = z.infer< - typeof EvalFlowStructuredInputSchema +export type EvalInferenceStructuredInput = z.infer< + typeof EvalInferenceStructuredInputSchema >; /** - * A dataset that is ready for eval:flow. + * A set of samples that is ready for inference. * - * This could be an array of input objects to the target flow, or - * It could be a JSON object as specified, with support for references. + * This should be used in user-facing surfaces (CLI/API inputs) to accommodate various user input formats. For internal wire-transfer/storage, prefer {@link Dataset}. */ -export const EvalFlowInputSchema = z.union([ +export const EvalInferenceInputSchema = z.union([ z.array(z.any()), - EvalFlowStructuredInputSchema, + EvalInferenceStructuredInputSchema, ]); -export type EvalFlowInput = z.infer; +export type EvalInferenceInput = z.infer; /** - * Alias for EvalFlowInput to be used in the DatasetStore related APIs. - * We may deprecate EvalFlowInput in favor of this in the future. + * Represents a Dataset, to be used for bulk-inference / evaluation. This is a more optionated form of EvalInferenceInput, which guarantees testCaseId for each test sample. */ -export type Dataset = z.infer; +export const DatasetSchema = z.array( + z.object({ + testCaseId: z.string(), + input: z.any(), + reference: z.any().optional(), + }) +); +export type Dataset = z.infer; /** * A record that is ready for evaluation. @@ -101,10 +107,17 @@ export type EvalResult = z.infer; export const EvalRunKeySchema = z.object({ actionRef: z.string().optional(), datasetId: z.string().optional(), + datasetVersion: z.number().optional(), evalRunId: z.string(), createdAt: z.string(), }); export type EvalRunKey = z.infer; +export const EvalKeyAugmentsSchema = EvalRunKeySchema.pick({ + datasetId: true, + datasetVersion: true, + actionRef: true, +}); +export type EvalKeyAugments = z.infer; /** * A container for the results of evaluation over a batch of test cases. diff --git a/genkit-tools/common/src/utils/eval.ts b/genkit-tools/common/src/utils/eval.ts index 8295c36df2..3bfdc0fc86 100644 --- a/genkit-tools/common/src/utils/eval.ts +++ b/genkit-tools/common/src/utils/eval.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { randomUUID } from 'crypto'; import * as inquirer from 'inquirer'; import { EvalField, @@ -209,3 +210,8 @@ export async function getEvalExtractors( } return Promise.resolve(composedExtractors); } + +/**Global function to generate testCaseId */ +export function generateTestCaseId() { + return randomUUID(); +} diff --git a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts index 953d43f3b9..fa404cc717 100644 --- a/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts +++ b/genkit-tools/common/tests/eval/localFileDatasetStore_test.ts @@ -29,38 +29,45 @@ import { CreateDatasetRequestSchema, UpdateDatasetRequestSchema, } from '../../src/types/apis'; -import { DatasetStore } from '../../src/types/eval'; +import { Dataset, DatasetStore } from '../../src/types/eval'; const FAKE_TIME = new Date('2024-02-03T12:05:33.243Z'); -const SAMPLE_DATASET_1_V1 = { - samples: [ - { - input: 'Cats are evil', - reference: 'Sorry no reference', - }, - { - input: 'Dogs are beautiful', - }, - ], -}; - -const SAMPLE_DATASET_1_V2 = { - samples: [ - { - input: 'Cats are evil', - reference: 'Sorry no reference', - }, - { - input: 'Dogs are angels', - }, - { - input: 'Dogs are also super cute', - }, - ], -}; +const SAMPLE_DATASET_1_V1 = [ + { + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + input: 'Dogs are beautiful', + }, +]; + +const SAMPLE_DATASET_1_WITH_IDS = [ + { + testCaseId: '1', + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + testCaseId: '2', + input: 'Dogs are beautiful', + }, +]; + +const SAMPLE_DATASET_1_V2 = [ + { + input: 'Cats are evil', + reference: 'Sorry no reference', + }, + { + input: 'Dogs are angels', + }, + { + input: 'Dogs are also super cute', + }, +]; -const SAMPLE_DATASET_NAME_1 = 'dataset-1'; const SAMPLE_DATASET_ID_1 = 'dataset-1-123456'; const SAMPLE_DATASET_METADATA_1_V1 = { @@ -79,16 +86,15 @@ const SAMPLE_DATASET_METADATA_1_V2 = { }; const CREATE_DATASET_REQUEST = CreateDatasetRequestSchema.parse({ - data: SAMPLE_DATASET_1_V1, + data: { samples: SAMPLE_DATASET_1_V1 }, }); const UPDATE_DATASET_REQUEST = UpdateDatasetRequestSchema.parse({ - data: SAMPLE_DATASET_1_V2, + data: { samples: SAMPLE_DATASET_1_V2 }, datasetId: SAMPLE_DATASET_ID_1, }); const SAMPLE_DATASET_ID_2 = 'dataset-2-123456'; -const SAMPLE_DATASET_NAME_2 = 'dataset-2'; const SAMPLE_DATASET_METADATA_2 = { datasetId: SAMPLE_DATASET_ID_2, @@ -108,6 +114,10 @@ jest.mock('crypto', () => { jest.mock('uuid'); const uuidSpy = jest.spyOn(uuid, 'v4'); +const TEST_CASE_ID = 'test-case-1234-1234-1234'; +jest.mock('../../src/utils', () => ({ + generateTestCaseId: jest.fn(() => TEST_CASE_ID), +})); jest.useFakeTimers({ advanceTimers: true }); jest.setSystemTime(FAKE_TIME); @@ -136,9 +146,49 @@ describe('localFileDatasetStore', () => { Promise.resolve(JSON.stringify({}) as any) ); fs.existsSync = jest.fn(() => false); + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s) => ({ + testCaseId: TEST_CASE_ID, + ...s, + })); + + const datasetMetadata = await DatasetStore.createDataset({ + ...CREATE_DATASET_REQUEST, + datasetId: SAMPLE_DATASET_ID_1, + }); + + expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 1, + expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), + JSON.stringify(dataset) + ); + const metadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, + }; + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('datasets/index.json'), + JSON.stringify(metadataMap) + ); + expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V1); + }); + + it('writes and updates index for new dataset, testCaseIds are provided', async () => { + fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); + fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); + // For index file reads + fs.promises.readFile = jest.fn(async () => + Promise.resolve(JSON.stringify({}) as any) + ); + fs.existsSync = jest.fn(() => false); + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s, i) => ({ + testCaseId: TEST_CASE_ID + `index${i}`, + ...s, + })); const datasetMetadata = await DatasetStore.createDataset({ ...CREATE_DATASET_REQUEST, + data: { samples: dataset }, datasetId: SAMPLE_DATASET_ID_1, }); @@ -146,7 +196,7 @@ describe('localFileDatasetStore', () => { expect(fs.promises.writeFile).toHaveBeenNthCalledWith( 1, expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), - JSON.stringify(CREATE_DATASET_REQUEST.data) + JSON.stringify(dataset) ); const metadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, @@ -188,7 +238,7 @@ describe('localFileDatasetStore', () => { }); describe('updateDataset', () => { - it('succeeds for existing dataset', async () => { + it('succeeds for existing dataset -- append', async () => { fs.existsSync = jest.fn(() => true); let metadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, @@ -200,16 +250,33 @@ describe('localFileDatasetStore', () => { ); fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); - - const datasetMetadata = await DatasetStore.updateDataset( - UPDATE_DATASET_REQUEST - ); + const dataset: Dataset = [...SAMPLE_DATASET_1_WITH_IDS]; + const getDatasetSpy = jest + .spyOn(DatasetStore, 'getDataset') + .mockImplementation(() => Promise.resolve(dataset)); + + const datasetMetadata = await DatasetStore.updateDataset({ + data: { + samples: [ + { + input: 'A new information on cat dog', + }, + ], + }, + datasetId: SAMPLE_DATASET_ID_1, + }); expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); expect(fs.promises.writeFile).toHaveBeenNthCalledWith( 1, expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), - JSON.stringify(SAMPLE_DATASET_1_V2) + JSON.stringify([ + ...dataset, + { + testCaseId: TEST_CASE_ID, + input: 'A new information on cat dog', + }, + ]) ); const updatedMetadataMap = { [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V2, @@ -220,21 +287,72 @@ describe('localFileDatasetStore', () => { expect.stringContaining('datasets/index.json'), JSON.stringify(updatedMetadataMap) ); + expect(getDatasetSpy).toHaveBeenCalledTimes(1); expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V2); }); - it('fails if data is not passed', async () => { + it('succeeds for existing dataset -- append and replace', async () => { fs.existsSync = jest.fn(() => true); - - expect(async () => { - await DatasetStore.updateDataset({ - datasetId: SAMPLE_DATASET_ID_1, - }); - }).rejects.toThrow( - new Error('Error: `data` is required for updateDataset') + let metadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V1, + [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, + }; + // For index file reads + fs.promises.readFile = jest.fn(async () => + Promise.resolve(JSON.stringify(metadataMap) as any) ); + fs.promises.writeFile = jest.fn(async () => Promise.resolve(undefined)); + fs.promises.appendFile = jest.fn(async () => Promise.resolve(undefined)); + const dataset: Dataset = [...SAMPLE_DATASET_1_WITH_IDS]; + const getDatasetSpy = jest + .spyOn(DatasetStore, 'getDataset') + .mockImplementation(() => Promise.resolve(dataset)); + + const datasetMetadata = await DatasetStore.updateDataset({ + data: { + samples: [ + { + input: 'A new information on cat dog', + }, + { + testCaseId: '1', + input: 'Other information on hot dog', + }, + ], + }, + datasetId: SAMPLE_DATASET_ID_1, + }); - expect(fs.promises.writeFile).toBeCalledTimes(0); + expect(fs.promises.writeFile).toHaveBeenCalledTimes(2); + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 1, + expect.stringContaining(`datasets/${SAMPLE_DATASET_ID_1}.json`), + JSON.stringify([ + { + testCaseId: '1', + input: 'Other information on hot dog', + }, + { + testCaseId: '2', + input: 'Dogs are beautiful', + }, + { + testCaseId: TEST_CASE_ID, + input: 'A new information on cat dog', + }, + ]) + ); + const updatedMetadataMap = { + [SAMPLE_DATASET_ID_1]: SAMPLE_DATASET_METADATA_1_V2, + [SAMPLE_DATASET_ID_2]: SAMPLE_DATASET_METADATA_2, + }; + expect(fs.promises.writeFile).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('datasets/index.json'), + JSON.stringify(updatedMetadataMap) + ); + expect(getDatasetSpy).toHaveBeenCalledTimes(1); + expect(datasetMetadata).toMatchObject(SAMPLE_DATASET_METADATA_1_V2); }); it('fails for non existing dataset', async () => { @@ -278,14 +396,18 @@ describe('localFileDatasetStore', () => { describe('getDataset', () => { it('succeeds for existing dataset', async () => { + const dataset: Dataset = SAMPLE_DATASET_1_V1.map((s) => ({ + ...s, + testCaseId: 'id-x', + })); fs.existsSync = jest.fn(() => true); fs.promises.readFile = jest.fn(async () => - Promise.resolve(JSON.stringify(SAMPLE_DATASET_1_V1) as any) + Promise.resolve(JSON.stringify(dataset) as any) ); const fetchedDataset = await DatasetStore.getDataset(SAMPLE_DATASET_ID_1); - expect(fetchedDataset).toMatchObject(SAMPLE_DATASET_1_V1); + expect(fetchedDataset).toMatchObject(dataset); }); it('fails for non existing dataset', async () => { diff --git a/js/ai/src/evaluator.ts b/js/ai/src/evaluator.ts index 1194e8dcc0..1e2db10848 100644 --- a/js/ai/src/evaluator.ts +++ b/js/ai/src/evaluator.ts @@ -18,6 +18,7 @@ import { Action, defineAction, z } from '@genkit-ai/core'; import { logger } from '@genkit-ai/core/logging'; import { lookupAction } from '@genkit-ai/core/registry'; import { SPAN_TYPE_ATTR, runInNewSpan } from '@genkit-ai/core/tracing'; +import { randomUUID } from 'crypto'; export const ATTR_PREFIX = 'genkit'; export const SPAN_STATE_ATTR = ATTR_PREFIX + ':state'; @@ -31,6 +32,12 @@ export const BaseDataPointSchema = z.object({ traceIds: z.array(z.string()).optional(), }); +// DataPoint that is to be used for actions. This needs testCaseId to be present. +export const BaseEvalDataPointSchema = BaseDataPointSchema.extend({ + testCaseId: z.string(), +}); +export type BaseEvalDataPoint = z.infer; + export const ScoreSchema = z.object({ score: z.union([z.number(), z.string(), z.boolean()]).optional(), // TODO: use StatusSchema @@ -56,7 +63,7 @@ export type Dataset< export const EvalResponseSchema = z.object({ sampleIndex: z.number().optional(), - testCaseId: z.string().optional(), + testCaseId: z.string(), traceId: z.string().optional(), spanId: z.string().optional(), evaluation: ScoreSchema, @@ -67,10 +74,11 @@ export const EvalResponsesSchema = z.array(EvalResponseSchema); export type EvalResponses = z.infer; type EvaluatorFn< - DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema, + EvalDataPoint extends + typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema, CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, > = ( - input: z.infer, + input: z.infer, evaluatorOptions?: z.infer ) => Promise; @@ -115,6 +123,8 @@ export interface EvaluatorParams< */ export function defineEvaluator< DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema, + EvalDataPoint extends + typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema, EvaluatorOptions extends z.ZodTypeAny = z.ZodTypeAny, >( options: { @@ -125,7 +135,7 @@ export function defineEvaluator< configSchema?: EvaluatorOptions; isBilled?: boolean; }, - runner: EvaluatorFn + runner: EvaluatorFn ) { const metadata = {}; metadata[EVALUATOR_METADATA_KEY_IS_BILLED] = @@ -149,7 +159,10 @@ export function defineEvaluator< async (i) => { let evalResponses: EvalResponses = []; for (let index = 0; index < i.dataset.length; index++) { - const datapoint = i.dataset[index]; + const datapoint: BaseEvalDataPoint = { + ...i.dataset[index], + testCaseId: i.dataset[index].testCaseId ?? randomUUID(), + }; try { await runInNewSpan( { diff --git a/js/plugins/evaluators/src/index.ts b/js/plugins/evaluators/src/index.ts index cca9e728ba..072d0e459d 100644 --- a/js/plugins/evaluators/src/index.ts +++ b/js/plugins/evaluators/src/index.ts @@ -22,7 +22,7 @@ import { z, } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, defineEvaluator, EvalResponse, evaluatorRef, @@ -88,7 +88,7 @@ function hasMetric(arr: GenkitMetric[] | undefined, metric: GenkitMetric) { return arr?.some((m) => m === metric); } -function fillScores(dataPoint: BaseDataPoint, score: Score): EvalResponse { +function fillScores(dataPoint: BaseEvalDataPoint, score: Score): EvalResponse { return { testCaseId: dataPoint.testCaseId, evaluation: score, @@ -118,7 +118,7 @@ export function genkitEvaluators< definition: 'Assesses how pertinent the generated answer is to the given prompt', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const answerRelevancy = await answerRelevancyScore( judge, datapoint, @@ -138,7 +138,7 @@ export function genkitEvaluators< definition: 'Measures the factual consistency of the generated answer against the given context', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const faithfulness = await faithfulnessScore( judge, datapoint, @@ -156,7 +156,7 @@ export function genkitEvaluators< definition: 'Measures whether the generated output intends to deceive, harm, or exploit', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const maliciousness = await maliciousnessScore( judge, datapoint, diff --git a/js/plugins/evaluators/src/metrics/answer_relevancy.ts b/js/plugins/evaluators/src/metrics/answer_relevancy.ts index 70e1797c14..ab5e02b79c 100644 --- a/js/plugins/evaluators/src/metrics/answer_relevancy.ts +++ b/js/plugins/evaluators/src/metrics/answer_relevancy.ts @@ -17,7 +17,7 @@ import similarity from 'compute-cosine-similarity'; import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; import { embed, EmbedderArgument } from 'genkit/embedder'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -32,7 +32,7 @@ export async function answerRelevancyScore< CustomEmbedderOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, embedder: EmbedderArgument, judgeConfig?: CustomModelOptions, embedderOptions?: z.infer diff --git a/js/plugins/evaluators/src/metrics/faithfulness.ts b/js/plugins/evaluators/src/metrics/faithfulness.ts index b8816164b4..fa80c6f4b7 100644 --- a/js/plugins/evaluators/src/metrics/faithfulness.ts +++ b/js/plugins/evaluators/src/metrics/faithfulness.ts @@ -15,7 +15,7 @@ */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -40,7 +40,7 @@ export async function faithfulnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { try { diff --git a/js/plugins/evaluators/src/metrics/maliciousness.ts b/js/plugins/evaluators/src/metrics/maliciousness.ts index 316e1eeda5..ad166d2b2c 100644 --- a/js/plugins/evaluators/src/metrics/maliciousness.ts +++ b/js/plugins/evaluators/src/metrics/maliciousness.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { generate, loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { getDirName } from './helper.js'; @@ -27,7 +27,7 @@ export async function maliciousnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/plugins/langchain/src/evaluators.ts b/js/plugins/langchain/src/evaluators.ts index dab2679136..b112149d20 100644 --- a/js/plugins/langchain/src/evaluators.ts +++ b/js/plugins/langchain/src/evaluators.ts @@ -15,7 +15,7 @@ */ import { ModelArgument } from 'genkit'; -import { BaseDataPointSchema, defineEvaluator } from 'genkit/evaluator'; +import { BaseEvalDataPoint, defineEvaluator } from 'genkit/evaluator'; import { Criteria, loadEvaluator } from 'langchain/evaluation'; import { genkitModel } from './model.js'; import { GenkitTracer } from './tracing.js'; @@ -31,9 +31,8 @@ export function langchainEvaluator( name: `langchain/${criteria}`, displayName: `${criteria}`, definition: `${criteria}: refer to https://js.langchain.com/docs/guides/evaluation`, - dataPointType: BaseDataPointSchema, }, - async (datapoint) => { + async (datapoint: BaseEvalDataPoint) => { try { switch (type) { case 'labeled_criteria': diff --git a/js/plugins/vertexai/src/evaluator_factory.ts b/js/plugins/vertexai/src/evaluator_factory.ts index 3cc0405a04..7fbb4f2991 100644 --- a/js/plugins/vertexai/src/evaluator_factory.ts +++ b/js/plugins/vertexai/src/evaluator_factory.ts @@ -15,7 +15,7 @@ */ import { Action, GENKIT_CLIENT_HEADER, z } from 'genkit'; -import { BaseDataPoint, Score, defineEvaluator } from 'genkit/evaluator'; +import { BaseEvalDataPoint, Score, defineEvaluator } from 'genkit/evaluator'; import { runInNewSpan } from 'genkit/tracing'; import { GoogleAuth } from 'google-auth-library'; import { VertexAIEvaluationMetricType } from './evaluation.js'; @@ -34,7 +34,7 @@ export class EvaluatorFactory { definition: string; responseSchema: ResponseType; }, - toRequest: (datapoint: BaseDataPoint) => any, + toRequest: (datapoint: BaseEvalDataPoint) => any, responseHandler: (response: z.infer) => Score ): Action { return defineEvaluator( @@ -43,7 +43,7 @@ export class EvaluatorFactory { displayName: config.displayName, definition: config.definition, }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const responseSchema = config.responseSchema; const response = await this.evaluateInstances( toRequest(datapoint), diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index f4aee93441..b065e0a212 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -28,7 +28,7 @@ importers: version: 1.9.0 '@types/node': specifier: ^20.11.19 - version: 20.11.30 + version: 20.16.9 colorette: specifier: ^2.0.20 version: 2.0.20 @@ -50,10 +50,10 @@ importers: version: 6.0.1 tsup: specifier: ^8.0.2 - version: 8.0.2(postcss@8.4.47)(typescript@4.9.5) + version: 8.3.0(postcss@8.4.47)(tsx@4.19.1)(typescript@4.9.5) tsx: specifier: ^4.7.1 - version: 4.7.1 + version: 4.19.1 typescript: specifier: ^4.9.0 version: 4.9.5 @@ -1563,138 +1563,282 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.23.1': + resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.19.12': resolution: {integrity: sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==} engines: {node: '>=12'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.23.1': + resolution: {integrity: sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.19.12': resolution: {integrity: sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==} engines: {node: '>=12'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.23.1': + resolution: {integrity: sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.19.12': resolution: {integrity: sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==} engines: {node: '>=12'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.23.1': + resolution: {integrity: sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.19.12': resolution: {integrity: sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==} engines: {node: '>=12'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.23.1': + resolution: {integrity: sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.19.12': resolution: {integrity: sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==} engines: {node: '>=12'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.23.1': + resolution: {integrity: sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.19.12': resolution: {integrity: sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==} engines: {node: '>=12'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.23.1': + resolution: {integrity: sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.19.12': resolution: {integrity: sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==} engines: {node: '>=12'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.23.1': + resolution: {integrity: sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.19.12': resolution: {integrity: sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==} engines: {node: '>=12'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.23.1': + resolution: {integrity: sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.19.12': resolution: {integrity: sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==} engines: {node: '>=12'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.23.1': + resolution: {integrity: sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.19.12': resolution: {integrity: sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==} engines: {node: '>=12'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.23.1': + resolution: {integrity: sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.19.12': resolution: {integrity: sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==} engines: {node: '>=12'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.23.1': + resolution: {integrity: sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.19.12': resolution: {integrity: sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==} engines: {node: '>=12'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.23.1': + resolution: {integrity: sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.19.12': resolution: {integrity: sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==} engines: {node: '>=12'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.23.1': + resolution: {integrity: sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.19.12': resolution: {integrity: sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==} engines: {node: '>=12'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.23.1': + resolution: {integrity: sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.19.12': resolution: {integrity: sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==} engines: {node: '>=12'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.23.1': + resolution: {integrity: sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.19.12': resolution: {integrity: sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==} engines: {node: '>=12'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.23.1': + resolution: {integrity: sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-x64@0.19.12': resolution: {integrity: sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==} engines: {node: '>=12'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.23.1': + resolution: {integrity: sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.23.1': + resolution: {integrity: sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.19.12': resolution: {integrity: sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==} engines: {node: '>=12'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.23.1': + resolution: {integrity: sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/sunos-x64@0.19.12': resolution: {integrity: sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==} engines: {node: '>=12'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.23.1': + resolution: {integrity: sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.19.12': resolution: {integrity: sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==} engines: {node: '>=12'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.23.1': + resolution: {integrity: sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.19.12': resolution: {integrity: sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==} engines: {node: '>=12'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.23.1': + resolution: {integrity: sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.19.12': resolution: {integrity: sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==} engines: {node: '>=12'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.23.1': + resolution: {integrity: sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} engines: {node: '>=14'} @@ -2276,12 +2420,24 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/context-async-hooks@1.26.0': + resolution: {integrity: sha512-HedpXXYzzbaoutw6DFLWLDket2FwLkLpil4hGCZ1xYEIMTcivdfwEOISgdbLEWyG3HW52gTq2V9mOVJrONgiwg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@1.25.1': resolution: {integrity: sha512-GeT/l6rBYWVQ4XArluLVB6WWQ8flHbdb6r2FCHC3smtdOAbrJBIv35tpV/yp9bmYUJf+xmZpu9DRTIeJVhFbEQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@1.26.0': + resolution: {integrity: sha512-1iKxXXE8415Cdv0yjG3G6hQnB5eVEsJce3QaawX8SjDn0mAS0ZM8fAbZZJD4ajvhC15cePvosSCut404KrIIvQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/exporter-trace-otlp-grpc@0.52.1': resolution: {integrity: sha512-pVkSH20crBwMTqB3nIN4jpQKUEoB0Z94drIHpYyEqs7UBr+I0cpYyOR3bqjA/UasQUMROb3GX8ZX4/9cVRqGBQ==} engines: {node: '>=14'} @@ -2628,6 +2784,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/resources@1.26.0': + resolution: {integrity: sha512-CPNYchBE7MBecCSVy0HKpUISEeJOniWqcHaAHpmasZ3j9o6V3AyBzhRc90jdmemq0HOxDr6ylhUbDhBqqPpeNw==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-logs@0.52.1': resolution: {integrity: sha512-MBYh+WcPPsN8YpRHRmK1Hsca9pVlyyKd4BxOC4SsgHACnl/bPp4Cri9hWhVm5+2tiQ9Zf4qSc1Jshw9tOLGWQA==} engines: {node: '>=14'} @@ -2640,6 +2802,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-metrics@1.26.0': + resolution: {integrity: sha512-0SvDXmou/JjzSDOjUmetAAvcKQW6ZrvosU0rkbDGpXvvZN+pQF6JbK/Kd4hNdK4q/22yeruqvukXEJyySTzyTQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-node@0.52.1': resolution: {integrity: sha512-uEG+gtEr6eKd8CVWeKMhH2olcCHM9dEK68pe0qE0be32BcCRsvYURhHaD1Srngh1SQcnQzZ4TP324euxqtBOJA==} engines: {node: '>=14'} @@ -2652,6 +2820,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-trace-base@1.26.0': + resolution: {integrity: sha512-olWQldtvbK4v22ymrKLbIcBi9L2SpMO84sCPY54IVsJhP9fRsxJT194C/AVaAuJzLE30EdhhM1VmvVYR7az+cw==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-trace-node@1.25.1': resolution: {integrity: sha512-nMcjFIKxnFqoez4gUmihdBrbpsEnAX/Xj16sGvZm+guceYE0NE00vLhpDVK6f3q8Q4VFI5xG8JjlXKMB/SkTTQ==} engines: {node: '>=14'} @@ -2670,6 +2844,10 @@ packages: resolution: {integrity: sha512-U9PJlOswJPSgQVPI+XEuNLElyFWkb0hAiMg+DExD9V0St03X2lPHGMdxMY/LrVmoukuIpXJ12oyrOtEZ4uXFkw==} engines: {node: '>=14'} + '@opentelemetry/semantic-conventions@1.27.0': + resolution: {integrity: sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==} + engines: {node: '>=14'} + '@opentelemetry/sql-common@0.40.1': resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==} engines: {node: '>=14'} @@ -2719,76 +2897,156 @@ packages: cpu: [arm] os: [android] + '@rollup/rollup-android-arm-eabi@4.22.4': + resolution: {integrity: sha512-Fxamp4aEZnfPOcGA8KSNEohV8hX7zVHOemC8jVBoBUHu5zpJK/Eu3uJwt6BMgy9fkvzxDaurgj96F/NiLukF2w==} + cpu: [arm] + os: [android] + '@rollup/rollup-android-arm64@4.13.2': resolution: {integrity: sha512-GdxxXbAuM7Y/YQM9/TwwP+L0omeE/lJAR1J+olu36c3LqqZEBdsIWeQ91KBe6nxwOnb06Xh7JS2U5ooWU5/LgQ==} cpu: [arm64] os: [android] + '@rollup/rollup-android-arm64@4.22.4': + resolution: {integrity: sha512-VXoK5UMrgECLYaMuGuVTOx5kcuap1Jm8g/M83RnCHBKOqvPPmROFJGQaZhGccnsFtfXQ3XYa4/jMCJvZnbJBdA==} + cpu: [arm64] + os: [android] + '@rollup/rollup-darwin-arm64@4.13.2': resolution: {integrity: sha512-mCMlpzlBgOTdaFs83I4XRr8wNPveJiJX1RLfv4hggyIVhfB5mJfN4P8Z6yKh+oE4Luz+qq1P3kVdWrCKcMYrrA==} cpu: [arm64] os: [darwin] + '@rollup/rollup-darwin-arm64@4.22.4': + resolution: {integrity: sha512-xMM9ORBqu81jyMKCDP+SZDhnX2QEVQzTcC6G18KlTQEzWK8r/oNZtKuZaCcHhnsa6fEeOBionoyl5JsAbE/36Q==} + cpu: [arm64] + os: [darwin] + '@rollup/rollup-darwin-x64@4.13.2': resolution: {integrity: sha512-yUoEvnH0FBef/NbB1u6d3HNGyruAKnN74LrPAfDQL3O32e3k3OSfLrPgSJmgb3PJrBZWfPyt6m4ZhAFa2nZp2A==} cpu: [x64] os: [darwin] + '@rollup/rollup-darwin-x64@4.22.4': + resolution: {integrity: sha512-aJJyYKQwbHuhTUrjWjxEvGnNNBCnmpHDvrb8JFDbeSH3m2XdHcxDd3jthAzvmoI8w/kSjd2y0udT+4okADsZIw==} + cpu: [x64] + os: [darwin] + '@rollup/rollup-linux-arm-gnueabihf@4.13.2': resolution: {integrity: sha512-GYbLs5ErswU/Xs7aGXqzc3RrdEjKdmoCrgzhJWyFL0r5fL3qd1NPcDKDowDnmcoSiGJeU68/Vy+OMUluRxPiLQ==} cpu: [arm] os: [linux] + '@rollup/rollup-linux-arm-gnueabihf@4.22.4': + resolution: {integrity: sha512-j63YtCIRAzbO+gC2L9dWXRh5BFetsv0j0va0Wi9epXDgU/XUi5dJKo4USTttVyK7fGw2nPWK0PbAvyliz50SCQ==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.22.4': + resolution: {integrity: sha512-dJnWUgwWBX1YBRsuKKMOlXCzh2Wu1mlHzv20TpqEsfdZLb3WoJW2kIEsGwLkroYf24IrPAvOT/ZQ2OYMV6vlrg==} + cpu: [arm] + os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.13.2': resolution: {integrity: sha512-L1+D8/wqGnKQIlh4Zre9i4R4b4noxzH5DDciyahX4oOz62CphY7WDWqJoQ66zNR4oScLNOqQJfNSIAe/6TPUmQ==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.22.4': + resolution: {integrity: sha512-AdPRoNi3NKVLolCN/Sp4F4N1d98c4SBnHMKoLuiG6RXgoZ4sllseuGioszumnPGmPM2O7qaAX/IJdeDU8f26Aw==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-arm64-musl@4.13.2': resolution: {integrity: sha512-tK5eoKFkXdz6vjfkSTCupUzCo40xueTOiOO6PeEIadlNBkadH1wNOH8ILCPIl8by/Gmb5AGAeQOFeLev7iZDOA==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-musl@4.22.4': + resolution: {integrity: sha512-Gl0AxBtDg8uoAn5CCqQDMqAx22Wx22pjDOjBdmG0VIWX3qUBHzYmOKh8KXHL4UpogfJ14G4wk16EQogF+v8hmA==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-powerpc64le-gnu@4.13.2': resolution: {integrity: sha512-zvXvAUGGEYi6tYhcDmb9wlOckVbuD+7z3mzInCSTACJ4DQrdSLPNUeDIcAQW39M3q6PDquqLWu7pnO39uSMRzQ==} cpu: [ppc64le] os: [linux] + '@rollup/rollup-linux-powerpc64le-gnu@4.22.4': + resolution: {integrity: sha512-3aVCK9xfWW1oGQpTsYJJPF6bfpWfhbRnhdlyhak2ZiyFLDaayz0EP5j9V1RVLAAxlmWKTDfS9wyRyY3hvhPoOg==} + cpu: [ppc64] + os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.13.2': resolution: {integrity: sha512-C3GSKvMtdudHCN5HdmAMSRYR2kkhgdOfye4w0xzyii7lebVr4riCgmM6lRiSCnJn2w1Xz7ZZzHKuLrjx5620kw==} cpu: [riscv64] os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.22.4': + resolution: {integrity: sha512-ePYIir6VYnhgv2C5Xe9u+ico4t8sZWXschR6fMgoPUK31yQu7hTEJb7bCqivHECwIClJfKgE7zYsh1qTP3WHUA==} + cpu: [riscv64] + os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.13.2': resolution: {integrity: sha512-l4U0KDFwzD36j7HdfJ5/TveEQ1fUTjFFQP5qIt9gBqBgu1G8/kCaq5Ok05kd5TG9F8Lltf3MoYsUMw3rNlJ0Yg==} cpu: [s390x] os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.22.4': + resolution: {integrity: sha512-GqFJ9wLlbB9daxhVlrTe61vJtEY99/xB3C8e4ULVsVfflcpmR6c8UZXjtkMA6FhNONhj2eA5Tk9uAVw5orEs4Q==} + cpu: [s390x] + os: [linux] + '@rollup/rollup-linux-x64-gnu@4.13.2': resolution: {integrity: sha512-xXMLUAMzrtsvh3cZ448vbXqlUa7ZL8z0MwHp63K2IIID2+DeP5iWIT6g1SN7hg1VxPzqx0xZdiDM9l4n9LRU1A==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-gnu@4.22.4': + resolution: {integrity: sha512-87v0ol2sH9GE3cLQLNEy0K/R0pz1nvg76o8M5nhMR0+Q+BBGLnb35P0fVz4CQxHYXaAOhE8HhlkaZfsdUOlHwg==} + cpu: [x64] + os: [linux] + '@rollup/rollup-linux-x64-musl@4.13.2': resolution: {integrity: sha512-M/JYAWickafUijWPai4ehrjzVPKRCyDb1SLuO+ZyPfoXgeCEAlgPkNXewFZx0zcnoIe3ay4UjXIMdXQXOZXWqA==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-musl@4.22.4': + resolution: {integrity: sha512-UV6FZMUgePDZrFjrNGIWzDo/vABebuXBhJEqrHxrGiU6HikPy0Z3LfdtciIttEUQfuDdCn8fqh7wiFJjCNwO+g==} + cpu: [x64] + os: [linux] + '@rollup/rollup-win32-arm64-msvc@4.13.2': resolution: {integrity: sha512-2YWwoVg9KRkIKaXSh0mz3NmfurpmYoBBTAXA9qt7VXk0Xy12PoOP40EFuau+ajgALbbhi4uTj3tSG3tVseCjuA==} cpu: [arm64] os: [win32] + '@rollup/rollup-win32-arm64-msvc@4.22.4': + resolution: {integrity: sha512-BjI+NVVEGAXjGWYHz/vv0pBqfGoUH0IGZ0cICTn7kB9PyjrATSkX+8WkguNjWoj2qSr1im/+tTGRaY+4/PdcQw==} + cpu: [arm64] + os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.13.2': resolution: {integrity: sha512-2FSsE9aQ6OWD20E498NYKEQLneShWes0NGMPQwxWOdws35qQXH+FplabOSP5zEe1pVjurSDOGEVCE2agFwSEsw==} cpu: [ia32] os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.22.4': + resolution: {integrity: sha512-SiWG/1TuUdPvYmzmYnmd3IEifzR61Tragkbx9D3+R8mzQqDBz8v+BvZNDlkiTtI9T15KYZhP0ehn3Dld4n9J5g==} + cpu: [ia32] + os: [win32] + '@rollup/rollup-win32-x64-msvc@4.13.2': resolution: {integrity: sha512-7h7J2nokcdPePdKykd8wtc8QqqkqxIrUz7MHj6aNr8waBRU//NLDVnNjQnqQO6fqtjrtCdftpbTuOKAyrAQETQ==} cpu: [x64] os: [win32] + '@rollup/rollup-win32-x64-msvc@4.22.4': + resolution: {integrity: sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q==} + cpu: [x64] + os: [win32] + '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} @@ -2889,14 +3147,17 @@ packages: '@types/node-fetch@2.6.11': resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} - '@types/node@18.19.31': - resolution: {integrity: sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==} + '@types/node@18.19.53': + resolution: {integrity: sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==} '@types/node@20.11.30': resolution: {integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw==} - '@types/node@22.4.1': - resolution: {integrity: sha512-1tbpb9325+gPnKK0dMm+/LMriX0vKxf6RnB0SZUqfyVkQ4fMgUSySqhxE/y8Jvs4NyF1yHzTfG9KlnkIODxPKg==} + '@types/node@20.16.9': + resolution: {integrity: sha512-rkvIVJxsOfBejxK7I0FO5sa2WxFmJCzoDwcd88+fq/CUfynNywTo/1/T6hyFz22CyztsnLS9nVlHOnTI36RH5w==} + + '@types/node@22.7.2': + resolution: {integrity: sha512-866lXSrpGpgyHBZUa2m9YNWqHDjjM0aBTJlNtYaGEw4rqY/dcD7deRVTbBBAJelfA7oaGDbNftXF/TL/A6RgoA==} '@types/pdf-parse@1.1.4': resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==} @@ -2993,6 +3254,9 @@ packages: ajv@8.12.0: resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -3172,6 +3436,12 @@ packages: peerDependencies: esbuild: '>=0.17' + bundle-require@5.0.0: + resolution: {integrity: sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + peerDependencies: + esbuild: '>=0.18' + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -3319,6 +3589,10 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + consola@3.2.3: + resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==} + engines: {node: ^14.18.0 || >=16.10.0} + console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} @@ -3404,6 +3678,15 @@ packages: supports-color: optional: true + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decamelize@1.2.0: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} @@ -3568,6 +3851,11 @@ packages: engines: {node: '>=12'} hasBin: true + esbuild@0.23.1: + resolution: {integrity: sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -3658,6 +3946,9 @@ packages: fast-text-encoding@1.0.6: resolution: {integrity: sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==} + fast-uri@3.0.1: + resolution: {integrity: sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==} + fast-xml-parser@4.3.6: resolution: {integrity: sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==} hasBin: true @@ -3672,6 +3963,14 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + fdir@6.3.0: + resolution: {integrity: sha512-QOnuT+BOtivR77wYvCWHfGt9s4Pz1VIMbD463vegT5MLqNXy8rYFT/lPVEqf/bhYeT6qmqrNHhsX+rWwe3rOCQ==} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} @@ -3855,6 +4154,9 @@ packages: get-tsconfig@4.7.3: resolution: {integrity: sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==} + get-tsconfig@4.8.1: + resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -4590,6 +4892,10 @@ packages: resolution: {integrity: sha512-O18pf7nyvHTckunPWCV1XUNXU1piu01y2b7ATJ0ppkUkk8ocqVWBrYjJBCwHDjD/ZWcfyrA0P4gKhzWGi5EINQ==} engines: {node: '>=14'} + lilconfig@3.1.2: + resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==} + engines: {node: '>=14'} + limiter@1.1.5: resolution: {integrity: sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==} @@ -5106,6 +5412,10 @@ packages: resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} engines: {node: '>=8.6'} + picomatch@4.0.2: + resolution: {integrity: sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==} + engines: {node: '>=12'} + pidtree@0.3.1: resolution: {integrity: sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==} engines: {node: '>=0.10'} @@ -5139,6 +5449,24 @@ packages: ts-node: optional: true + postcss-load-config@6.0.1: + resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} + engines: {node: '>= 18'} + peerDependencies: + jiti: '>=1.21.0' + postcss: '>=8.0.9' + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + jiti: + optional: true + postcss: + optional: true + tsx: + optional: true + yaml: + optional: true + postcss@8.4.47: resolution: {integrity: sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==} engines: {node: ^10 || ^12 || >=14} @@ -5337,6 +5665,11 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rollup@4.22.4: + resolution: {integrity: sha512-vD8HJ5raRcWOyymsR6Z3o6+RzfEPCnVLMFJ6vRslO1jt4LO6dUo5Qnpg7y4RkZFM2DMe3WUirkI5c16onjrc6A==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -5619,6 +5952,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + tinyglobby@0.2.6: + resolution: {integrity: sha512-NbBoFBpqfcgd1tCiO8Lkfdk+xrA7mlLR9zgvZcZWQQwU63XAfUePyd6wZBaU93Hqw347lHnwFzttAkemHzzz4g==} + engines: {node: '>=12.0.0'} + tmpl@1.0.5: resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} @@ -5701,6 +6038,30 @@ packages: typescript: optional: true + tsup@8.3.0: + resolution: {integrity: sha512-ALscEeyS03IomcuNdFdc0YWGVIkwH1Ws7nfTbAPuoILvEV2hpGQAY72LIOjglGo4ShWpZfpBqP/jpQVCzqYQag==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + + tsx@4.19.1: + resolution: {integrity: sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==} + engines: {node: '>=18.0.0'} + hasBin: true + tsx@4.7.1: resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==} engines: {node: '>=18.0.0'} @@ -5948,6 +6309,11 @@ packages: peerDependencies: zod: ^3.22.4 + zod-to-json-schema@3.23.3: + resolution: {integrity: sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==} + peerDependencies: + zod: ^3.23.3 + zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} @@ -5963,7 +6329,7 @@ snapshots: '@anthropic-ai/sdk@0.24.3(encoding@0.1.13)': dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -5976,7 +6342,7 @@ snapshots: '@anthropic-ai/sdk@0.9.1(encoding@0.1.13)': dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -6208,72 +6574,144 @@ snapshots: '@esbuild/aix-ppc64@0.19.12': optional: true + '@esbuild/aix-ppc64@0.23.1': + optional: true + '@esbuild/android-arm64@0.19.12': optional: true + '@esbuild/android-arm64@0.23.1': + optional: true + '@esbuild/android-arm@0.19.12': optional: true + '@esbuild/android-arm@0.23.1': + optional: true + '@esbuild/android-x64@0.19.12': optional: true + '@esbuild/android-x64@0.23.1': + optional: true + '@esbuild/darwin-arm64@0.19.12': optional: true + '@esbuild/darwin-arm64@0.23.1': + optional: true + '@esbuild/darwin-x64@0.19.12': optional: true + '@esbuild/darwin-x64@0.23.1': + optional: true + '@esbuild/freebsd-arm64@0.19.12': optional: true + '@esbuild/freebsd-arm64@0.23.1': + optional: true + '@esbuild/freebsd-x64@0.19.12': optional: true + '@esbuild/freebsd-x64@0.23.1': + optional: true + '@esbuild/linux-arm64@0.19.12': optional: true + '@esbuild/linux-arm64@0.23.1': + optional: true + '@esbuild/linux-arm@0.19.12': optional: true + '@esbuild/linux-arm@0.23.1': + optional: true + '@esbuild/linux-ia32@0.19.12': optional: true + '@esbuild/linux-ia32@0.23.1': + optional: true + '@esbuild/linux-loong64@0.19.12': optional: true + '@esbuild/linux-loong64@0.23.1': + optional: true + '@esbuild/linux-mips64el@0.19.12': optional: true + '@esbuild/linux-mips64el@0.23.1': + optional: true + '@esbuild/linux-ppc64@0.19.12': optional: true + '@esbuild/linux-ppc64@0.23.1': + optional: true + '@esbuild/linux-riscv64@0.19.12': optional: true + '@esbuild/linux-riscv64@0.23.1': + optional: true + '@esbuild/linux-s390x@0.19.12': optional: true + '@esbuild/linux-s390x@0.23.1': + optional: true + '@esbuild/linux-x64@0.19.12': optional: true + '@esbuild/linux-x64@0.23.1': + optional: true + '@esbuild/netbsd-x64@0.19.12': optional: true + '@esbuild/netbsd-x64@0.23.1': + optional: true + + '@esbuild/openbsd-arm64@0.23.1': + optional: true + '@esbuild/openbsd-x64@0.19.12': optional: true + '@esbuild/openbsd-x64@0.23.1': + optional: true + '@esbuild/sunos-x64@0.19.12': optional: true + '@esbuild/sunos-x64@0.23.1': + optional: true + '@esbuild/win32-arm64@0.19.12': optional: true + '@esbuild/win32-arm64@0.23.1': + optional: true + '@esbuild/win32-ia32@0.19.12': optional: true + '@esbuild/win32-ia32@0.23.1': + optional: true + '@esbuild/win32-x64@0.19.12': optional: true + '@esbuild/win32-x64@0.23.1': + optional: true + '@fastify/busboy@2.1.1': {} '@fastify/busboy@3.0.0': {} @@ -6325,7 +6763,7 @@ snapshots: dependencies: '@genkit-ai/core': 0.6.0-dev.2 '@opentelemetry/api': 1.9.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 colorette: 2.0.20 json5: 2.2.3 node-fetch: 3.3.2 @@ -6336,13 +6774,13 @@ snapshots: '@genkit-ai/core@0.6.0-dev.2': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) + '@opentelemetry/context-async-hooks': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.26.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': 0.52.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - ajv: 8.12.0 - ajv-formats: 3.0.1(ajv@8.12.0) + '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) async-mutex: 0.5.0 body-parser: 1.20.3 cors: 2.8.5 @@ -6350,7 +6788,7 @@ snapshots: get-port: 5.1.0 json-schema: 0.4.0 zod: 3.23.8 - zod-to-json-schema: 3.22.5(zod@3.23.8) + zod-to-json-schema: 3.23.3(zod@3.23.8) transitivePeerDependencies: - supports-color @@ -6908,11 +7346,20 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 + '@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/exporter-trace-otlp-grpc@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.10.10 @@ -7386,6 +7833,12 @@ snapshots: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/resources@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/sdk-logs@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -7400,6 +7853,12 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) lodash.merge: 4.6.2 + '@opentelemetry/sdk-metrics@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-node@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -7426,6 +7885,13 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + '@opentelemetry/sdk-trace-node@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -7442,6 +7908,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.26.0': {} + '@opentelemetry/semantic-conventions@1.27.0': {} + '@opentelemetry/sql-common@0.40.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -7483,48 +7951,96 @@ snapshots: '@rollup/rollup-android-arm-eabi@4.13.2': optional: true + '@rollup/rollup-android-arm-eabi@4.22.4': + optional: true + '@rollup/rollup-android-arm64@4.13.2': optional: true + '@rollup/rollup-android-arm64@4.22.4': + optional: true + '@rollup/rollup-darwin-arm64@4.13.2': optional: true + '@rollup/rollup-darwin-arm64@4.22.4': + optional: true + '@rollup/rollup-darwin-x64@4.13.2': optional: true + '@rollup/rollup-darwin-x64@4.22.4': + optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.13.2': optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.22.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.22.4': + optional: true + '@rollup/rollup-linux-arm64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-arm64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-arm64-musl@4.13.2': optional: true + '@rollup/rollup-linux-arm64-musl@4.22.4': + optional: true + '@rollup/rollup-linux-powerpc64le-gnu@4.13.2': optional: true + '@rollup/rollup-linux-powerpc64le-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-riscv64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-riscv64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-s390x-gnu@4.13.2': optional: true + '@rollup/rollup-linux-s390x-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-x64-gnu@4.13.2': optional: true + '@rollup/rollup-linux-x64-gnu@4.22.4': + optional: true + '@rollup/rollup-linux-x64-musl@4.13.2': optional: true + '@rollup/rollup-linux-x64-musl@4.22.4': + optional: true + '@rollup/rollup-win32-arm64-msvc@4.13.2': optional: true + '@rollup/rollup-win32-arm64-msvc@4.22.4': + optional: true + '@rollup/rollup-win32-ia32-msvc@4.13.2': optional: true + '@rollup/rollup-win32-ia32-msvc@4.22.4': + optional: true + '@rollup/rollup-win32-x64-msvc@4.13.2': optional: true + '@rollup/rollup-win32-x64-msvc@4.22.4': + optional: true + '@sinclair/typebox@0.27.8': {} '@sinclair/typebox@0.29.6': {} @@ -7565,31 +8081,31 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/bunyan@1.8.9': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/caseless@0.12.5': {} '@types/connect@3.4.36': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/connect@3.4.38': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/cors@2.8.17': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/estree@1.0.5': {} '@types/express-serve-static-core@4.17.43': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/qs': 6.9.14 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -7630,13 +8146,13 @@ snapshots: '@types/jsonwebtoken@9.0.6': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/long@4.0.2': {} '@types/memcached@2.2.10': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/mime@1.3.5': {} @@ -7644,14 +8160,14 @@ snapshots: '@types/mysql@2.15.22': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/node-fetch@2.6.11': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 form-data: 4.0.0 - '@types/node@18.19.31': + '@types/node@18.19.53': dependencies: undici-types: 5.26.5 @@ -7659,7 +8175,11 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@22.4.1': + '@types/node@20.16.9': + dependencies: + undici-types: 6.19.8 + + '@types/node@22.7.2': dependencies: undici-types: 6.19.8 @@ -7671,7 +8191,7 @@ snapshots: '@types/pg@8.6.1': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 pg-protocol: 1.6.0 pg-types: 2.2.0 @@ -7682,7 +8202,7 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 @@ -7691,13 +8211,13 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/serve-static@1.15.5': dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/shimmer@1.0.5': {} @@ -7705,7 +8225,7 @@ snapshots: '@types/tedious@4.0.14': dependencies: - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/tough-cookie@4.0.5': {} @@ -7757,6 +8277,10 @@ snapshots: optionalDependencies: ajv: 8.12.0 + ajv-formats@3.0.1(ajv@8.17.1): + optionalDependencies: + ajv: 8.17.1 + ajv@8.12.0: dependencies: fast-deep-equal: 3.1.3 @@ -7764,6 +8288,13 @@ snapshots: require-from-string: 2.0.2 uri-js: 4.4.1 + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.0.1 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -7996,6 +8527,11 @@ snapshots: esbuild: 0.19.12 load-tsconfig: 0.2.5 + bundle-require@5.0.0(esbuild@0.23.1): + dependencies: + esbuild: 0.23.1 + load-tsconfig: 0.2.5 + bytes@3.1.2: {} cac@6.7.14: {} @@ -8142,6 +8678,8 @@ snapshots: concat-map@0.0.1: {} + consola@3.2.3: {} + console-control-strings@1.1.0: optional: true @@ -8231,6 +8769,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.3.7: + dependencies: + ms: 2.1.3 + decamelize@1.2.0: {} decompress-response@4.2.1: @@ -8436,6 +8978,33 @@ snapshots: '@esbuild/win32-ia32': 0.19.12 '@esbuild/win32-x64': 0.19.12 + esbuild@0.23.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.23.1 + '@esbuild/android-arm': 0.23.1 + '@esbuild/android-arm64': 0.23.1 + '@esbuild/android-x64': 0.23.1 + '@esbuild/darwin-arm64': 0.23.1 + '@esbuild/darwin-x64': 0.23.1 + '@esbuild/freebsd-arm64': 0.23.1 + '@esbuild/freebsd-x64': 0.23.1 + '@esbuild/linux-arm': 0.23.1 + '@esbuild/linux-arm64': 0.23.1 + '@esbuild/linux-ia32': 0.23.1 + '@esbuild/linux-loong64': 0.23.1 + '@esbuild/linux-mips64el': 0.23.1 + '@esbuild/linux-ppc64': 0.23.1 + '@esbuild/linux-riscv64': 0.23.1 + '@esbuild/linux-s390x': 0.23.1 + '@esbuild/linux-x64': 0.23.1 + '@esbuild/netbsd-x64': 0.23.1 + '@esbuild/openbsd-arm64': 0.23.1 + '@esbuild/openbsd-x64': 0.23.1 + '@esbuild/sunos-x64': 0.23.1 + '@esbuild/win32-arm64': 0.23.1 + '@esbuild/win32-ia32': 0.23.1 + '@esbuild/win32-x64': 0.23.1 + escalade@3.1.2: {} escalade@3.2.0: {} @@ -8580,6 +9149,8 @@ snapshots: fast-text-encoding@1.0.6: optional: true + fast-uri@3.0.1: {} + fast-xml-parser@4.3.6: dependencies: strnum: 1.0.5 @@ -8597,6 +9168,10 @@ snapshots: dependencies: bser: 2.1.1 + fdir@6.3.0(picomatch@4.0.2): + optionalDependencies: + picomatch: 4.0.2 + fecha@4.2.3: {} fetch-blob@3.2.0: @@ -8650,7 +9225,7 @@ snapshots: '@fastify/busboy': 2.1.1 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 20.11.30 + '@types/node': 20.16.9 farmhash: 3.3.1 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -8669,7 +9244,7 @@ snapshots: '@fastify/busboy': 2.1.1 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 20.11.30 + '@types/node': 20.16.9 farmhash-modern: 1.1.0 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -8688,7 +9263,7 @@ snapshots: '@fastify/busboy': 3.0.0 '@firebase/database-compat': 1.0.4 '@firebase/database-types': 1.0.2 - '@types/node': 22.4.1 + '@types/node': 22.7.2 farmhash-modern: 1.1.0 jsonwebtoken: 9.0.2 jwks-rsa: 3.1.0 @@ -8877,6 +9452,10 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + get-tsconfig@4.8.1: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -9844,6 +10423,8 @@ snapshots: lilconfig@3.1.1: {} + lilconfig@3.1.2: {} + limiter@1.1.5: {} lines-and-columns@1.2.4: {} @@ -10181,7 +10762,7 @@ snapshots: openai@4.53.0(encoding@0.1.13): dependencies: - '@types/node': 18.19.31 + '@types/node': 18.19.53 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -10322,6 +10903,8 @@ snapshots: picomatch@2.3.1: {} + picomatch@4.0.2: {} + pidtree@0.3.1: {} pify@3.0.0: {} @@ -10341,6 +10924,13 @@ snapshots: optionalDependencies: postcss: 8.4.47 + postcss-load-config@6.0.1(postcss@8.4.47)(tsx@4.19.1): + dependencies: + lilconfig: 3.1.2 + optionalDependencies: + postcss: 8.4.47 + tsx: 4.19.1 + postcss@8.4.47: dependencies: nanoid: 3.3.7 @@ -10417,7 +11007,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 long: 5.2.3 protobufjs@7.3.2: @@ -10432,7 +11022,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 long: 5.2.3 proxy-addr@2.0.7: @@ -10584,6 +11174,28 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.13.2 fsevents: 2.3.3 + rollup@4.22.4: + dependencies: + '@types/estree': 1.0.5 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.22.4 + '@rollup/rollup-android-arm64': 4.22.4 + '@rollup/rollup-darwin-arm64': 4.22.4 + '@rollup/rollup-darwin-x64': 4.22.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.22.4 + '@rollup/rollup-linux-arm-musleabihf': 4.22.4 + '@rollup/rollup-linux-arm64-gnu': 4.22.4 + '@rollup/rollup-linux-arm64-musl': 4.22.4 + '@rollup/rollup-linux-powerpc64le-gnu': 4.22.4 + '@rollup/rollup-linux-riscv64-gnu': 4.22.4 + '@rollup/rollup-linux-s390x-gnu': 4.22.4 + '@rollup/rollup-linux-x64-gnu': 4.22.4 + '@rollup/rollup-linux-x64-musl': 4.22.4 + '@rollup/rollup-win32-arm64-msvc': 4.22.4 + '@rollup/rollup-win32-ia32-msvc': 4.22.4 + '@rollup/rollup-win32-x64-msvc': 4.22.4 + fsevents: 2.3.3 + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -10932,6 +11544,11 @@ snapshots: dependencies: any-promise: 1.3.0 + tinyglobby@0.2.6: + dependencies: + fdir: 6.3.0(picomatch@4.0.2) + picomatch: 4.0.2 + tmpl@1.0.5: {} to-fast-properties@2.0.0: {} @@ -11000,6 +11617,40 @@ snapshots: - supports-color - ts-node + tsup@8.3.0(postcss@8.4.47)(tsx@4.19.1)(typescript@4.9.5): + dependencies: + bundle-require: 5.0.0(esbuild@0.23.1) + cac: 6.7.14 + chokidar: 3.6.0 + consola: 3.2.3 + debug: 4.3.7 + esbuild: 0.23.1 + execa: 5.1.1 + joycon: 3.1.1 + picocolors: 1.1.0 + postcss-load-config: 6.0.1(postcss@8.4.47)(tsx@4.19.1) + resolve-from: 5.0.0 + rollup: 4.22.4 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyglobby: 0.2.6 + tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.4.47 + typescript: 4.9.5 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + tsx@4.19.1: + dependencies: + esbuild: 0.23.1 + get-tsconfig: 4.8.1 + optionalDependencies: + fsevents: 2.3.3 + tsx@4.7.1: dependencies: esbuild: 0.19.12 @@ -11254,6 +11905,10 @@ snapshots: dependencies: zod: 3.23.8 + zod-to-json-schema@3.23.3(zod@3.23.8): + dependencies: + zod: 3.23.8 + zod@3.22.4: {} zod@3.23.8: {} diff --git a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts index bdfd12be08..22713cb244 100644 --- a/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts +++ b/js/testapps/byo-evaluator/src/deliciousness/deliciousness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { ModelArgument, loadPromptFile, z } from 'genkit'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { ai } from '../index.js'; @@ -31,7 +31,7 @@ export async function deliciousnessScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts b/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts index 8eb70dd4b0..8d0cc0be20 100644 --- a/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts +++ b/js/testapps/byo-evaluator/src/deliciousness/deliciousness_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -42,7 +42,7 @@ export function createDeliciousnessEvaluator< displayName: 'Deliciousness', definition: 'Determines if output is considered delicous.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await deliciousnessScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/funniness/funniness.ts b/js/testapps/byo-evaluator/src/funniness/funniness.ts index 26a5634018..68540904e2 100644 --- a/js/testapps/byo-evaluator/src/funniness/funniness.ts +++ b/js/testapps/byo-evaluator/src/funniness/funniness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { ModelArgument, loadPromptFile, z } from 'genkit'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { ai } from '../index.js'; @@ -33,7 +33,7 @@ const FunninessResponseSchema = z.object({ export async function funninessScore( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: z.infer ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts b/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts index 326568266e..34cd850ca5 100644 --- a/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts +++ b/js/testapps/byo-evaluator/src/funniness/funniness_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -43,7 +43,7 @@ export function createFunninessEvaluator< definition: 'Judges whether a statement is a joke and whether that joke is funny.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await funninessScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/pii/pii_detection.ts b/js/testapps/byo-evaluator/src/pii/pii_detection.ts index ae6b2eb5f8..ea0099f4a9 100644 --- a/js/testapps/byo-evaluator/src/pii/pii_detection.ts +++ b/js/testapps/byo-evaluator/src/pii/pii_detection.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { loadPromptFile, ModelArgument, z } from 'genkit'; -import { BaseDataPoint, Score } from 'genkit/evaluator'; +import { ModelArgument, loadPromptFile, z } from 'genkit'; +import { BaseEvalDataPoint, Score } from 'genkit/evaluator'; import path from 'path'; import { ai } from '../index.js'; @@ -28,7 +28,7 @@ export async function piiDetectionScore< CustomModelOptions extends z.ZodTypeAny, >( judgeLlm: ModelArgument, - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, judgeConfig?: CustomModelOptions ): Promise { const d = dataPoint; diff --git a/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts b/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts index 247e4a7bbf..ad644a5dda 100644 --- a/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts +++ b/js/testapps/byo-evaluator/src/pii/pii_evaluator.ts @@ -16,7 +16,7 @@ import { ModelReference, z } from 'genkit'; import { - BaseDataPoint, + BaseEvalDataPoint, EvaluatorAction, defineEvaluator, } from 'genkit/evaluator'; @@ -40,7 +40,7 @@ export function createPiiEvaluator( displayName: 'PII Detection', definition: 'Detects whether PII is present in the output.', }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await piiDetectionScore(judge, datapoint, judgeConfig); return { testCaseId: datapoint.testCaseId, diff --git a/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts b/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts index 6f0e084a85..cb0f8c2298 100644 --- a/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts +++ b/js/testapps/byo-evaluator/src/regex/regex_evaluator.ts @@ -15,7 +15,7 @@ */ import { - BaseDataPoint, + BaseEvalDataPoint, EvalResponse, EvaluatorAction, Score, @@ -29,7 +29,7 @@ const REGEX_MATCH_NAME_PREFIX = 'REGEX_MATCH'; /** * Create an EvalResponse from an individual scored datapoint. */ -function fillScores(dataPoint: BaseDataPoint, score: Score): EvalResponse { +function fillScores(dataPoint: BaseEvalDataPoint, score: Score): EvalResponse { return { testCaseId: dataPoint.testCaseId, evaluation: score, @@ -70,7 +70,7 @@ export function createRegexEvaluators( 'Runs the output against a regex and responds with 1 if a match is found and 0 otherwise.', isBilled: false, }, - async (datapoint: BaseDataPoint) => { + async (datapoint: BaseEvalDataPoint) => { const score = await regexMatchScore(datapoint, regexMetric.regex); return fillScores(datapoint, score); } @@ -82,7 +82,7 @@ export function createRegexEvaluators( * Score an individual datapoint. */ export async function regexMatchScore( - dataPoint: BaseDataPoint, + dataPoint: BaseEvalDataPoint, regex: RegExp ): Promise { const d = dataPoint;