diff --git a/js/plugins/vertexai/src/evaluation/index.ts b/js/plugins/vertexai/src/evaluation/index.ts index 799c777329..cba7193626 100644 --- a/js/plugins/vertexai/src/evaluation/index.ts +++ b/js/plugins/vertexai/src/evaluation/index.ts @@ -21,6 +21,7 @@ import { vertexEvaluators } from './evaluation.js'; import { PluginOptions } from './types.js'; export { VertexAIEvaluationMetricType } from './types.js'; export { PluginOptions }; + /** * Add Google Cloud Vertex AI Rerankers API to Genkit. */ @@ -28,11 +29,6 @@ export function vertexAIEvaluation(options: PluginOptions): GenkitPlugin { return genkitPlugin('vertexAIEvaluation', async (ai: Genkit) => { const { projectId, location, authClient } = await getDerivedParams(options); - const metrics = - options?.evaluation && options.evaluation.metrics.length > 0 - ? options.evaluation.metrics - : []; - - vertexEvaluators(ai, authClient, metrics, projectId, location); + vertexEvaluators(ai, authClient, options.metrics, projectId, location); }); } diff --git a/js/plugins/vertexai/src/evaluation/types.ts b/js/plugins/vertexai/src/evaluation/types.ts index 7ab48c0556..a5924bb08e 100644 --- a/js/plugins/vertexai/src/evaluation/types.ts +++ b/js/plugins/vertexai/src/evaluation/types.ts @@ -45,10 +45,7 @@ export type VertexAIEvaluationMetric = /** Options specific to evaluation configuration */ export interface EvaluationOptions { - /** Configure Vertex AI evaluators */ - evaluation?: { - metrics: VertexAIEvaluationMetric[]; - }; + metrics: VertexAIEvaluationMetric[]; } export interface PluginOptions extends CommonPluginOptions, EvaluationOptions {} diff --git a/js/plugins/vertexai/src/modelgarden/index.ts b/js/plugins/vertexai/src/modelgarden/index.ts index 8acbe81d4d..52ac287bfe 100644 --- a/js/plugins/vertexai/src/modelgarden/index.ts +++ b/js/plugins/vertexai/src/modelgarden/index.ts @@ -23,6 +23,7 @@ import { modelGardenOpenaiCompatibleModel, } from './model_garden.js'; import { PluginOptions } from './types.js'; + /** * Add Google Cloud Vertex AI Rerankers API to Genkit. */ @@ -30,8 +31,7 @@ export function vertexAIModelGarden(options: PluginOptions): GenkitPlugin { return genkitPlugin('vertexAIModelGarden', async (ai: Genkit) => { const { projectId, location, authClient } = await getDerivedParams(options); - const mgModels = options?.modelGardenModels || options?.modelGarden?.models; - mgModels!.forEach((m) => { + options.models.forEach((m) => { const anthropicEntry = Object.entries(SUPPORTED_ANTHROPIC_MODELS).find( ([_, value]) => value.name === m.name ); @@ -49,7 +49,7 @@ export function vertexAIModelGarden(options: PluginOptions): GenkitPlugin { projectId, location, authClient, - options.modelGarden?.openAiBaseUrlTemplate + options.openAiBaseUrlTemplate ); return; } diff --git a/js/plugins/vertexai/src/modelgarden/types.ts b/js/plugins/vertexai/src/modelgarden/types.ts index d63dcca33a..b7748ad37d 100644 --- a/js/plugins/vertexai/src/modelgarden/types.ts +++ b/js/plugins/vertexai/src/modelgarden/types.ts @@ -17,34 +17,17 @@ import { ModelReference } from 'genkit'; import { CommonPluginOptions } from '../common/types.js'; -export enum VertexAIEvaluationMetricType { - // Update genkit/docs/plugins/vertex-ai.md when modifying the list of enums - BLEU = 'BLEU', - ROUGE = 'ROUGE', - FLUENCY = 'FLEUNCY', - SAFETY = 'SAFETY', - GROUNDEDNESS = 'GROUNDEDNESS', - SUMMARIZATION_QUALITY = 'SUMMARIZATION_QUALITY', - SUMMARIZATION_HELPFULNESS = 'SUMMARIZATION_HELPFULNESS', - SUMMARIZATION_VERBOSITY = 'SUMMARIZATION_VERBOSITY', -} - /** * Evaluation metric config. Use `metricSpec` to define the behavior of the metric. * The value of `metricSpec` will be included in the request to the API. See the API documentation * for details on the possible values of `metricSpec` for each metric. * https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation#parameter-list */ + /** Options specific to Model Garden configuration */ export interface ModelGardenOptions { - /** - * @deprecated use `modelGarden.models` - */ - modelGardenModels?: ModelReference[]; - modelGarden?: { - models: ModelReference[]; - openAiBaseUrlTemplate?: string; - }; + models: ModelReference[]; + openAiBaseUrlTemplate?: string; } export interface PluginOptions diff --git a/js/plugins/vertexai/src/rerankers/index.ts b/js/plugins/vertexai/src/rerankers/index.ts index 33cd8cf52f..0a3b90b343 100644 --- a/js/plugins/vertexai/src/rerankers/index.ts +++ b/js/plugins/vertexai/src/rerankers/index.ts @@ -36,7 +36,9 @@ export function vertexAIRerankers(options: PluginOptions): GenkitPlugin { projectId, location, authClient, - rerankOptions: options.rerankOptions, + rerankOptions: options.rerankers.map((o) => + typeof o === 'string' ? { model: o } : o + ), }); }); } diff --git a/js/plugins/vertexai/src/rerankers/reranker.ts b/js/plugins/vertexai/src/rerankers/reranker.ts index cc807cb75e..f09e20d59d 100644 --- a/js/plugins/vertexai/src/rerankers/reranker.ts +++ b/js/plugins/vertexai/src/rerankers/reranker.ts @@ -34,7 +34,7 @@ export async function vertexAiRerankers( const rerankOptions = options.rerankOptions; if (rerankOptions.length === 0) { - return; + throw new Error('Provide at least one reranker configuration.'); } const auth = options.authClient; @@ -42,6 +42,9 @@ export async function vertexAiRerankers( const projectId = options.projectId; for (const rerankOption of rerankOptions) { + if (!rerankOption.name && !rerankOption.model) { + throw new Error('At least one of name or model must be provided.'); + } ai.defineReranker( { name: `vertexai/${rerankOption.name || rerankOption.model}`, diff --git a/js/plugins/vertexai/src/rerankers/types.ts b/js/plugins/vertexai/src/rerankers/types.ts index 4f5b00d5f6..e5b8a34b48 100644 --- a/js/plugins/vertexai/src/rerankers/types.ts +++ b/js/plugins/vertexai/src/rerankers/types.ts @@ -33,13 +33,12 @@ export type VertexAIRerankerOptions = z.infer< typeof VertexAIRerankerOptionsSchema >; -// Define the structure for each individual reranker configuration -export const VertexRerankerConfigSchema = z.object({ - name: z.string().optional().describe('Name of the reranker'), // Optional: Name of the reranker - model: z.string().optional().describe('Model name for reranking'), // Optional: Model name, defaults to a pre-defined model -}); - -export type VertexRerankerConfig = z.infer; +export interface VertexRerankerConfig { + // Optional: Name of the reranker + name?: string; + // Optional: Model name, defaults to a pre-defined model + model?: string; +} export interface VertexRerankOptions { authClient: GoogleAuth; @@ -50,7 +49,7 @@ export interface VertexRerankOptions { export interface RerankerOptions { /** Configure reranker options */ - rerankOptions: VertexRerankerConfig[]; + rerankers: (string | VertexRerankerConfig)[]; } export interface PluginOptions extends CommonPluginOptions, RerankerOptions {} diff --git a/js/testapps/anthropic-models/src/index.ts b/js/testapps/anthropic-models/src/index.ts index 2c5bb8582c..b14d505363 100644 --- a/js/testapps/anthropic-models/src/index.ts +++ b/js/testapps/anthropic-models/src/index.ts @@ -37,9 +37,7 @@ const ai = genkit({ }), vertexAIModelGarden({ location: 'europe-west1', - modelGarden: { - models: [claude35Sonnet], - }, + models: [claude35Sonnet], }), ], }); diff --git a/js/testapps/dev-ui-gallery/src/genkit.ts b/js/testapps/dev-ui-gallery/src/genkit.ts index e22fc439fa..56ae4ab922 100644 --- a/js/testapps/dev-ui-gallery/src/genkit.ts +++ b/js/testapps/dev-ui-gallery/src/genkit.ts @@ -78,27 +78,23 @@ export const ai = genkit({ }), vertexAIModelGarden({ location: 'us-central1', - modelGarden: { - models: [claude35Sonnet], - }, + models: [claude35Sonnet], }), vertexAIEvaluation({ location: 'us-central1', - evaluation: { - metrics: [ - VertexAIEvaluationMetricType.BLEU, - VertexAIEvaluationMetricType.GROUNDEDNESS, - VertexAIEvaluationMetricType.SAFETY, - { - type: VertexAIEvaluationMetricType.ROUGE, - metricSpec: { - rougeType: 'rougeLsum', - useStemmer: true, - splitSummaries: 'true', - }, + metrics: [ + VertexAIEvaluationMetricType.BLEU, + VertexAIEvaluationMetricType.GROUNDEDNESS, + VertexAIEvaluationMetricType.SAFETY, + { + type: VertexAIEvaluationMetricType.ROUGE, + metricSpec: { + rougeType: 'rougeLsum', + useStemmer: true, + splitSummaries: 'true', }, - ], - }, + }, + ], }), // vector stores diff --git a/js/testapps/format-tester/src/index.ts b/js/testapps/format-tester/src/index.ts index 857487e385..195b914692 100644 --- a/js/testapps/format-tester/src/index.ts +++ b/js/testapps/format-tester/src/index.ts @@ -34,7 +34,7 @@ const ai = genkit({ }), vertexAIModelGarden({ location: 'us-east5', - modelGarden: { models: [claude35Sonnet, claude35SonnetV2] }, + models: [claude35Sonnet, claude35SonnetV2], }), ], }); diff --git a/js/testapps/model-tester/src/index.ts b/js/testapps/model-tester/src/index.ts index 7c1fe1fcbf..46a1e18cf0 100644 --- a/js/testapps/model-tester/src/index.ts +++ b/js/testapps/model-tester/src/index.ts @@ -35,9 +35,7 @@ export const ai = genkit({ }), vertexAIModelGarden({ location: 'us-central1', - modelGarden: { - models: [claude3Sonnet, llama31], - }, + models: [claude3Sonnet, llama31], }), ollama({ models: [ diff --git a/js/testapps/rag/src/genkit.ts b/js/testapps/rag/src/genkit.ts index b8436f752c..13fad83b2e 100644 --- a/js/testapps/rag/src/genkit.ts +++ b/js/testapps/rag/src/genkit.ts @@ -79,9 +79,7 @@ export const ai = genkit({ }), vertexAIModelGarden({ location: 'us-central1', - modelGarden: { - models: [claude3Sonnet, llama31], - }, + models: [claude3Sonnet, llama31], }), pinecone([ { diff --git a/js/testapps/vertexai-reranker/src/index.ts b/js/testapps/vertexai-reranker/src/index.ts index 5ed30acaa6..60278fef8b 100644 --- a/js/testapps/vertexai-reranker/src/index.ts +++ b/js/testapps/vertexai-reranker/src/index.ts @@ -35,11 +35,7 @@ const ai = genkit({ vertexAIRerankers({ projectId: PROJECT_ID, location: LOCATION, - rerankOptions: [ - { - model: 'vertexai/reranker', - }, - ], + rerankers: ['vertexai/reranker'], }), ], });