fix: make customOptions consistent in action metadata

genkit-tools/common/src/eval/parser.ts

@@ -67,8 +67,12 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   const metadata: Record<string, any> = {};
   for (const action of evaluatorActions) {
     metadata[action.name] = {
-      displayName: action.metadata![EVALUATOR_METADATA_KEY_DISPLAY_NAME],
-      definition: action.metadata![EVALUATOR_METADATA_KEY_DEFINITION],
+      displayName: (action.metadata!.evaluator as any)[
+        EVALUATOR_METADATA_KEY_DISPLAY_NAME
+      ],
+      definition: (action.metadata!.evaluator as any)[
+        EVALUATOR_METADATA_KEY_DEFINITION
+      ],
     };
   }
   return metadata;

go/ai/evaluator.go

@@ -202,7 +202,7 @@ func DefineBatchEvaluator(r *registry.Registry, provider, name string, options *
 	metadataMap["evaluatorDisplayName"] = options.DisplayName
 	metadataMap["evaluatorDefinition"] = options.Definition
 
-	return (*evaluatorActionDef)(core.DefineAction(r, provider, name, atype.Evaluator, metadataMap, batchEval)), nil
+	return (*evaluatorActionDef)(core.DefineAction(r, provider, name, atype.Evaluator, map[string]any{"evaluator": metadataMap}, batchEval)), nil
 }
 
 // IsDefinedEvaluator reports whether an [Evaluator] is defined.

js/ai/src/embedder.ts

@@ -16,6 +16,7 @@
 
 import { Action, defineAction, z } from '@genkit-ai/core';
 import { Registry } from '@genkit-ai/core/registry';
+import { toJsonSchema } from '@genkit-ai/core/schema';
 import { Document, DocumentData, DocumentDataSchema } from './document.js';
 
 /**
@@ -115,6 +116,11 @@ export function defineEmbedder<
       metadata: {
         type: 'embedder',
         info: options.info,
+        embedder: {
+          customOptions: options.configSchema
+            ? toJsonSchema({ schema: options.configSchema })
+            : undefined,
+        },
       },
     },
     (i) =>

js/ai/src/evaluator.ts

@@ -17,6 +17,7 @@
 import { Action, defineAction, z } from '@genkit-ai/core';
 import { logger } from '@genkit-ai/core/logging';
 import { Registry } from '@genkit-ai/core/registry';
+import { toJsonSchema } from '@genkit-ai/core/schema';
 import { SPAN_TYPE_ATTR, runInNewSpan } from '@genkit-ai/core/tracing';
 import { randomUUID } from 'crypto';
 
@@ -146,11 +147,16 @@ export function defineEvaluator<
   },
   runner: EvaluatorFn<EvalDataPoint, EvaluatorOptions>
 ) {
-  const metadata = {};
-  metadata[EVALUATOR_METADATA_KEY_IS_BILLED] =
+  const evalMetadata = {};
+  evalMetadata[EVALUATOR_METADATA_KEY_IS_BILLED] =
     options.isBilled == undefined ? true : options.isBilled;
-  metadata[EVALUATOR_METADATA_KEY_DISPLAY_NAME] = options.displayName;
-  metadata[EVALUATOR_METADATA_KEY_DEFINITION] = options.definition;
+  evalMetadata[EVALUATOR_METADATA_KEY_DISPLAY_NAME] = options.displayName;
+  evalMetadata[EVALUATOR_METADATA_KEY_DEFINITION] = options.definition;
+  if (options.configSchema) {
+    evalMetadata['customOptions'] = toJsonSchema({
+      schema: options.configSchema,
+    });
+  }
   const evaluator = defineAction(
     registry,
     {
@@ -164,7 +170,10 @@ export function defineEvaluator<
         evalRunId: z.string(),
       }),
       outputSchema: EvalResponsesSchema,
-      metadata: metadata,
+      metadata: {
+        type: 'evaluator',
+        evaluator: evalMetadata,
+      },
     },
     async (i) => {
       let evalResponses: EvalResponses = [];

js/ai/src/reranker.ts

@@ -16,6 +16,7 @@
 
 import { Action, defineAction, z } from '@genkit-ai/core';
 import { Registry } from '@genkit-ai/core/registry';
+import { toJsonSchema } from '@genkit-ai/core/schema';
 import { Part, PartSchema } from './document.js';
 import { Document, DocumentData, DocumentDataSchema } from './retriever.js';
 
@@ -124,6 +125,11 @@ export function defineReranker<OptionsType extends z.ZodTypeAny = z.ZodTypeAny>(
       metadata: {
         type: 'reranker',
         info: options.info,
+        reranker: {
+          customOptions: options.configSchema
+            ? toJsonSchema({ schema: options.configSchema })
+            : undefined,
+        },
       },
     },
     (i) =>

js/ai/src/retriever.ts

@@ -16,6 +16,7 @@
 
 import { Action, GenkitError, defineAction, z } from '@genkit-ai/core';
 import { Registry } from '@genkit-ai/core/registry';
+import { toJsonSchema } from '@genkit-ai/core/schema';
 import { Document, DocumentData, DocumentDataSchema } from './document.js';
 import { EmbedderInfo } from './embedder.js';
 
@@ -144,6 +145,11 @@ export function defineRetriever<
       metadata: {
         type: 'retriever',
         info: options.info,
+        retriever: {
+          customOptions: options.configSchema
+            ? toJsonSchema({ schema: options.configSchema })
+            : undefined,
+        },
       },
     },
     (i) => runner(new Document(i.query), i.options)
@@ -184,6 +190,11 @@ export function defineIndexer<IndexerOptions extends z.ZodTypeAny>(
       metadata: {
         type: 'indexer',
         embedderInfo: options.embedderInfo,
+        indexer: {
+          customOptions: options.configSchema
+            ? toJsonSchema({ schema: options.configSchema })
+            : undefined,
+        },
       },
     },
     (i) =>