Add metadata support to requests (#1548)

* add metadata/store to requests

* ✨ Add metadata handling for caching and completions

Introduced metadata support for enhanced caching and API queries.

* ✨ refactor chat storage logic in session execution

Simplified assignment of `store` variable by refining metadata check.

demo/genaisrc/haiku.genai.mts

@@ -1,6 +1,2 @@
-script({ model: "echo"})
+script({ metadata: { name: "haiku" } })
 $`Write a haiku about code`
-await env.output.image(
-    "https://github.com/microsoft/genaiscript/blob/main/docs/public/images/favicon.png?raw=true",
-    "icon"
-)

docs/public/schemas/llms.json

@@ -124,6 +124,10 @@
                         "type": "boolean",
                         "description": "Indicates if the provider uses a single model"
                     },
+                    "metadata": {
+                        "type": "boolean",
+                        "description": "Indicates if metadata is supported along with 'store' field"
+                    },
                     "aliases": {
                         "type": "object",
                         "description": "List of model aliases for the provider",

docs/src/content/docs/reference/scripts/medata.md

@@ -0,0 +1,21 @@
+---
+title: Metadata
+description: Metadata for the script
+keywords: [metadata, script]
+sidebar:
+    order: 110
+---
+
+Metadata is a map of key-value pairs that is used to enable stored completions. A feature in OpenAI and Azure OpenAI that allows you to store and retrieve completions for a given prompt. This is used for distillation and evaluation purposes.
+
+```js
+script({
+    metadata: {
+        name: "my_script",
+    },
+})
+```
+
+Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured format, and querying for objects via API or the dashboard.
+
+Keys are strings with a maximum length of 64 characters. Values are strings with a maximum length of 512 characters.

packages/core/src/chat.ts

@@ -1190,6 +1190,7 @@ export async function executeChatSession(
         topLogprobs,
         cache,
         inner,
+        metadata,
         partialCb,
     } = genOptions
     assert(!!model, "model is required")
@@ -1228,6 +1229,7 @@ export async function executeChatSession(
         ? getChatCompletionCache(typeof cache === "string" ? cache : "chat")
         : undefined
     const chatTrace = trace.startTraceDetails(`💬 chat`, { expanded: true })
+    const store = !!metadata ? true : undefined
     const timer = measure("chat")
     const cacheImage = async (url: string) =>
         await fileCacheImage(url, {
@@ -1287,6 +1289,8 @@ export async function executeChatSession(
                     req = {
                         model,
                         temperature,
+                        store,
+                        metadata: store ? metadata : undefined,
                         reasoning_effort: reasoningEffort,
                         top_p: topP,
                         tool_choice:
@@ -1499,6 +1503,11 @@ function updateChatFeatures(
         req.max_completion_tokens = req.max_tokens
         delete req.max_tokens
     }
+    if (req.store && !features?.metadata) {
+        dbg(`metadata: disabled, not supported by ${provider}`)
+        delete req.metadata
+        delete req.store
+    }
 
     deleteUndefinedValues(req)
 }

packages/core/src/constants.ts

@@ -248,6 +248,7 @@ export const MODEL_PROVIDERS = Object.freeze<
         hidden?: boolean
         imageGeneration?: boolean
         singleModel?: boolean
+        metadata?: boolean
         responseType?: "json" | "json_object" | "json_schema"
         reasoningEfforts?: Record<string, number>
         aliases?: Record<string, string>
@@ -443,4 +444,4 @@ export const CACHE_SHA_LENGTH = 32
 export const MCP_RESOURCE_PROTOCOL = TOOL_ID
 
 export const RESOURCE_MAX_SIZE = 1024 * 1024 * 10 // 10MB
-export const MIN_NODE_VERSION_MAJOR = 20
+export const MIN_NODE_VERSION_MAJOR = 20

packages/core/src/expander.ts

@@ -31,6 +31,7 @@ import { installGlobalPromptContext } from "./globals"
 import { mark } from "./performance"
 import { nodeIsPackageTypeModule } from "./nodepackage"
 import { parseModelIdentifier } from "./models"
+import { metadataMerge } from "./metadata"
 
 /**
  * Executes a prompt expansion process based on the provided prompt script, variables, and options.
@@ -253,6 +254,7 @@ export async function expandTemplate(
         template.flexTokens
     const fenceFormat = options.fenceFormat ?? template.fenceFormat
     const cache = options.cache ?? template.cache
+    const metadata = metadataMerge(template.metadata, options.metadata)
     let seed = options.seed ?? normalizeInt(env.vars["seed"]) ?? template.seed
     if (seed !== undefined) seed = seed >> 0
     let logprobs = options.logprobs || template.logprobs
@@ -445,6 +447,7 @@ export async function expandTemplate(
         logprobs,
         topLogprobs,
         disposables,
+        metadata,
         fallbackTools: options.fallbackTools,
     }
 }

packages/core/src/generation.ts

@@ -16,7 +16,8 @@ export interface GenerationOptions
         ModelOptions,
         EmbeddingsModelOptions,
         ContentSafetyOptions,
-        ScriptRuntimeOptions {
+        ScriptRuntimeOptions,
+        MetadataOptions {
     inner: boolean // Indicates if the process is an inner operation
     runId?: string
     runDir?: string
@@ -26,7 +27,6 @@ export interface GenerationOptions
     outputTrace?: MarkdownTrace
     maxCachedTemperature?: number // Maximum temperature for caching purposes
     maxCachedTopP?: number // Maximum top-p value for caching
-    fallbackTools?: boolean // Disables model tools
     label?: string // Optional label for the operation
     cliInfo?: {
         files: string[] // Information about files in the CLI context

packages/core/src/llms.json

@@ -11,6 +11,7 @@
             "listModels": true,
             "imageGeneration": true,
             "responseFormat": "json_schema",
+            "metadata": true,
             "aliases": {
                 "large": "gpt-4.1",
                 "small": "gpt-4.1-mini",
@@ -58,6 +59,7 @@
             "speech": true,
             "imageGeneration": true,
             "aliases": {},
+            "metadata": true,
             "models": {
                 "o1-preview": {
                     "tools": false

packages/core/src/metadata.ts

@@ -0,0 +1,28 @@
+export function metadataValidate(
+    metadata: Record<string, string>
+): Record<string, string> | undefined {
+    if (!metadata) return undefined
+    const entries = Object.entries(metadata)
+    if (entries.length > 16)
+        throw new Error("Metadata can only have 16 entries")
+    for (const [key, value] of entries) {
+        if (typeof key !== "string" || key.length > 64)
+            throw new Error("Invalid metadata key")
+        if (typeof value !== "string" || value.length > 512)
+            throw new Error("Invalid metadata value")
+    }
+    return metadata
+}
+
+export function metadataMerge(
+    source: Record<string, string> | undefined,
+    update: Record<string, string> | undefined
+): Record<string, string> | undefined {
+    if (!source) return update
+    if (!update) return source
+    const res = {
+        ...(source || {}),
+        ...(update || {}),
+    }
+    return metadataValidate(res)
+}

packages/core/src/promptrunner.ts

@@ -188,6 +188,7 @@ export async function runTemplate(
             topLogprobs,
             disposables,
             cache,
+            metadata
         } = await expandTemplate(prj, template, options, env)
         const { output, generator, secrets, dbg: envDbg, ...restEnv } = env
 
@@ -279,6 +280,7 @@ export async function runTemplate(
             logprobs,
             topLogprobs,
             fallbackTools,
+            metadata,
             stats: runStats,
         }
         const chatResult = await executeChatSession(

packages/core/src/template.ts

@@ -9,6 +9,8 @@ import { host } from "./host"
 import { JSON5TryParse } from "./json5"
 import { humanize } from "./inflection"
 import { promptyParse, promptyToGenAIScript } from "./prompty"
+import { metadataValidate } from "./metadata"
+import { deleteUndefinedValues } from "./cleaners"
 
 /**
  * Extracts a template ID from the given filename by removing specific extensions
@@ -45,7 +47,8 @@ export function parsePromptScriptMeta(
         meta.group = meta.group || "system"
     }
     meta.defTools = parsePromptScriptTools(jsSource)
-    return meta
+    meta.metadata = metadataValidate(meta.metadata)
+    return deleteUndefinedValues(meta)
 }
 
 function parsePromptScriptTools(jsSource: string) {

packages/core/src/types/prompt_template.d.ts

@@ -783,6 +783,15 @@ interface McpToolAnnotations {
     }
 }
 
+interface MetadataOptions {
+    /**
+     * Set of 16 key-value pairs that can be attached to an object.
+     * This can be useful for storing additional information about the object in a structured format, and querying for objects via API or the dashboard.
+     * Keys are strings with a maximum length of 64 characters. Values are strings with a maximum length of 512 characters.
+     */
+    metadata?: Record<string, string>
+}
+
 interface PromptScript
     extends PromptLike,
         ModelOptions,
@@ -793,7 +802,8 @@ interface PromptScript
         SecretDetectionOptions,
         GitIgnoreFilterOptions,
         ScriptRuntimeOptions,
-        McpToolAnnotations {
+        McpToolAnnotations,
+        MetadataOptions {
     /**
      * Which provider to prefer when picking a model.
      */
@@ -1774,6 +1784,7 @@ type PromptSystemArgs = Omit<
     | "files"
     | "modelConcurrency"
     | "redteam"
+    | "metadata"
 >
 
 type StringLike = string | WorkspaceFile | WorkspaceFile[]

packages/sample/genaisrc/metadata.genai.mts

@@ -0,0 +1,6 @@
+script({
+    metadata: { name: "metadata", tool: "genaiscript" },
+    temperature: 1,
+})
+
+$`Write a joke in emojis.`