From 72f12710618da8e459675320af24b5fb0fd9930b Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Tue, 2 Dec 2025 16:30:27 +0000 Subject: [PATCH 01/10] feat(anthropic): add structured output --- js/plugins/anthropic/package.json | 2 +- js/plugins/anthropic/src/runner/beta.ts | 40 ++++++++++++++++++------- js/testapps/anthropic/src/beta/basic.ts | 7 ++++- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/js/plugins/anthropic/package.json b/js/plugins/anthropic/package.json index 32b1c4ba87..d8bbd2196b 100644 --- a/js/plugins/anthropic/package.json +++ b/js/plugins/anthropic/package.json @@ -29,7 +29,7 @@ "genkit": "workspace:^" }, "dependencies": { - "@anthropic-ai/sdk": "^0.68.0" + "@anthropic-ai/sdk": "^0.71.0" }, "devDependencies": { "@types/node": "^20.11.16", diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 6a71fa71d5..34e2f1faa2 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -66,6 +66,29 @@ const BETA_UNSUPPORTED_SERVER_TOOL_BLOCK_TYPES = new Set([ 'container_upload', ]); +const BETA_APIS = [ + // 'message-batches-2024-09-24', + // 'prompt-caching-2024-07-31', + // 'computer-use-2025-01-24', + // 'pdfs-2024-09-25', + // 'token-counting-2024-11-01', + // 'token-efficient-tools-2025-02-19', + // 'output-128k-2025-02-19', + // 'files-api-2025-04-14', + // 'mcp-client-2025-04-04', + // 'dev-full-thinking-2025-05-14', + // 'interleaved-thinking-2025-05-14', + // 'code-execution-2025-05-22', + // 'extended-cache-ttl-2025-04-11', + // 'context-1m-2025-08-07', + // 'context-management-2025-06-27', + // 'model-context-window-exceeded-2025-08-26', + // 'skills-2025-10-02', + // 'effort-param-2025-11-24', + // 'advanced-tool-use-2025-11-20', + 'structured-outputs-2025-11-13', +] + const unsupportedServerToolError = (blockType: string): string => `Anthropic beta runner does not yet support server-managed tool block '${blockType}'. Please retry against the stable API or wait for dedicated support.`; @@ -281,12 +304,6 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } - if (request.output?.format && request.output.format !== 'text') { - throw new Error( - `Only text output format is supported for Claude models currently` - ); - } - return body; } @@ -349,12 +366,15 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } - if (request.output?.format && request.output.format !== 'text') { - throw new Error( - `Only text output format is supported for Claude models currently` - ); + if (request.output?.schema) { + body.output_format = { + type: 'json_schema', + schema: request.output?.schema, + } } + body.betas = BETA_APIS; + return body; } diff --git a/js/testapps/anthropic/src/beta/basic.ts b/js/testapps/anthropic/src/beta/basic.ts index d1309b3400..d5623310f8 100644 --- a/js/testapps/anthropic/src/beta/basic.ts +++ b/js/testapps/anthropic/src/beta/basic.ts @@ -20,7 +20,11 @@ import { genkit } from 'genkit'; const ai = genkit({ plugins: [ // Default all flows in this sample to the beta surface - anthropic({ apiVersion: 'beta', cacheSystemPrompt: true }), + anthropic({ + apiVersion: 'beta', + cacheSystemPrompt: true, + apiKey: process.env.ANTHROPIC_API_KEY, + }), ], }); @@ -34,6 +38,7 @@ ai.defineFlow('anthropic-beta-hello', async () => { prompt: 'You are Claude on the beta API. Provide a concise greeting that mentions that you are using the beta API.', config: { temperature: 0.6 }, + output: {}, }); return text; From 2c0fa8a25b35fd3eca755f81d517c46282ed35ce Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 2 Dec 2025 16:49:34 +0000 Subject: [PATCH 02/10] test(js/plugins/anthropic): add live test for structured output --- js/plugins/anthropic/package.json | 1 + js/plugins/anthropic/src/runner/beta.ts | 4 +- js/plugins/anthropic/tests/live_test.ts | 49 +++++++++++++++++++++++++ js/pnpm-lock.yaml | 10 ++--- 4 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 js/plugins/anthropic/tests/live_test.ts diff --git a/js/plugins/anthropic/package.json b/js/plugins/anthropic/package.json index d8bbd2196b..998296e1cf 100644 --- a/js/plugins/anthropic/package.json +++ b/js/plugins/anthropic/package.json @@ -64,6 +64,7 @@ "build": "npm-run-all build:clean check compile", "build:watch": "tsup-node --watch", "test": "tsx --test tests/*_test.ts", + "test:live": "tsx --test tests/live_test.ts", "test:file": "tsx --test", "test:coverage": "check-node-version --node '>=22' && tsx --test --experimental-test-coverage --test-coverage-include='src/**/*.ts' ./tests/**/*_test.ts" } diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 34e2f1faa2..dc84455723 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -87,7 +87,7 @@ const BETA_APIS = [ // 'effort-param-2025-11-24', // 'advanced-tool-use-2025-11-20', 'structured-outputs-2025-11-13', -] +]; const unsupportedServerToolError = (blockType: string): string => `Anthropic beta runner does not yet support server-managed tool block '${blockType}'. Please retry against the stable API or wait for dedicated support.`; @@ -370,7 +370,7 @@ export class BetaRunner extends BaseRunner { body.output_format = { type: 'json_schema', schema: request.output?.schema, - } + }; } body.betas = BETA_APIS; diff --git a/js/plugins/anthropic/tests/live_test.ts b/js/plugins/anthropic/tests/live_test.ts new file mode 100644 index 0000000000..5f95beddd8 --- /dev/null +++ b/js/plugins/anthropic/tests/live_test.ts @@ -0,0 +1,49 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { genkit, z } from 'genkit'; +import { describe, it } from 'node:test'; +import { anthropic } from '../src/index.js'; + +const SKIP_LIVE_TESTS = !process.env.ANTHROPIC_API_KEY; + +describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { + it('should return structured output matching the schema', async () => { + const ai = genkit({ + plugins: [anthropic({ apiVersion: 'beta' })], + }); + + const schema = z.object({ + name: z.string(), + age: z.number(), + city: z.string(), + }); + + const result = await ai.generate({ + model: 'anthropic/claude-sonnet-4-5', + prompt: + 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', + output: { schema }, + }); + + const parsed = result.output; + assert.ok(parsed, 'Should have parsed output'); + assert.strictEqual(parsed.name, 'Alice'); + assert.strictEqual(parsed.age, 30); + assert.strictEqual(parsed.city, 'New York'); + }); +}); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 109b07c772..8ca620edda 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -260,8 +260,8 @@ importers: plugins/anthropic: dependencies: '@anthropic-ai/sdk': - specifier: ^0.68.0 - version: 0.68.0(zod@3.25.67) + specifier: ^0.71.0 + version: 0.71.0(zod@3.25.67) devDependencies: '@types/node': specifier: ^20.11.16 @@ -2092,8 +2092,8 @@ packages: '@anthropic-ai/sdk@0.24.3': resolution: {integrity: sha512-916wJXO6T6k8R6BAAcLhLPv/pnLGy7YSEBZXZ1XTFbLcTZE8oTy3oDW9WJf9KKZwMvVcePIfoTSvzXHRcGxkQQ==} - '@anthropic-ai/sdk@0.68.0': - resolution: {integrity: sha512-SMYAmbbiprG8k1EjEPMTwaTqssDT7Ae+jxcR5kWXiqTlbwMR2AthXtscEVWOHkRfyAV5+y3PFYTJRNa3OJWIEw==} + '@anthropic-ai/sdk@0.71.0': + resolution: {integrity: sha512-go1XeWXmpxuiTkosSXpb8tokLk2ZLkIRcXpbWVwJM6gH5OBtHOVsfPfGuqI1oW7RRt4qc59EmYbrXRZ0Ng06Jw==} hasBin: true peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -7931,7 +7931,7 @@ snapshots: transitivePeerDependencies: - encoding - '@anthropic-ai/sdk@0.68.0(zod@3.25.67)': + '@anthropic-ai/sdk@0.71.0(zod@3.25.67)': dependencies: json-schema-to-ts: 3.1.1 optionalDependencies: From f76acf27282b30c99cb629f76a440989b0c9e688 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 2 Dec 2025 17:13:24 +0000 Subject: [PATCH 03/10] fix(js/plugins/anthropic): filter by model and dynamically enhance outputs --- js/plugins/anthropic/src/models.ts | 29 ++++++++++-- js/plugins/anthropic/src/runner/beta.ts | 61 ++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/js/plugins/anthropic/src/models.ts b/js/plugins/anthropic/src/models.ts index 2ee33d933c..555d701a15 100644 --- a/js/plugins/anthropic/src/models.ts +++ b/js/plugins/anthropic/src/models.ts @@ -201,6 +201,15 @@ export function claudeModelReference( }); } +/** + * Models that support structured outputs in the beta API. + * @see https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs + */ +const STRUCTURED_OUTPUT_MODELS = new Set([ + 'claude-sonnet-4-5', + 'claude-opus-4-1', +]); + /** * Defines a Claude model with the given name and Anthropic client. * Accepts any model name and lets the API validate it. If the model is in KNOWN_CLAUDE_MODELS, uses that modelRef @@ -216,9 +225,23 @@ export function claudeModel( defaultApiVersion: apiVersion, } = params; // Use supported model ref if available, otherwise create generic model ref - const modelRef = KNOWN_CLAUDE_MODELS[name]; - const modelInfo = modelRef ? modelRef.info : GENERIC_CLAUDE_MODEL_INFO; - const configSchema = modelRef?.configSchema ?? AnthropicConfigSchema; + const knownModelRef = KNOWN_CLAUDE_MODELS[name]; + let modelInfo = knownModelRef + ? knownModelRef.info + : GENERIC_CLAUDE_MODEL_INFO; + const configSchema = knownModelRef?.configSchema ?? AnthropicConfigSchema; + + // Enhance model info with structured output support when using beta API + if (apiVersion === 'beta' && STRUCTURED_OUTPUT_MODELS.has(name)) { + modelInfo = { + ...modelInfo, + supports: { + ...modelInfo?.supports, + output: ['text', 'json'], + constrained: 'all', + }, + }; + } return model< AnthropicBaseConfigSchemaType | AnthropicThinkingConfigSchemaType diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index dc84455723..73f8757d37 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -89,6 +89,42 @@ const BETA_APIS = [ 'structured-outputs-2025-11-13', ]; +/** + * Models that support structured outputs (JSON schema). + * See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs + */ +const STRUCTURED_OUTPUT_MODELS = new Set([ + 'claude-sonnet-4-5', + 'claude-opus-4-1', +]); + +/** + * Transforms a JSON schema to be compatible with Anthropic's structured output requirements. + * Anthropic requires `additionalProperties: false` on all object types. + */ +function toAnthropicSchema( + schema: Record +): Record { + const out = structuredClone(schema); + + // Remove $schema if present + delete out.$schema; + + // Add additionalProperties: false to objects + if (out.type === 'object') { + out.additionalProperties = false; + } + + // Recursively process nested objects + for (const key in out) { + if (typeof out[key] === 'object' && out[key] !== null) { + out[key] = toAnthropicSchema(out[key] as Record); + } + } + + return out; +} + const unsupportedServerToolError = (blockType: string): string => `Anthropic beta runner does not yet support server-managed tool block '${blockType}'. Please retry against the stable API or wait for dedicated support.`; @@ -304,6 +340,21 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } + // Apply structured output when model supports it and constrained output is requested + const useStructuredOutput = + STRUCTURED_OUTPUT_MODELS.has(modelName) && + request.output?.constrained && + request.output?.schema; + + if (useStructuredOutput) { + body.output_format = { + type: 'json_schema', + schema: toAnthropicSchema(request.output.schema), + }; + } + + body.betas = BETA_APIS; + return body; } @@ -366,10 +417,16 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } - if (request.output?.schema) { + // Apply structured output when model supports it and constrained output is requested + const useStructuredOutput = + STRUCTURED_OUTPUT_MODELS.has(modelName) && + request.output?.constrained && + request.output?.schema; + + if (useStructuredOutput) { body.output_format = { type: 'json_schema', - schema: request.output?.schema, + schema: toAnthropicSchema(request.output.schema), }; } From e79bea51f8812762f218c27e7af2f94bb6c6d5dd Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Wed, 3 Dec 2025 12:35:48 +0000 Subject: [PATCH 04/10] fix(js/plugins/anthropic): pass through constrained output options correctly --- js/plugins/anthropic/src/models.ts | 45 ++- js/plugins/anthropic/src/runner/beta.ts | 30 +- js/plugins/anthropic/tests/live_test.ts | 14 +- .../anthropic/tests/mocks/anthropic-client.ts | 1 + .../anthropic/tests/structured_output_test.ts | 307 ++++++++++++++++++ 5 files changed, 357 insertions(+), 40 deletions(-) create mode 100644 js/plugins/anthropic/tests/structured_output_test.ts diff --git a/js/plugins/anthropic/src/models.ts b/js/plugins/anthropic/src/models.ts index 555d701a15..1c5cb99d68 100644 --- a/js/plugins/anthropic/src/models.ts +++ b/js/plugins/anthropic/src/models.ts @@ -81,7 +81,17 @@ export const KNOWN_CLAUDE_MODELS: Record< 'claude-opus-4': commonRef('claude-opus-4', AnthropicThinkingConfigSchema), 'claude-sonnet-4-5': commonRef( 'claude-sonnet-4-5', - AnthropicThinkingConfigSchema + AnthropicThinkingConfigSchema, + { + supports: { + multiturn: true, + tools: true, + media: true, + systemRole: true, + output: ['text', 'json'], + constrained: 'all', + }, + } ), 'claude-haiku-4-5': commonRef( 'claude-haiku-4-5', @@ -89,7 +99,17 @@ export const KNOWN_CLAUDE_MODELS: Record< ), 'claude-opus-4-1': commonRef( 'claude-opus-4-1', - AnthropicThinkingConfigSchema + AnthropicThinkingConfigSchema, + { + supports: { + multiturn: true, + tools: true, + media: true, + systemRole: true, + output: ['text', 'json'], + constrained: 'all', + }, + } ), }; @@ -201,15 +221,6 @@ export function claudeModelReference( }); } -/** - * Models that support structured outputs in the beta API. - * @see https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs - */ -const STRUCTURED_OUTPUT_MODELS = new Set([ - 'claude-sonnet-4-5', - 'claude-opus-4-1', -]); - /** * Defines a Claude model with the given name and Anthropic client. * Accepts any model name and lets the API validate it. If the model is in KNOWN_CLAUDE_MODELS, uses that modelRef @@ -231,18 +242,6 @@ export function claudeModel( : GENERIC_CLAUDE_MODEL_INFO; const configSchema = knownModelRef?.configSchema ?? AnthropicConfigSchema; - // Enhance model info with structured output support when using beta API - if (apiVersion === 'beta' && STRUCTURED_OUTPUT_MODELS.has(name)) { - modelInfo = { - ...modelInfo, - supports: { - ...modelInfo?.supports, - output: ['text', 'json'], - constrained: 'all', - }, - }; - } - return model< AnthropicBaseConfigSchemaType | AnthropicThinkingConfigSchemaType >( diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 73f8757d37..65bcc1bc53 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -89,15 +89,6 @@ const BETA_APIS = [ 'structured-outputs-2025-11-13', ]; -/** - * Models that support structured outputs (JSON schema). - * See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs - */ -const STRUCTURED_OUTPUT_MODELS = new Set([ - 'claude-sonnet-4-5', - 'claude-opus-4-1', -]); - /** * Transforms a JSON schema to be compatible with Anthropic's structured output requirements. * Anthropic requires `additionalProperties: false` on all object types. @@ -341,15 +332,19 @@ export class BetaRunner extends BaseRunner { } // Apply structured output when model supports it and constrained output is requested + + // TODO: factor out into a helper function? and make it cleaner const useStructuredOutput = - STRUCTURED_OUTPUT_MODELS.has(modelName) && - request.output?.constrained && - request.output?.schema; + request.output !== undefined && + request.output.schema !== undefined && + request.output.constrained && + request.output?.schema !== undefined && + request.output?.format === 'json'; if (useStructuredOutput) { body.output_format = { type: 'json_schema', - schema: toAnthropicSchema(request.output.schema), + schema: toAnthropicSchema(request.output!.schema!), }; } @@ -418,15 +413,18 @@ export class BetaRunner extends BaseRunner { } // Apply structured output when model supports it and constrained output is requested + // TODO: factor out into a helper function? and make it cleaner const useStructuredOutput = - STRUCTURED_OUTPUT_MODELS.has(modelName) && + request.output !== undefined && + request.output.schema !== undefined && request.output?.constrained && - request.output?.schema; + request.output?.schema !== undefined && + request.output?.format === 'json'; if (useStructuredOutput) { body.output_format = { type: 'json_schema', - schema: toAnthropicSchema(request.output.schema), + schema: toAnthropicSchema(request.output!.schema!), }; } diff --git a/js/plugins/anthropic/tests/live_test.ts b/js/plugins/anthropic/tests/live_test.ts index 5f95beddd8..bf249bd6a6 100644 --- a/js/plugins/anthropic/tests/live_test.ts +++ b/js/plugins/anthropic/tests/live_test.ts @@ -21,6 +21,7 @@ import { anthropic } from '../src/index.js'; const SKIP_LIVE_TESTS = !process.env.ANTHROPIC_API_KEY; +// TODO: clean this test up a little bit describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { it('should return structured output matching the schema', async () => { const ai = genkit({ @@ -31,13 +32,18 @@ describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { name: z.string(), age: z.number(), city: z.string(), + isStudent: z.boolean(), + isEmployee: z.boolean(), + isRetired: z.boolean(), + isUnemployed: z.boolean(), + isDisabled: z.boolean(), }); const result = await ai.generate({ model: 'anthropic/claude-sonnet-4-5', prompt: 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', - output: { schema }, + output: { schema, format: 'json', constrained: true }, }); const parsed = result.output; @@ -45,5 +51,11 @@ describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { assert.strictEqual(parsed.name, 'Alice'); assert.strictEqual(parsed.age, 30); assert.strictEqual(parsed.city, 'New York'); + // assert the others are NOT undefined + assert.notStrictEqual(parsed.isStudent, undefined); + assert.notStrictEqual(parsed.isEmployee, undefined); + assert.notStrictEqual(parsed.isRetired, undefined); + assert.notStrictEqual(parsed.isUnemployed, undefined); + assert.notStrictEqual(parsed.isDisabled, undefined); }); }); diff --git a/js/plugins/anthropic/tests/mocks/anthropic-client.ts b/js/plugins/anthropic/tests/mocks/anthropic-client.ts index 321df8f24f..e2dd4a7458 100644 --- a/js/plugins/anthropic/tests/mocks/anthropic-client.ts +++ b/js/plugins/anthropic/tests/mocks/anthropic-client.ts @@ -366,6 +366,7 @@ export function mockMessageWithContent( } function toBetaMessage(message: Message): BetaMessage { + // @ts-ignore return { ...message, container: null, diff --git a/js/plugins/anthropic/tests/structured_output_test.ts b/js/plugins/anthropic/tests/structured_output_test.ts new file mode 100644 index 0000000000..585e8a3de2 --- /dev/null +++ b/js/plugins/anthropic/tests/structured_output_test.ts @@ -0,0 +1,307 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { genkit, z } from 'genkit'; +import { describe, test } from 'node:test'; +import { anthropic } from '../src/index.js'; +import { __testClient } from '../src/types.js'; +import { + createMockAnthropicClient, + createMockAnthropicMessage, +} from './mocks/anthropic-client.js'; + +describe('Structured Output Tests', () => { + test('should pass output_format to API when using beta API with constrained output', async () => { + // Set up mock client to return a mock response + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: '{"name":"Alice","age":30,"city":"New York","isStudent":false,"isEmployee":true,"isRetired":false,"isUnemployed":false,"isDisabled":false}', + }), + }); + + // Set up plugin with beta API enabled + const plugin = anthropic({ + apiKey: 'test-key', + apiVersion: 'beta', + // @ts-ignore + [__testClient]: mockClient, + }); + + // Create Genkit instance with the plugin + const ai = genkit({ + plugins: [plugin], + }); + + // Call generate with sonnet 4.5 (supports native constrained output) + await ai.generate({ + model: 'anthropic/claude-sonnet-4-5', + prompt: + 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', + output: { + schema: z.object({ + name: z.string(), + age: z.number(), + city: z.string(), + isStudent: z.boolean(), + isEmployee: z.boolean(), + isRetired: z.boolean(), + isUnemployed: z.boolean(), + isDisabled: z.boolean(), + }), + format: 'json', + constrained: true, + }, + }); + + // Verify the beta API was called with output_format + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual( + betaCreateStub.mock.calls.length, + 1, + 'Beta API should be called once' + ); + + const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; + assert.ok(apiRequest.output_format, 'Request should have output_format'); + assert.strictEqual( + apiRequest.output_format.type, + 'json_schema', + 'output_format type should be json_schema' + ); + assert.ok( + apiRequest.output_format.schema, + 'output_format should have schema' + ); + + // Verify schema transformation: additionalProperties should be false + assert.strictEqual( + apiRequest.output_format.schema.additionalProperties, + false, + 'Schema should have additionalProperties: false' + ); + }); + + // TODO: finish off this test suite + // test('should NOT pass output_format when constrained is false', async () => { + // const mockClient = createMockAnthropicClient({ + // messageResponse: createMockAnthropicMessage({ + // text: '{"name":"Alice"}', + // }), + // }); + + // const plugin = anthropic({ + // apiKey: 'test-key', + // apiVersion: 'beta', + // // @ts-ignore + // [__testClient]: mockClient, + // }); + + // // @ts-ignore + // const modelAction = plugin.resolve( + // 'model', + // 'claude-3-5-sonnet-20241022' + // ) as ModelAction; + + // const request: GenerateRequest = { + // messages: [ + // { + // role: 'user', + // content: [{ text: 'Generate JSON' }], + // }, + // ], + // output: { + // format: 'json', + // constrained: false, + // schema: { + // type: 'object', + // properties: { + // name: { type: 'string' }, + // }, + // }, + // }, + // }; + + // await modelAction(request, { + // streamingRequested: false, + // sendChunk: mock.fn(), + // abortSignal: new AbortController().signal, + // }); + + // const betaCreateStub = mockClient.beta.messages.create as any; + // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; + // assert.strictEqual( + // apiRequest.output_format, + // undefined, + // 'Request should NOT have output_format when constrained is false' + // ); + // }); + + // test('should NOT pass output_format when using stable API', async () => { + // const mockClient = createMockAnthropicClient({ + // messageResponse: createMockAnthropicMessage({ + // text: '{"name":"Alice","age":30,"city":"New York"}', + // }), + // }); + + // const plugin = anthropic({ + // apiKey: 'test-key', + // apiVersion: 'stable', + // // @ts-ignore + // [__testClient]: mockClient, + // }); + + // const modelAction = plugin.resolve( + // 'model', + // 'claude-3-5-sonnet-20241022' + // ) as ModelAction; + + // const request: GenerateRequest = { + // messages: [ + // { + // role: 'user', + // content: [{ text: 'Generate JSON' }], + // }, + // ], + // output: { + // format: 'json', + // constrained: true, + // schema: { + // type: 'object', + // properties: { + // name: { type: 'string' }, + // age: { type: 'number' }, + // city: { type: 'string' }, + // }, + // }, + // }, + // }; + + // await modelAction(request, { + // streamingRequested: false, + // sendChunk: mock.fn(), + // abortSignal: new AbortController().signal, + // }); + + // // Stable API should be called, not beta + // const stableCreateStub = mockClient.messages.create as any; + // assert.strictEqual( + // stableCreateStub.mock.calls.length, + // 1, + // 'Stable API should be called once' + // ); + + // const apiRequest = stableCreateStub.mock.calls[0].arguments[0]; + // assert.strictEqual( + // apiRequest.output_format, + // undefined, + // 'Stable API request should NOT have output_format' + // ); + // }); + + // test('should NOT pass output_format when format is not json', async () => { + // const mockClient = createMockAnthropicClient({ + // messageResponse: createMockAnthropicMessage({ + // text: 'Some text response', + // }), + // }); + + // const plugin = anthropic({ + // apiKey: 'test-key', + // apiVersion: 'beta', + // [__testClient]: mockClient, + // }); + + // const modelAction = plugin.resolve( + // 'model', + // 'claude-3-5-sonnet-20241022' + // ) as ModelAction; + + // const request: GenerateRequest = { + // messages: [ + // { + // role: 'user', + // content: [{ text: 'Generate text' }], + // }, + // ], + // output: { + // format: 'text', + // constrained: true, + // }, + // }; + + // await modelAction(request, { + // streamingRequested: false, + // sendChunk: mock.fn(), + // abortSignal: new AbortController().signal, + // }); + + // const betaCreateStub = mockClient.beta.messages.create as any; + // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; + // assert.strictEqual( + // apiRequest.output_format, + // undefined, + // 'Request should NOT have output_format when format is text' + // ); + // }); + + // test('should NOT pass output_format when schema is not provided', async () => { + // const mockClient = createMockAnthropicClient({ + // messageResponse: createMockAnthropicMessage({ + // text: '{"anything": "goes"}', + // }), + // }); + + // const plugin = anthropic({ + // apiKey: 'test-key', + // apiVersion: 'beta', + // [__testClient]: mockClient, + // }); + + // const modelAction = plugin.resolve( + // 'model', + // 'claude-3-5-sonnet-20241022' + // ) as ModelAction; + + // const request: GenerateRequest = { + // messages: [ + // { + // role: 'user', + // content: [{ text: 'Generate JSON' }], + // }, + // ], + // output: { + // format: 'json', + // constrained: true, + // // No schema provided + // }, + // }; + + // await modelAction(request, { + // streamingRequested: false, + // sendChunk: mock.fn(), + // abortSignal: new AbortController().signal, + // }); + + // const betaCreateStub = mockClient.beta.messages.create as any; + // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; + // assert.strictEqual( + // apiRequest.output_format, + // undefined, + // 'Request should NOT have output_format when schema is not provided' + // ); + // }); +}); From 30f65ae509fe44ffae6f72a31045340f6d946352 Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Wed, 3 Dec 2025 18:25:20 +0000 Subject: [PATCH 05/10] fix(anthropic): fix and add tests --- .../anthropic/tests/structured_output_test.ts | 509 ++++++++++-------- 1 file changed, 280 insertions(+), 229 deletions(-) diff --git a/js/plugins/anthropic/tests/structured_output_test.ts b/js/plugins/anthropic/tests/structured_output_test.ts index 585e8a3de2..9a1e31fcd5 100644 --- a/js/plugins/anthropic/tests/structured_output_test.ts +++ b/js/plugins/anthropic/tests/structured_output_test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type Anthropic from '@anthropic-ai/sdk'; import * as assert from 'assert'; import { genkit, z } from 'genkit'; import { describe, test } from 'node:test'; @@ -24,9 +25,89 @@ import { createMockAnthropicMessage, } from './mocks/anthropic-client.js'; +/** + * Test constants for consistent test setup + */ +const TEST_API_KEY = 'test-key'; +const SUPPORTING_MODEL = 'anthropic/claude-sonnet-4-5'; +const NON_SUPPORTING_MODEL = 'anthropic/claude-sonnet-4'; + +/** + * Options for creating a plugin with a mock client + */ +interface CreatePluginOptions { + apiVersion?: 'beta' | 'stable'; + mockClient: Anthropic; +} + +/** + * Creates an Anthropic plugin configured with a mock client for testing + */ +function createPlugin(options: CreatePluginOptions) { + return anthropic({ + apiKey: TEST_API_KEY, + apiVersion: options.apiVersion, + // @ts-ignore + [__testClient]: options.mockClient, + }); +} + +/** + * Creates a Genkit instance with the given plugin + */ +function createGenkitInstance(plugin: ReturnType) { + return genkit({ + plugins: [plugin], + }); +} + +/** + * Helper to get the proper create stub from the mock client for a given API version. + */ +function getCreateStub(mockClient: Anthropic, apiVersion: 'beta' | 'stable') { + return apiVersion === 'beta' + ? (mockClient.beta.messages.create as any) + : (mockClient.messages.create as any); +} + +/** + * Extracts the API request object from the mock for verification + * @param apiVersion - 'beta' or 'stable' to determine which API endpoint to check + */ +function getApiRequest( + mockClient: Anthropic, + apiVersion: 'beta' | 'stable', + callIndex: number = 0 +) { + const stub = getCreateStub(mockClient, apiVersion); + return stub.mock.calls[callIndex]?.arguments[0]; +} + +/** + * Verifies that the API was called the expected number of times + * @param apiVersion - 'beta' or 'stable' to determine which API endpoint to verify + */ +function verifyApiCalled( + mockClient: Anthropic, + apiVersion: 'beta' | 'stable', + expectedCalls: number = 1 +) { + const stub = getCreateStub(mockClient, apiVersion); + assert.strictEqual( + stub.mock.calls.length, + expectedCalls, + `${apiVersion === 'beta' ? 'Beta' : 'Stable'} API should be called ${expectedCalls} time(s)` + ); +} + +/** + * Tests for structured output (constrained generation) functionality. + * These tests verify that output_format is correctly passed to the Anthropic API + * when using the beta API with constrained output, and that it's NOT passed + * in various edge cases (stable API, non-json format, missing schema, etc.) + */ describe('Structured Output Tests', () => { test('should pass output_format to API when using beta API with constrained output', async () => { - // Set up mock client to return a mock response const mockClient = createMockAnthropicClient({ messageResponse: createMockAnthropicMessage({ text: '{"name":"Alice","age":30,"city":"New York","isStudent":false,"isEmployee":true,"isRetired":false,"isUnemployed":false,"isDisabled":false}', @@ -34,21 +115,16 @@ describe('Structured Output Tests', () => { }); // Set up plugin with beta API enabled - const plugin = anthropic({ - apiKey: 'test-key', + const plugin = createPlugin({ apiVersion: 'beta', - // @ts-ignore - [__testClient]: mockClient, + mockClient, }); - // Create Genkit instance with the plugin - const ai = genkit({ - plugins: [plugin], - }); + const ai = createGenkitInstance(plugin); // Call generate with sonnet 4.5 (supports native constrained output) await ai.generate({ - model: 'anthropic/claude-sonnet-4-5', + model: SUPPORTING_MODEL, prompt: 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', output: { @@ -67,15 +143,11 @@ describe('Structured Output Tests', () => { }, }); - // Verify the beta API was called with output_format - const betaCreateStub = mockClient.beta.messages.create as any; - assert.strictEqual( - betaCreateStub.mock.calls.length, - 1, - 'Beta API should be called once' - ); + // Verify the beta API was called + verifyApiCalled(mockClient, 'beta'); - const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; + // Verify output_format was included in the API request + const apiRequest = getApiRequest(mockClient, 'beta'); assert.ok(apiRequest.output_format, 'Request should have output_format'); assert.strictEqual( apiRequest.output_format.type, @@ -86,8 +158,7 @@ describe('Structured Output Tests', () => { apiRequest.output_format.schema, 'output_format should have schema' ); - - // Verify schema transformation: additionalProperties should be false + // Verify schema transformation: additionalProperties should be false for constrained output assert.strictEqual( apiRequest.output_format.schema.additionalProperties, false, @@ -95,213 +166,193 @@ describe('Structured Output Tests', () => { ); }); - // TODO: finish off this test suite - // test('should NOT pass output_format when constrained is false', async () => { - // const mockClient = createMockAnthropicClient({ - // messageResponse: createMockAnthropicMessage({ - // text: '{"name":"Alice"}', - // }), - // }); - - // const plugin = anthropic({ - // apiKey: 'test-key', - // apiVersion: 'beta', - // // @ts-ignore - // [__testClient]: mockClient, - // }); - - // // @ts-ignore - // const modelAction = plugin.resolve( - // 'model', - // 'claude-3-5-sonnet-20241022' - // ) as ModelAction; - - // const request: GenerateRequest = { - // messages: [ - // { - // role: 'user', - // content: [{ text: 'Generate JSON' }], - // }, - // ], - // output: { - // format: 'json', - // constrained: false, - // schema: { - // type: 'object', - // properties: { - // name: { type: 'string' }, - // }, - // }, - // }, - // }; - - // await modelAction(request, { - // streamingRequested: false, - // sendChunk: mock.fn(), - // abortSignal: new AbortController().signal, - // }); - - // const betaCreateStub = mockClient.beta.messages.create as any; - // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; - // assert.strictEqual( - // apiRequest.output_format, - // undefined, - // 'Request should NOT have output_format when constrained is false' - // ); - // }); - - // test('should NOT pass output_format when using stable API', async () => { - // const mockClient = createMockAnthropicClient({ - // messageResponse: createMockAnthropicMessage({ - // text: '{"name":"Alice","age":30,"city":"New York"}', - // }), - // }); - - // const plugin = anthropic({ - // apiKey: 'test-key', - // apiVersion: 'stable', - // // @ts-ignore - // [__testClient]: mockClient, - // }); - - // const modelAction = plugin.resolve( - // 'model', - // 'claude-3-5-sonnet-20241022' - // ) as ModelAction; - - // const request: GenerateRequest = { - // messages: [ - // { - // role: 'user', - // content: [{ text: 'Generate JSON' }], - // }, - // ], - // output: { - // format: 'json', - // constrained: true, - // schema: { - // type: 'object', - // properties: { - // name: { type: 'string' }, - // age: { type: 'number' }, - // city: { type: 'string' }, - // }, - // }, - // }, - // }; - - // await modelAction(request, { - // streamingRequested: false, - // sendChunk: mock.fn(), - // abortSignal: new AbortController().signal, - // }); - - // // Stable API should be called, not beta - // const stableCreateStub = mockClient.messages.create as any; - // assert.strictEqual( - // stableCreateStub.mock.calls.length, - // 1, - // 'Stable API should be called once' - // ); - - // const apiRequest = stableCreateStub.mock.calls[0].arguments[0]; - // assert.strictEqual( - // apiRequest.output_format, - // undefined, - // 'Stable API request should NOT have output_format' - // ); - // }); - - // test('should NOT pass output_format when format is not json', async () => { - // const mockClient = createMockAnthropicClient({ - // messageResponse: createMockAnthropicMessage({ - // text: 'Some text response', - // }), - // }); - - // const plugin = anthropic({ - // apiKey: 'test-key', - // apiVersion: 'beta', - // [__testClient]: mockClient, - // }); - - // const modelAction = plugin.resolve( - // 'model', - // 'claude-3-5-sonnet-20241022' - // ) as ModelAction; - - // const request: GenerateRequest = { - // messages: [ - // { - // role: 'user', - // content: [{ text: 'Generate text' }], - // }, - // ], - // output: { - // format: 'text', - // constrained: true, - // }, - // }; - - // await modelAction(request, { - // streamingRequested: false, - // sendChunk: mock.fn(), - // abortSignal: new AbortController().signal, - // }); - - // const betaCreateStub = mockClient.beta.messages.create as any; - // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; - // assert.strictEqual( - // apiRequest.output_format, - // undefined, - // 'Request should NOT have output_format when format is text' - // ); - // }); - - // test('should NOT pass output_format when schema is not provided', async () => { - // const mockClient = createMockAnthropicClient({ - // messageResponse: createMockAnthropicMessage({ - // text: '{"anything": "goes"}', - // }), - // }); - - // const plugin = anthropic({ - // apiKey: 'test-key', - // apiVersion: 'beta', - // [__testClient]: mockClient, - // }); - - // const modelAction = plugin.resolve( - // 'model', - // 'claude-3-5-sonnet-20241022' - // ) as ModelAction; - - // const request: GenerateRequest = { - // messages: [ - // { - // role: 'user', - // content: [{ text: 'Generate JSON' }], - // }, - // ], - // output: { - // format: 'json', - // constrained: true, - // // No schema provided - // }, - // }; - - // await modelAction(request, { - // streamingRequested: false, - // sendChunk: mock.fn(), - // abortSignal: new AbortController().signal, - // }); - - // const betaCreateStub = mockClient.beta.messages.create as any; - // const apiRequest = betaCreateStub.mock.calls[0].arguments[0]; - // assert.strictEqual( - // apiRequest.output_format, - // undefined, - // 'Request should NOT have output_format when schema is not provided' - // ); - // }); + test('should NOT pass output_format to API when constrained is false and using beta API', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: '{"name":"Alice"}', + }), + }); + + // Set up plugin with beta API enabled + const plugin = createPlugin({ + apiVersion: 'beta', + mockClient, + }); + + const ai = createGenkitInstance(plugin); + + // Call generate with constrained: false + await ai.generate({ + model: SUPPORTING_MODEL, + prompt: 'Generate JSON', + output: { + format: 'json', + constrained: false, + schema: z.object({ + name: z.string(), + }), + }, + }); + + // Verify the beta API was called + verifyApiCalled(mockClient, 'beta'); + + // Verify output_format was NOT included when constrained is false + const apiRequest = getApiRequest(mockClient, 'beta'); + assert.strictEqual( + apiRequest.output_format, + undefined, + 'Request should NOT have output_format when constrained is false' + ); + }); + + test('should NOT pass output_format to API when format is not json and using beta API', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Some text response', + }), + }); + + // Set up plugin with beta API enabled + const plugin = createPlugin({ + apiVersion: 'beta', + mockClient, + }); + + const ai = createGenkitInstance(plugin); + + // Call generate with format: 'text' (not 'json') + await ai.generate({ + model: SUPPORTING_MODEL, + prompt: 'Generate text', + output: { + format: 'text', + constrained: true, + }, + }); + + // Verify the beta API was called + verifyApiCalled(mockClient, 'beta'); + + // Verify output_format was NOT included when format is not json + const apiRequest = getApiRequest(mockClient, 'beta'); + assert.strictEqual( + apiRequest.output_format, + undefined, + 'Request should NOT have output_format when format is text' + ); + }); + + test('should NOT pass output_format to API when schema is not provided and using beta API', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: '{"anything": "goes"}', + }), + }); + + // Set up plugin with beta API enabled + const plugin = createPlugin({ + apiVersion: 'beta', + mockClient, + }); + + const ai = createGenkitInstance(plugin); + + // Call generate with constrained: true but no schema + await ai.generate({ + model: SUPPORTING_MODEL, + prompt: 'Generate JSON', + output: { + format: 'json', + constrained: true, + // No schema provided + }, + }); + + // Verify the beta API was called + verifyApiCalled(mockClient, 'beta'); + + // Verify output_format was NOT included when schema is missing + const apiRequest = getApiRequest(mockClient, 'beta'); + assert.strictEqual( + apiRequest.output_format, + undefined, + 'Request should NOT have output_format when schema is not provided' + ); + }); + + test('should NOT pass output_format to API when model does not support structured output and using beta API', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: '{"name":"Alice"}', + }), + }); + + // Set up plugin with beta API enabled + const plugin = createPlugin({ + apiVersion: 'beta', + mockClient, + }); + + const ai = createGenkitInstance(plugin); + + // Call generate with model that does not support structured output + await ai.generate({ + model: NON_SUPPORTING_MODEL, + prompt: 'Generate JSON', + output: { + format: 'json', + constrained: true, + }, + }); + + // Verify the beta API was called + verifyApiCalled(mockClient, 'beta'); + + // Verify output_format was NOT included when model does not support structured output + const apiRequest = getApiRequest(mockClient, 'beta'); + assert.strictEqual( + apiRequest.output_format, + undefined, + 'Request should NOT have output_format when model does not support structured output' + ); + }); + + test('should throw an error when using stable API with non-text output format', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: '{"name":"Alice","age":30,"city":"New York"}', + }), + }); + + // Set up plugin with stable API (not beta) + const plugin = createPlugin({ + apiVersion: 'stable', + mockClient, + }); + + const ai = createGenkitInstance(plugin); + + // Call generate with constrained output (would work with beta API) + // Expect an error to be thrown since only text output is supported for stable API + await assert.rejects( + async () => { + await ai.generate({ + model: SUPPORTING_MODEL, + prompt: 'Generate JSON', + output: { + format: 'json', + constrained: true, + schema: z.object({ + name: z.string(), + age: z.number(), + city: z.string(), + }), + }, + }); + }, + /Only text output format is supported for Claude models currently/, + 'Should throw an error for non-text output on stable API' + ); + }); }); From 5366cc48bb4f19cb76e988aaeb3de68c80be20b9 Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Thu, 4 Dec 2025 15:21:38 +0000 Subject: [PATCH 06/10] refactor(anthropic): beta api addition moved --- js/plugins/anthropic/src/runner/beta.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 65bcc1bc53..8291353456 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -304,6 +304,7 @@ export class BetaRunner extends BaseRunner { max_tokens: request.config?.maxOutputTokens ?? this.DEFAULT_MAX_OUTPUT_TOKENS, messages, + betas: BETA_APIS, }; if (betaSystem !== undefined) body.system = betaSystem; @@ -348,8 +349,6 @@ export class BetaRunner extends BaseRunner { }; } - body.betas = BETA_APIS; - return body; } @@ -385,6 +384,7 @@ export class BetaRunner extends BaseRunner { request.config?.maxOutputTokens ?? this.DEFAULT_MAX_OUTPUT_TOKENS, messages, stream: true, + betas: BETA_APIS, }; if (betaSystem !== undefined) body.system = betaSystem; @@ -428,8 +428,6 @@ export class BetaRunner extends BaseRunner { }; } - body.betas = BETA_APIS; - return body; } From ccde30aa06785e769318103193f8b611c5695eed Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Thu, 4 Dec 2025 15:23:10 +0000 Subject: [PATCH 07/10] chore(anthropic): remove some comments --- js/plugins/anthropic/src/runner/beta.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 8291353456..2a085fdfb2 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -332,9 +332,6 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } - // Apply structured output when model supports it and constrained output is requested - - // TODO: factor out into a helper function? and make it cleaner const useStructuredOutput = request.output !== undefined && request.output.schema !== undefined && From ba9b86a0d6c26932d1a19803c6fd33ac6e8de023 Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Thu, 4 Dec 2025 15:33:38 +0000 Subject: [PATCH 08/10] refactor(anthropic): clean code a bit --- js/plugins/anthropic/src/runner/beta.ts | 29 ++++++++++--------------- js/plugins/anthropic/tests/live_test.ts | 26 ++++++++++++---------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 2a085fdfb2..92ad7c2598 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -332,14 +332,8 @@ export class BetaRunner extends BaseRunner { body.thinking = thinkingConfig as BetaMessageCreateParams['thinking']; } - const useStructuredOutput = - request.output !== undefined && - request.output.schema !== undefined && - request.output.constrained && - request.output?.schema !== undefined && - request.output?.format === 'json'; - - if (useStructuredOutput) { + // Apply structured output when model supports it and constrained output is requested + if (this.isStructuredOutputEnabled(request)) { body.output_format = { type: 'json_schema', schema: toAnthropicSchema(request.output!.schema!), @@ -410,21 +404,12 @@ export class BetaRunner extends BaseRunner { } // Apply structured output when model supports it and constrained output is requested - // TODO: factor out into a helper function? and make it cleaner - const useStructuredOutput = - request.output !== undefined && - request.output.schema !== undefined && - request.output?.constrained && - request.output?.schema !== undefined && - request.output?.format === 'json'; - - if (useStructuredOutput) { + if (this.isStructuredOutputEnabled(request)) { body.output_format = { type: 'json_schema', schema: toAnthropicSchema(request.output!.schema!), }; } - return body; } @@ -561,4 +546,12 @@ export class BetaRunner extends BaseRunner { return 'other'; } } + + private isStructuredOutputEnabled(request: GenerateRequest): boolean { + return !!( + request.output?.schema && + request.output.constrained && + request.output.format === 'json' + ); + } } diff --git a/js/plugins/anthropic/tests/live_test.ts b/js/plugins/anthropic/tests/live_test.ts index bf249bd6a6..287413351f 100644 --- a/js/plugins/anthropic/tests/live_test.ts +++ b/js/plugins/anthropic/tests/live_test.ts @@ -21,7 +21,6 @@ import { anthropic } from '../src/index.js'; const SKIP_LIVE_TESTS = !process.env.ANTHROPIC_API_KEY; -// TODO: clean this test up a little bit describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { it('should return structured output matching the schema', async () => { const ai = genkit({ @@ -41,21 +40,24 @@ describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { const result = await ai.generate({ model: 'anthropic/claude-sonnet-4-5', - prompt: - 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', + prompt: 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', output: { schema, format: 'json', constrained: true }, }); const parsed = result.output; assert.ok(parsed, 'Should have parsed output'); - assert.strictEqual(parsed.name, 'Alice'); - assert.strictEqual(parsed.age, 30); - assert.strictEqual(parsed.city, 'New York'); - // assert the others are NOT undefined - assert.notStrictEqual(parsed.isStudent, undefined); - assert.notStrictEqual(parsed.isEmployee, undefined); - assert.notStrictEqual(parsed.isRetired, undefined); - assert.notStrictEqual(parsed.isUnemployed, undefined); - assert.notStrictEqual(parsed.isDisabled, undefined); + assert.deepStrictEqual( + { name: parsed.name, age: parsed.age, city: parsed.city }, + { name: 'Alice', age: 30, city: 'New York' } + ); + + // Check that boolean fields are present and are actually booleans + for (const key of ['isStudent', 'isEmployee', 'isRetired', 'isUnemployed', 'isDisabled']) { + assert.strictEqual( + typeof parsed[key], + 'boolean', + `Field ${key} should be a boolean but got: ${typeof parsed[key]}` + ); + } }); }); From a221d2126b9cb4f2928d3638af465e972d84aaf9 Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Mon, 8 Dec 2025 08:35:22 +0000 Subject: [PATCH 09/10] chore: format --- js/plugins/anthropic/src/runner/beta.ts | 4 +++- js/plugins/anthropic/tests/live_test.ts | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/js/plugins/anthropic/src/runner/beta.ts b/js/plugins/anthropic/src/runner/beta.ts index 92ad7c2598..9fd7a306a1 100644 --- a/js/plugins/anthropic/src/runner/beta.ts +++ b/js/plugins/anthropic/src/runner/beta.ts @@ -547,7 +547,9 @@ export class BetaRunner extends BaseRunner { } } - private isStructuredOutputEnabled(request: GenerateRequest): boolean { + private isStructuredOutputEnabled( + request: GenerateRequest + ): boolean { return !!( request.output?.schema && request.output.constrained && diff --git a/js/plugins/anthropic/tests/live_test.ts b/js/plugins/anthropic/tests/live_test.ts index 287413351f..ae83a4d4fc 100644 --- a/js/plugins/anthropic/tests/live_test.ts +++ b/js/plugins/anthropic/tests/live_test.ts @@ -40,7 +40,8 @@ describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { const result = await ai.generate({ model: 'anthropic/claude-sonnet-4-5', - prompt: 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', + prompt: + 'Generate a fictional person with name "Alice", age 30, and city "New York". Return only the JSON.', output: { schema, format: 'json', constrained: true }, }); @@ -52,7 +53,13 @@ describe('Anthropic Live Tests', { skip: SKIP_LIVE_TESTS }, () => { ); // Check that boolean fields are present and are actually booleans - for (const key of ['isStudent', 'isEmployee', 'isRetired', 'isUnemployed', 'isDisabled']) { + for (const key of [ + 'isStudent', + 'isEmployee', + 'isRetired', + 'isUnemployed', + 'isDisabled', + ]) { assert.strictEqual( typeof parsed[key], 'boolean', From f849352abf2d5c3ae61eb6237f00fd1d6e60c396 Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Tue, 9 Dec 2025 17:13:48 +0000 Subject: [PATCH 10/10] feat(anthropic): support structured outputs for Claude Haiku 4.5 --- js/plugins/anthropic/package.json | 2 +- js/plugins/anthropic/src/models.ts | 12 ++++++- js/pnpm-lock.yaml | 50 ++++++++++++++++++++---------- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/js/plugins/anthropic/package.json b/js/plugins/anthropic/package.json index 998296e1cf..b2262b0d7c 100644 --- a/js/plugins/anthropic/package.json +++ b/js/plugins/anthropic/package.json @@ -29,7 +29,7 @@ "genkit": "workspace:^" }, "dependencies": { - "@anthropic-ai/sdk": "^0.71.0" + "@anthropic-ai/sdk": "^0.71.2" }, "devDependencies": { "@types/node": "^20.11.16", diff --git a/js/plugins/anthropic/src/models.ts b/js/plugins/anthropic/src/models.ts index 1c5cb99d68..54225f2b24 100644 --- a/js/plugins/anthropic/src/models.ts +++ b/js/plugins/anthropic/src/models.ts @@ -95,7 +95,17 @@ export const KNOWN_CLAUDE_MODELS: Record< ), 'claude-haiku-4-5': commonRef( 'claude-haiku-4-5', - AnthropicThinkingConfigSchema + AnthropicThinkingConfigSchema, + { + supports: { + multiturn: true, + tools: true, + media: true, + systemRole: true, + output: ['text', 'json'], + constrained: 'all', + }, + } ), 'claude-opus-4-1': commonRef( 'claude-opus-4-1', diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 8ca620edda..2565d73c8c 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -153,6 +153,10 @@ importers: zod-to-json-schema: specifier: ^3.22.4 version: 3.24.5(zod@3.25.67) + optionalDependencies: + '@genkit-ai/firebase': + specifier: ^1.16.1 + version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) devDependencies: '@types/express': specifier: ^4.17.21 @@ -178,10 +182,6 @@ importers: typescript: specifier: ^4.9.0 version: 4.9.5 - optionalDependencies: - '@genkit-ai/firebase': - specifier: ^1.16.1 - version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) doc-snippets: dependencies: @@ -260,8 +260,8 @@ importers: plugins/anthropic: dependencies: '@anthropic-ai/sdk': - specifier: ^0.71.0 - version: 0.71.0(zod@3.25.67) + specifier: ^0.71.2 + version: 0.71.2(zod@3.25.67) devDependencies: '@types/node': specifier: ^20.11.16 @@ -961,7 +961,7 @@ importers: version: 0.24.3(encoding@0.1.13) '@anthropic-ai/vertex-sdk': specifier: ^0.4.0 - version: 0.4.3(encoding@0.1.13) + version: 0.4.3(encoding@0.1.13)(zod@3.25.67) '@google-cloud/aiplatform': specifier: ^3.23.0 version: 3.35.0(encoding@0.1.13) @@ -986,6 +986,13 @@ importers: openai: specifier: ^4.52.7 version: 4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.67) + optionalDependencies: + '@google-cloud/bigquery': + specifier: ^7.8.0 + version: 7.9.4(encoding@0.1.13) + firebase-admin: + specifier: '>=12.2' + version: 13.4.0(encoding@0.1.13) devDependencies: '@types/node': specifier: ^20.11.16 @@ -1017,13 +1024,6 @@ importers: typescript: specifier: ^4.9.0 version: 4.9.5 - optionalDependencies: - '@google-cloud/bigquery': - specifier: ^7.8.0 - version: 7.9.4(encoding@0.1.13) - firebase-admin: - specifier: '>=12.2' - version: 13.4.0(encoding@0.1.13) testapps/anthropic: dependencies: @@ -2101,6 +2101,15 @@ packages: zod: optional: true + '@anthropic-ai/sdk@0.71.2': + resolution: {integrity: sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@anthropic-ai/sdk@0.9.1': resolution: {integrity: sha512-wa1meQ2WSfoY8Uor3EdrJq0jTiZJoKoSii2ZVWRY1oN4Tlr5s59pADg9T79FTbPe1/se5c3pBeZgJL63wmuoBA==} @@ -6564,6 +6573,7 @@ packages: next@15.3.3: resolution: {integrity: sha512-JqNj29hHNmCLtNvd090SyRbXJiivQ+58XjCcrC50Crb5g5u2zi7Y2YivbsEfzk6AtVI80akdOQbaMZwWB1Hthw==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} + deprecated: This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/CVE-2025-66478 for more details. hasBin: true peerDependencies: '@opentelemetry/api': ^1.1.0 @@ -7271,6 +7281,7 @@ packages: source-map@0.8.0-beta.0: resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} engines: {node: '>= 8'} + deprecated: The work that was done in this beta branch won't be included in future versions spdx-correct@3.2.0: resolution: {integrity: sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==} @@ -7937,6 +7948,12 @@ snapshots: optionalDependencies: zod: 3.25.67 + '@anthropic-ai/sdk@0.71.2(zod@3.25.67)': + dependencies: + json-schema-to-ts: 3.1.1 + optionalDependencies: + zod: 3.25.67 + '@anthropic-ai/sdk@0.9.1(encoding@0.1.13)': dependencies: '@types/node': 18.19.112 @@ -7951,13 +7968,14 @@ snapshots: transitivePeerDependencies: - encoding - '@anthropic-ai/vertex-sdk@0.4.3(encoding@0.1.13)': + '@anthropic-ai/vertex-sdk@0.4.3(encoding@0.1.13)(zod@3.25.67)': dependencies: - '@anthropic-ai/sdk': 0.24.3(encoding@0.1.13) + '@anthropic-ai/sdk': 0.71.0(zod@3.25.67) google-auth-library: 9.15.1(encoding@0.1.13) transitivePeerDependencies: - encoding - supports-color + - zod '@babel/code-frame@7.25.7': dependencies: