From 75d389831915baf06aaabb28bbdd36630c3ab687 Mon Sep 17 00:00:00 2001 From: Ingrid Fielker Date: Tue, 11 Nov 2025 15:24:32 -0500 Subject: [PATCH] feat(js/plugins/google-genai): Support for gemini 3.0 thinkingLevel. --- .../google-genai/src/common/converters.ts | 85 +++++++++++++------ js/plugins/google-genai/src/common/types.ts | 2 + js/plugins/google-genai/src/common/utils.ts | 5 +- .../google-genai/src/googleai/gemini.ts | 11 ++- .../google-genai/src/vertexai/gemini.ts | 11 ++- .../tests/common/converters_test.ts | 35 ++++++++ .../tests/googleai/gemini_test.ts | 37 +++++++- .../tests/vertexai/gemini_test.ts | 23 +++++ js/testapps/basic-gemini/package.json | 3 +- .../basic-gemini/src/index-vertexai.ts | 29 +++++++ js/testapps/basic-gemini/src/index.ts | 29 ++++++- 11 files changed, 238 insertions(+), 32 deletions(-) diff --git a/js/plugins/google-genai/src/common/converters.ts b/js/plugins/google-genai/src/common/converters.ts index 940d36162b..49227928b6 100644 --- a/js/plugins/google-genai/src/common/converters.ts +++ b/js/plugins/google-genai/src/common/converters.ts @@ -20,6 +20,7 @@ import { MessageData, ModelReference, Part, + TextPart, ToolDefinition, } from 'genkit/model'; import { @@ -131,26 +132,26 @@ function toGeminiMedia(part: Part): GeminiPart { media.videoMetadata = { ...videoMetadata }; } - return media; + return maybeAddGeminiThoughtSignature(part, media); } function toGeminiToolRequest(part: Part): GeminiPart { if (!part.toolRequest?.input) { throw Error('Invalid ToolRequestPart: input was missing.'); } - return { + return maybeAddGeminiThoughtSignature(part, { functionCall: { name: part.toolRequest.name, args: part.toolRequest.input, }, - }; + }); } function toGeminiToolResponse(part: Part): GeminiPart { if (!part.toolResponse?.output) { throw Error('Invalid ToolResponsePart: output was missing.'); } - return { + return maybeAddGeminiThoughtSignature(part, { functionResponse: { name: part.toolResponse.name, response: { @@ -158,7 +159,7 @@ function toGeminiToolResponse(part: Part): GeminiPart { content: part.toolResponse.output, }, }, - }; + }); } function toGeminiReasoning(part: Part): GeminiPart { @@ -174,21 +175,38 @@ function toGeminiReasoning(part: Part): GeminiPart { function toGeminiCustom(part: Part): GeminiPart { if (part.custom?.codeExecutionResult) { - return { + return maybeAddGeminiThoughtSignature(part, { codeExecutionResult: part.custom.codeExecutionResult, - }; + }); } if (part.custom?.executableCode) { - return { + return maybeAddGeminiThoughtSignature(part, { executableCode: part.custom.executableCode, - }; + }); } throw new Error('Unsupported Custom Part type'); } +function toGeminiText(part: Part): GeminiPart { + return maybeAddGeminiThoughtSignature(part, { text: part.text ?? '' }); +} + +function maybeAddGeminiThoughtSignature( + part: Part, + geminiPart: GeminiPart +): GeminiPart { + if (part.metadata?.thoughtSignature) { + return { + ...geminiPart, + thoughtSignature: part.metadata.thoughtSignature as string, + }; + } + return geminiPart; +} + function toGeminiPart(part: Part): GeminiPart { - if (part.text) { - return { text: part.text }; + if (typeof part.text === 'string') { + return toGeminiText(part); } if (part.media) { return toGeminiMedia(part); @@ -314,6 +332,7 @@ function fromGeminiFinishReason( case 'SPII': // blocked for potentially containing Sensitive Personally Identifiable Information return 'blocked'; case 'MALFORMED_FUNCTION_CALL': + case 'MISSING_THOUGHT_SIGNATURE': case 'OTHER': return 'other'; default: @@ -321,6 +340,19 @@ function fromGeminiFinishReason( } } +function maybeAddThoughtSignature(geminiPart: GeminiPart, part: Part): Part { + if (geminiPart.thoughtSignature) { + return { + ...part, + metadata: { + ...part?.metadata, + thoughtSignature: geminiPart.thoughtSignature, + }, + }; + } + return part; +} + function fromGeminiThought(part: GeminiPart): Part { return { reasoning: part.text || '', @@ -340,12 +372,13 @@ function fromGeminiInlineData(part: GeminiPart): Part { const { mimeType, data } = part.inlineData; // Combine data and mimeType into a data URL const dataUrl = `data:${mimeType};base64,${data}`; - return { + + return maybeAddThoughtSignature(part, { media: { url: dataUrl, contentType: mimeType, }, - }; + }); } function fromGeminiFileData(part: GeminiPart): Part { @@ -359,12 +392,12 @@ function fromGeminiFileData(part: GeminiPart): Part { ); } - return { + return maybeAddThoughtSignature(part, { media: { url: part.fileData?.fileUri, contentType: part.fileData?.mimeType, }, - }; + }); } function fromGeminiFunctionCall(part: GeminiPart, ref: string): Part { @@ -373,13 +406,13 @@ function fromGeminiFunctionCall(part: GeminiPart, ref: string): Part { 'Invalid Gemini Function Call Part: missing function call data' ); } - return { + return maybeAddThoughtSignature(part, { toolRequest: { name: part.functionCall.name, input: part.functionCall.args, ref, }, - }; + }); } function fromGeminiFunctionResponse(part: GeminiPart, ref?: string): Part { @@ -388,46 +421,50 @@ function fromGeminiFunctionResponse(part: GeminiPart, ref?: string): Part { 'Invalid Gemini Function Call Part: missing function call data' ); } - return { + return maybeAddThoughtSignature(part, { toolResponse: { name: part.functionResponse.name.replace(/__/g, '/'), // restore slashes output: part.functionResponse.response, ref, }, - }; + }); } function fromExecutableCode(part: GeminiPart): Part { if (!part.executableCode) { throw new Error('Invalid GeminiPart: missing executableCode'); } - return { + return maybeAddThoughtSignature(part, { custom: { executableCode: { language: part.executableCode.language, code: part.executableCode.code, }, }, - }; + }); } function fromCodeExecutionResult(part: GeminiPart): Part { if (!part.codeExecutionResult) { throw new Error('Invalid GeminiPart: missing codeExecutionResult'); } - return { + return maybeAddThoughtSignature(part, { custom: { codeExecutionResult: { outcome: part.codeExecutionResult.outcome, output: part.codeExecutionResult.output, }, }, - }; + }); +} + +function fromGeminiText(part: GeminiPart): Part { + return maybeAddThoughtSignature(part, { text: part.text } as TextPart); } function fromGeminiPart(part: GeminiPart, ref: string): Part { if (part.thought) return fromGeminiThought(part as any); - if (typeof part.text === 'string') return { text: part.text }; + if (typeof part.text === 'string') return fromGeminiText(part); if (part.inlineData) return fromGeminiInlineData(part); if (part.fileData) return fromGeminiFileData(part); if (part.functionCall) return fromGeminiFunctionCall(part, ref); diff --git a/js/plugins/google-genai/src/common/types.ts b/js/plugins/google-genai/src/common/types.ts index 4782140851..6ada7b6d58 100644 --- a/js/plugins/google-genai/src/common/types.ts +++ b/js/plugins/google-genai/src/common/types.ts @@ -448,6 +448,8 @@ export enum FinishReason { SPII = 'SPII', // The function call generated by the model is invalid. MALFORMED_FUNCTION_CALL = 'MALFORMED_FUNCTION_CALL', + // At least one thought signature from a previous call is missing. + MISSING_THOUGHT_SIGNATURE = 'MISSING_THOUGHT_SIGNATURE', // Unknown reason. OTHER = 'OTHER', } diff --git a/js/plugins/google-genai/src/common/utils.ts b/js/plugins/google-genai/src/common/utils.ts index 0dbb1af961..2864b7511c 100644 --- a/js/plugins/google-genai/src/common/utils.ts +++ b/js/plugins/google-genai/src/common/utils.ts @@ -479,7 +479,10 @@ function aggregateResponses( if (part.thought) { newPart.thought = part.thought; } - if (part.text) { + if (part.thoughtSignature) { + newPart.thoughtSignature = part.thoughtSignature; + } + if (typeof part.text === 'string') { newPart.text = part.text; } if (part.functionCall) { diff --git a/js/plugins/google-genai/src/googleai/gemini.ts b/js/plugins/google-genai/src/googleai/gemini.ts index 85a5687ad5..bfe336f1db 100644 --- a/js/plugins/google-genai/src/googleai/gemini.ts +++ b/js/plugins/google-genai/src/googleai/gemini.ts @@ -253,7 +253,7 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({ .min(0) .max(24576) .describe( - 'Indicates the thinking budget in tokens. 0 is DISABLED. ' + + 'For Gemini 2.5 - Indicates the thinking budget in tokens. 0 is DISABLED. ' + '-1 is AUTOMATIC. The default values and allowed ranges are model ' + 'dependent. The thinking budget parameter gives the model guidance ' + 'on the number of thinking tokens it can use when generating a ' + @@ -262,6 +262,14 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({ 'tasks. ' ) .optional(), + thinkingLevel: z + .enum(['LOW', 'MEDIUM', 'HIGH']) + .describe( + 'For Gemini 3.0 - Indicates the thinking level. A higher level ' + + 'is associated with more detailed thinking, which is needed for solving ' + + 'more complex tasks.' + ) + .optional(), }) .passthrough() .optional(), @@ -364,6 +372,7 @@ const GENERIC_GEMMA_MODEL = commonRef( ); const KNOWN_GEMINI_MODELS = { + 'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'), 'gemini-2.5-pro': commonRef('gemini-2.5-pro'), 'gemini-2.5-flash': commonRef('gemini-2.5-flash'), 'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'), diff --git a/js/plugins/google-genai/src/vertexai/gemini.ts b/js/plugins/google-genai/src/vertexai/gemini.ts index c3e47329c2..65ef0ca2d3 100644 --- a/js/plugins/google-genai/src/vertexai/gemini.ts +++ b/js/plugins/google-genai/src/vertexai/gemini.ts @@ -301,7 +301,7 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({ .min(0) .max(24576) .describe( - 'Indicates the thinking budget in tokens. 0 is DISABLED. ' + + 'For Gemini 2.5 - Indicates the thinking budget in tokens. 0 is DISABLED. ' + '-1 is AUTOMATIC. The default values and allowed ranges are model ' + 'dependent. The thinking budget parameter gives the model guidance ' + 'on the number of thinking tokens it can use when generating a ' + @@ -310,6 +310,14 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({ 'tasks. ' ) .optional(), + thinkingLevel: z + .enum(['LOW', 'MEDIUM', 'HIGH']) + .describe( + 'For Gemini 3.0 - Indicates the thinking level. A higher level ' + + 'is associated with more detailed thinking, which is needed for solving ' + + 'more complex tasks.' + ) + .optional(), }) .passthrough() .optional(), @@ -376,6 +384,7 @@ function commonRef( export const GENERIC_MODEL = commonRef('gemini'); export const KNOWN_MODELS = { + 'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'), 'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'), 'gemini-2.5-pro': commonRef('gemini-2.5-pro'), 'gemini-2.5-flash': commonRef('gemini-2.5-flash'), diff --git a/js/plugins/google-genai/tests/common/converters_test.ts b/js/plugins/google-genai/tests/common/converters_test.ts index 40d22a3f20..1da39d49ab 100644 --- a/js/plugins/google-genai/tests/common/converters_test.ts +++ b/js/plugins/google-genai/tests/common/converters_test.ts @@ -450,6 +450,41 @@ describe('fromGeminiCandidate', () => { }, }, }, + { + should: + 'should transform gemini candidate with thoughtSignature correctly', + geminiCandidate: { + index: 0, + content: { + role: 'model', + parts: [ + { + text: 'I have a thought.', + thoughtSignature: 'xyz-789', + }, + ], + }, + finishReason: 'STOP', + }, + expectedOutput: { + index: 0, + message: { + role: 'model', + content: [ + { + text: 'I have a thought.', + metadata: { thoughtSignature: 'xyz-789' }, + }, + ], + }, + finishReason: 'stop', + finishMessage: undefined, + custom: { + citationMetadata: undefined, + safetyRatings: undefined, + }, + }, + }, { should: 'should transform gemini candidate to genkit candidate (function call parts) correctly', diff --git a/js/plugins/google-genai/tests/googleai/gemini_test.ts b/js/plugins/google-genai/tests/googleai/gemini_test.ts index 28c6f879d9..a70d24ab32 100644 --- a/js/plugins/google-genai/tests/googleai/gemini_test.ts +++ b/js/plugins/google-genai/tests/googleai/gemini_test.ts @@ -93,7 +93,10 @@ describe('Google AI Gemini', () => { const mockCandidate = { index: 0, - content: { role: 'model', parts: [{ text: 'Hi there' }] }, + content: { + role: 'model', + parts: [{ text: 'Hi there', thoughtSignature: 'test-signature' }], + }, finishReason: 'STOP' as FinishReason, }; @@ -204,7 +207,12 @@ describe('Google AI Gemini', () => { const chunkArg = sendChunkSpy.lastCall.args[0]; assert.deepStrictEqual(chunkArg, { index: 0, - content: [{ text: 'Hi there' }], + content: [ + { + text: 'Hi there', + metadata: { thoughtSignature: 'test-signature' }, + }, + ], }); }); @@ -314,6 +322,29 @@ describe('Google AI Gemini', () => { `Expected URL to start with "https://my.custom.base.path/v1custom/models", but it was "${url}"` ); }); + + it('passes thinkingLevel to the API', async () => { + const model = defineModel('gemini-3-pro-preview', defaultPluginOptions); + mockFetchResponse(defaultApiResponse); + const request: GenerateRequest = { + ...minimalRequest, + config: { + thinkingConfig: { + thinkingLevel: 'HIGH', + }, + }, + }; + await model.run(request); + + const apiRequest: GenerateContentRequest = JSON.parse( + fetchStub.lastCall.args[1].body + ); + assert.deepStrictEqual(apiRequest.generationConfig, { + thinkingConfig: { + thinkingLevel: 'HIGH', + }, + }); + }); }); describe('Error Handling', () => { @@ -384,7 +415,7 @@ describe('Google AI Gemini', () => { }); it('returns a ModelReference for an unknown model string', () => { - const name = 'gemini-3.0-flash'; + const name = 'gemini-42.0-flash'; const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); diff --git a/js/plugins/google-genai/tests/vertexai/gemini_test.ts b/js/plugins/google-genai/tests/vertexai/gemini_test.ts index ed70978531..affcdfd19e 100644 --- a/js/plugins/google-genai/tests/vertexai/gemini_test.ts +++ b/js/plugins/google-genai/tests/vertexai/gemini_test.ts @@ -333,6 +333,29 @@ describe('Vertex AI Gemini', () => { assert.strictEqual(apiRequest.generationConfig?.maxOutputTokens, 100); }); + it('passes thinkingLevel to the API', async () => { + mockFetchResponse(defaultApiResponse); + const request: GenerateRequest = { + ...minimalRequest, + config: { + thinkingConfig: { + thinkingLevel: 'HIGH', + }, + }, + }; + const model = defineModel('gemini-3-pro-preview', clientOptions); + await model.run(request); + + const apiRequest: GenerateContentRequest = JSON.parse( + fetchStub.lastCall.args[1].body + ); + assert.deepStrictEqual(apiRequest.generationConfig, { + thinkingConfig: { + thinkingLevel: 'HIGH', + }, + }); + }); + it('sends labels when provided in config', async () => { mockFetchResponse(defaultApiResponse); const myLabels = { env: 'test', version: '1' }; diff --git a/js/testapps/basic-gemini/package.json b/js/testapps/basic-gemini/package.json index b7bbcf3161..afaeffc52f 100644 --- a/js/testapps/basic-gemini/package.json +++ b/js/testapps/basic-gemini/package.json @@ -8,7 +8,8 @@ "start": "node lib/index.js", "build": "tsc", "build:watch": "tsc --watch", - "genkit:dev": "genkit start -- npx tsx --watch src/index.ts" + "genkit:dev:googleai": "genkit start -- npx tsx --watch src/index.ts", + "genkit:dev:vertexai": "genkit start -- npx tsx --watch src/index-vertexai.ts" }, "keywords": [], "author": "", diff --git a/js/testapps/basic-gemini/src/index-vertexai.ts b/js/testapps/basic-gemini/src/index-vertexai.ts index fdc3e7437d..b182ec3a90 100644 --- a/js/testapps/basic-gemini/src/index-vertexai.ts +++ b/js/testapps/basic-gemini/src/index-vertexai.ts @@ -36,6 +36,35 @@ ai.defineFlow('basic-hi', async () => { return text; }); +// Gemini 3.0 thinkingLevel config +ai.defineFlow( + { + name: 'thinking-level', + inputSchema: z.enum(['LOW', 'MEDIUM', 'HIGH']), + outputSchema: z.any(), + }, + async (level) => { + const { text } = await ai.generate({ + model: vertexAI.model('gemini-3-pro-preview'), + prompt: + 'Alice, Bob, and Carol each live in a different house on the ' + + 'same street: red, green, and blue. The person who lives in the red house ' + + 'owns a cat. Bob does not live in the green house. Carol owns a dog. The ' + + 'green house is to the left of the red house. Alice does not own a cat. ' + + 'The person in the blue house owns a fish. ' + + 'Who lives in each house, and what pet do they own? Provide your ' + + 'step-by-step reasoning.', + config: { + location: 'global', + thinkingConfig: { + thinkingLevel: level, + }, + }, + }); + return text; + } +); + // Multimodal input ai.defineFlow('multimodal-input', async () => { const photoBase64 = fs.readFileSync('photo.jpg', { encoding: 'base64' }); diff --git a/js/testapps/basic-gemini/src/index.ts b/js/testapps/basic-gemini/src/index.ts index bca19a8fdf..81aabaa6bf 100644 --- a/js/testapps/basic-gemini/src/index.ts +++ b/js/testapps/basic-gemini/src/index.ts @@ -66,7 +66,7 @@ ai.defineFlow('basic-hi-with-retry', async () => { ai.defineFlow('basic-hi-with-fallback', async () => { const { text } = await ai.generate({ - model: googleAI.model('gemini-2.5-soemthing-that-does-not-exist'), + model: googleAI.model('gemini-2.5-something-that-does-not-exist'), prompt: 'You are a helpful AI assistant named Walt, say hello', use: [ fallback(ai, { @@ -79,6 +79,33 @@ ai.defineFlow('basic-hi-with-fallback', async () => { return text; }); +// Gemini 3.0 thinkingLevel config +ai.defineFlow( + { + name: 'thinking-level', + inputSchema: z.enum(['LOW', 'MEDIUM', 'HIGH']), + }, + async (level) => { + const { text } = await ai.generate({ + model: googleAI.model('gemini-3-pro-preview'), + prompt: + 'Alice, Bob, and Carol each live in a different house on the ' + + 'same street: red, green, and blue. The person who lives in the red house ' + + 'owns a cat. Bob does not live in the green house. Carol owns a dog. The ' + + 'green house is to the left of the red house. Alice does not own a cat. ' + + 'The person in the blue house owns a fish. ' + + 'Who lives in each house, and what pet do they own? Provide your ' + + 'step-by-step reasoning.', + config: { + thinkingConfig: { + thinkingLevel: level, + }, + }, + }); + return text; + } +); + // Multimodal input ai.defineFlow('multimodal-input', async () => { const photoBase64 = fs.readFileSync('photo.jpg', { encoding: 'base64' });