Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 61 additions & 24 deletions js/plugins/google-genai/src/common/converters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
MessageData,
ModelReference,
Part,
TextPart,
ToolDefinition,
} from 'genkit/model';
import {
Expand Down Expand Up @@ -131,34 +132,34 @@ function toGeminiMedia(part: Part): GeminiPart {
media.videoMetadata = { ...videoMetadata };
}

return media;
return maybeAddGeminiThoughtSignature(part, media);
}

function toGeminiToolRequest(part: Part): GeminiPart {
if (!part.toolRequest?.input) {
throw Error('Invalid ToolRequestPart: input was missing.');
}
return {
return maybeAddGeminiThoughtSignature(part, {
functionCall: {
name: part.toolRequest.name,
args: part.toolRequest.input,
},
};
});
}

function toGeminiToolResponse(part: Part): GeminiPart {
if (!part.toolResponse?.output) {
throw Error('Invalid ToolResponsePart: output was missing.');
}
return {
return maybeAddGeminiThoughtSignature(part, {
functionResponse: {
name: part.toolResponse.name,
response: {
name: part.toolResponse.name,
content: part.toolResponse.output,
},
},
};
});
}

function toGeminiReasoning(part: Part): GeminiPart {
Expand All @@ -174,21 +175,38 @@ function toGeminiReasoning(part: Part): GeminiPart {

function toGeminiCustom(part: Part): GeminiPart {
if (part.custom?.codeExecutionResult) {
return {
return maybeAddGeminiThoughtSignature(part, {
codeExecutionResult: part.custom.codeExecutionResult,
};
});
}
if (part.custom?.executableCode) {
return {
return maybeAddGeminiThoughtSignature(part, {
executableCode: part.custom.executableCode,
};
});
}
throw new Error('Unsupported Custom Part type');
}

function toGeminiText(part: Part): GeminiPart {
return maybeAddGeminiThoughtSignature(part, { text: part.text ?? '' });
}

function maybeAddGeminiThoughtSignature(
part: Part,
geminiPart: GeminiPart
): GeminiPart {
if (part.metadata?.thoughtSignature) {
return {
...geminiPart,
thoughtSignature: part.metadata.thoughtSignature as string,
};
}
return geminiPart;
}

function toGeminiPart(part: Part): GeminiPart {
if (part.text) {
return { text: part.text };
if (typeof part.text === 'string') {
return toGeminiText(part);
}
if (part.media) {
return toGeminiMedia(part);
Expand Down Expand Up @@ -314,13 +332,27 @@ function fromGeminiFinishReason(
case 'SPII': // blocked for potentially containing Sensitive Personally Identifiable Information
return 'blocked';
case 'MALFORMED_FUNCTION_CALL':
case 'MISSING_THOUGHT_SIGNATURE':
case 'OTHER':
return 'other';
default:
return 'unknown';
}
}

function maybeAddThoughtSignature(geminiPart: GeminiPart, part: Part): Part {
if (geminiPart.thoughtSignature) {
return {
...part,
metadata: {
...part?.metadata,
thoughtSignature: geminiPart.thoughtSignature,
},
};
}
return part;
}

function fromGeminiThought(part: GeminiPart): Part {
return {
reasoning: part.text || '',
Expand All @@ -340,12 +372,13 @@ function fromGeminiInlineData(part: GeminiPart): Part {
const { mimeType, data } = part.inlineData;
// Combine data and mimeType into a data URL
const dataUrl = `data:${mimeType};base64,${data}`;
return {

return maybeAddThoughtSignature(part, {
media: {
url: dataUrl,
contentType: mimeType,
},
};
});
}

function fromGeminiFileData(part: GeminiPart): Part {
Expand All @@ -359,12 +392,12 @@ function fromGeminiFileData(part: GeminiPart): Part {
);
}

return {
return maybeAddThoughtSignature(part, {
media: {
url: part.fileData?.fileUri,
contentType: part.fileData?.mimeType,
},
};
});
}

function fromGeminiFunctionCall(part: GeminiPart, ref: string): Part {
Expand All @@ -373,13 +406,13 @@ function fromGeminiFunctionCall(part: GeminiPart, ref: string): Part {
'Invalid Gemini Function Call Part: missing function call data'
);
}
return {
return maybeAddThoughtSignature(part, {
toolRequest: {
name: part.functionCall.name,
input: part.functionCall.args,
ref,
},
};
});
}

function fromGeminiFunctionResponse(part: GeminiPart, ref?: string): Part {
Expand All @@ -388,46 +421,50 @@ function fromGeminiFunctionResponse(part: GeminiPart, ref?: string): Part {
'Invalid Gemini Function Call Part: missing function call data'
);
}
return {
return maybeAddThoughtSignature(part, {
toolResponse: {
name: part.functionResponse.name.replace(/__/g, '/'), // restore slashes
output: part.functionResponse.response,
ref,
},
};
});
}

function fromExecutableCode(part: GeminiPart): Part {
if (!part.executableCode) {
throw new Error('Invalid GeminiPart: missing executableCode');
}
return {
return maybeAddThoughtSignature(part, {
custom: {
executableCode: {
language: part.executableCode.language,
code: part.executableCode.code,
},
},
};
});
}

function fromCodeExecutionResult(part: GeminiPart): Part {
if (!part.codeExecutionResult) {
throw new Error('Invalid GeminiPart: missing codeExecutionResult');
}
return {
return maybeAddThoughtSignature(part, {
custom: {
codeExecutionResult: {
outcome: part.codeExecutionResult.outcome,
output: part.codeExecutionResult.output,
},
},
};
});
}

function fromGeminiText(part: GeminiPart): Part {
return maybeAddThoughtSignature(part, { text: part.text } as TextPart);
}

function fromGeminiPart(part: GeminiPart, ref: string): Part {
if (part.thought) return fromGeminiThought(part as any);
if (typeof part.text === 'string') return { text: part.text };
if (typeof part.text === 'string') return fromGeminiText(part);
if (part.inlineData) return fromGeminiInlineData(part);
if (part.fileData) return fromGeminiFileData(part);
if (part.functionCall) return fromGeminiFunctionCall(part, ref);
Expand Down
2 changes: 2 additions & 0 deletions js/plugins/google-genai/src/common/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,8 @@ export enum FinishReason {
SPII = 'SPII',
// The function call generated by the model is invalid.
MALFORMED_FUNCTION_CALL = 'MALFORMED_FUNCTION_CALL',
// At least one thought signature from a previous call is missing.
MISSING_THOUGHT_SIGNATURE = 'MISSING_THOUGHT_SIGNATURE',
// Unknown reason.
OTHER = 'OTHER',
}
Expand Down
5 changes: 4 additions & 1 deletion js/plugins/google-genai/src/common/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,10 @@ function aggregateResponses(
if (part.thought) {
newPart.thought = part.thought;
}
if (part.text) {
if (part.thoughtSignature) {
newPart.thoughtSignature = part.thoughtSignature;
}
if (typeof part.text === 'string') {
newPart.text = part.text;
}
if (part.functionCall) {
Expand Down
11 changes: 10 additions & 1 deletion js/plugins/google-genai/src/googleai/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({
.min(0)
.max(24576)
.describe(
'Indicates the thinking budget in tokens. 0 is DISABLED. ' +
'For Gemini 2.5 - Indicates the thinking budget in tokens. 0 is DISABLED. ' +
'-1 is AUTOMATIC. The default values and allowed ranges are model ' +
'dependent. The thinking budget parameter gives the model guidance ' +
'on the number of thinking tokens it can use when generating a ' +
Expand All @@ -262,6 +262,14 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({
'tasks. '
)
.optional(),
thinkingLevel: z
.enum(['LOW', 'MEDIUM', 'HIGH'])
.describe(
'For Gemini 3.0 - Indicates the thinking level. A higher level ' +
'is associated with more detailed thinking, which is needed for solving ' +
'more complex tasks.'
)
.optional(),
})
.passthrough()
.optional(),
Expand Down Expand Up @@ -364,6 +372,7 @@ const GENERIC_GEMMA_MODEL = commonRef(
);

const KNOWN_GEMINI_MODELS = {
'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'),
'gemini-2.5-pro': commonRef('gemini-2.5-pro'),
'gemini-2.5-flash': commonRef('gemini-2.5-flash'),
'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'),
Expand Down
11 changes: 10 additions & 1 deletion js/plugins/google-genai/src/vertexai/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({
.min(0)
.max(24576)
.describe(
'Indicates the thinking budget in tokens. 0 is DISABLED. ' +
'For Gemini 2.5 - Indicates the thinking budget in tokens. 0 is DISABLED. ' +
'-1 is AUTOMATIC. The default values and allowed ranges are model ' +
'dependent. The thinking budget parameter gives the model guidance ' +
'on the number of thinking tokens it can use when generating a ' +
Expand All @@ -310,6 +310,14 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({
'tasks. '
)
.optional(),
thinkingLevel: z
.enum(['LOW', 'MEDIUM', 'HIGH'])
.describe(
'For Gemini 3.0 - Indicates the thinking level. A higher level ' +
'is associated with more detailed thinking, which is needed for solving ' +
'more complex tasks.'
)
.optional(),
})
.passthrough()
.optional(),
Expand Down Expand Up @@ -376,6 +384,7 @@ function commonRef(
export const GENERIC_MODEL = commonRef('gemini');

export const KNOWN_MODELS = {
'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'),
'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'),
'gemini-2.5-pro': commonRef('gemini-2.5-pro'),
'gemini-2.5-flash': commonRef('gemini-2.5-flash'),
Expand Down
35 changes: 35 additions & 0 deletions js/plugins/google-genai/tests/common/converters_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,41 @@ describe('fromGeminiCandidate', () => {
},
},
},
{
should:
'should transform gemini candidate with thoughtSignature correctly',
geminiCandidate: {
index: 0,
content: {
role: 'model',
parts: [
{
text: 'I have a thought.',
thoughtSignature: 'xyz-789',
},
],
},
finishReason: 'STOP',
},
expectedOutput: {
index: 0,
message: {
role: 'model',
content: [
{
text: 'I have a thought.',
metadata: { thoughtSignature: 'xyz-789' },
},
],
},
finishReason: 'stop',
finishMessage: undefined,
custom: {
citationMetadata: undefined,
safetyRatings: undefined,
},
},
},
{
should:
'should transform gemini candidate to genkit candidate (function call parts) correctly',
Expand Down
Loading