diff --git a/js/plugins/google-genai/src/googleai/gemini.ts b/js/plugins/google-genai/src/googleai/gemini.ts index 9631b58383..970982fd82 100644 --- a/js/plugins/google-genai/src/googleai/gemini.ts +++ b/js/plugins/google-genai/src/googleai/gemini.ts @@ -313,6 +313,31 @@ export const GeminiTtsConfigSchema = GeminiConfigSchema.extend({ export type GeminiTtsConfigSchemaType = typeof GeminiTtsConfigSchema; export type GeminiTtsConfig = z.infer; +export const GeminiImageConfigSchema = GeminiConfigSchema.extend({ + imageConfig: z + .object({ + aspectRatio: z + .enum([ + '1:1', + '2:3', + '3:2', + '3:4', + '4:3', + '4:5', + '5:4', + '9:16', + '16:9', + '21:9', + ]) + .optional(), + imageSize: z.enum(['1K', '2K', '4K']).optional(), + }) + .passthrough() + .optional(), +}).passthrough(); +export type GeminiImageConfigSchemaType = typeof GeminiImageConfigSchema; +export type GeminiImageConfig = z.infer; + export const GemmaConfigSchema = GeminiConfigSchema.extend({ temperature: z .number() @@ -331,7 +356,9 @@ export type GemmaConfig = z.infer; type ConfigSchemaType = | GeminiConfigSchemaType | GeminiTtsConfigSchemaType + | GeminiImageConfigSchemaType | GemmaConfigSchemaType; +type ConfigSchema = z.infer; function commonRef( name: string, @@ -370,6 +397,20 @@ const GENERIC_TTS_MODEL = commonRef( }, GeminiTtsConfigSchema ); +const GENERIC_IMAGE_MODEL = commonRef( + 'gemini-image', + { + supports: { + multiturn: true, + media: true, + tools: true, + toolChoice: true, + systemRole: true, + constrained: 'no-tools', + }, + }, + GeminiImageConfigSchema +); const GENERIC_GEMMA_MODEL = commonRef( 'gemma-generic', undefined, @@ -381,15 +422,17 @@ const KNOWN_GEMINI_MODELS = { 'gemini-2.5-pro': commonRef('gemini-2.5-pro'), 'gemini-2.5-flash': commonRef('gemini-2.5-flash'), 'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'), - 'gemini-2.5-flash-image-preview': commonRef('gemini-2.5-flash-image-preview'), - 'gemini-2.5-flash-image': commonRef('gemini-2.5-flash-image'), 'gemini-2.0-flash': commonRef('gemini-2.0-flash'), 'gemini-2.0-flash-lite': commonRef('gemini-2.0-flash-lite'), }; export type KnownGeminiModels = keyof typeof KNOWN_GEMINI_MODELS; export type GeminiModelName = `gemini-${string}`; export function isGeminiModelName(value: string): value is GeminiModelName { - return value.startsWith('gemini-') && !value.endsWith('-tts'); + return ( + value.startsWith('gemini-') && + !value.endsWith('-tts') && + !value.includes('-image') + ); } const KNOWN_TTS_MODELS = { @@ -410,6 +453,29 @@ export function isTTSModelName(value: string): value is TTSModelName { return value.startsWith('gemini-') && value.endsWith('-tts'); } +const KNOWN_IMAGE_MODELS = { + 'gemini-3-pro-image-preview': commonRef( + 'gemini-3-pro-image-preview', + { ...GENERIC_IMAGE_MODEL.info }, + GeminiImageConfigSchema + ), + 'gemini-2.5-flash-image-preview': commonRef( + 'gemini-2.5-flash-image-preview', + { ...GENERIC_IMAGE_MODEL.info }, + GeminiImageConfigSchema + ), + 'gemini-2.5-flash-image': commonRef( + 'gemini-2.5-flash-image', + { ...GENERIC_IMAGE_MODEL.info }, + GeminiImageConfigSchema + ), +} as const; +export type KnownImageModels = keyof typeof KNOWN_IMAGE_MODELS; +export type ImageModelName = `gemini-${string}-image${string}`; +export function isImageModelName(value: string): value is ImageModelName { + return value.startsWith('gemini-') && value.includes('-image'); +} + const KNOWN_GEMMA_MODELS = { 'gemma-3-12b-it': commonRef('gemma-3-12b-it', undefined, GemmaConfigSchema), 'gemma-3-1b-it': commonRef('gemma-3-1b-it', undefined, GemmaConfigSchema), @@ -426,12 +492,13 @@ export function isGemmaModelName(value: string): value is GemmaModelName { const KNOWN_MODELS = { ...KNOWN_GEMINI_MODELS, ...KNOWN_TTS_MODELS, + ...KNOWN_IMAGE_MODELS, ...KNOWN_GEMMA_MODELS, }; export function model( version: string, - config: GeminiConfig | GeminiTtsConfig | GemmaConfig = {} + config: ConfigSchema = {} ): ModelReference { const name = checkModelName(version); @@ -444,6 +511,15 @@ export function model( }); } + if (isImageModelName(name)) { + return modelRef({ + name: `googleai/${name}`, + config, + configSchema: GeminiImageConfigSchema, + info: { ...GENERIC_IMAGE_MODEL.info }, + }); + } + if (isGemmaModelName(name)) { return modelRef({ name: `googleai/${name}`, @@ -561,7 +637,7 @@ export function defineModel( }); } - const requestOptions: z.infer = { + const requestOptions: ConfigSchema = { ...request.config, }; const { diff --git a/js/plugins/google-genai/src/googleai/index.ts b/js/plugins/google-genai/src/googleai/index.ts index 220854ce04..8aedfb293f 100644 --- a/js/plugins/google-genai/src/googleai/index.ts +++ b/js/plugins/google-genai/src/googleai/index.ts @@ -61,7 +61,7 @@ async function resolver( } else if (imagen.isImagenModelName(actionName)) { return await imagen.defineModel(actionName, options); } else { - // gemini, tts, gemma, unknown models + // gemini, tts, image, gemma, unknown models return await gemini.defineModel(actionName, options); } break; @@ -129,6 +129,10 @@ export type GoogleAIPlugin = { name: gemini.KnownTtsModels | (gemini.TTSModelName & {}), config: gemini.GeminiTtsConfig ): ModelReference; + model( + name: gemini.KnownImageModels | (gemini.ImageModelName & {}), + config: gemini.GeminiImageConfig + ): ModelReference; model( name: gemini.KnownGeminiModels | (gemini.GeminiModelName & {}), config?: gemini.GeminiConfig @@ -163,7 +167,7 @@ export const googleAI = googleAIPlugin as GoogleAIPlugin; if (imagen.isImagenModelName(name)) { return imagen.model(name, config); } - // gemma, tts, gemini and unknown model families. + // gemma, tts, image, gemini and unknown model families. return gemini.model(name, config); }; googleAI.embedder = ( diff --git a/js/plugins/google-genai/src/vertexai/gemini.ts b/js/plugins/google-genai/src/vertexai/gemini.ts index 65ef0ca2d3..0fdb549bc9 100644 --- a/js/plugins/google-genai/src/vertexai/gemini.ts +++ b/js/plugins/google-genai/src/vertexai/gemini.ts @@ -357,8 +357,34 @@ export type GeminiConfigSchemaType = typeof GeminiConfigSchema; */ export type GeminiConfig = z.infer; +export const GeminiImageConfigSchema = GeminiConfigSchema.extend({ + imageConfig: z + .object({ + aspectRatio: z + .enum([ + '1:1', + '2:3', + '3:2', + '3:4', + '4:3', + '4:5', + '5:4', + '9:16', + '16:9', + '21:9', + ]) + .optional(), + imageSize: z.enum(['1K', '2K', '4K']).optional(), + }) + .passthrough() + .optional(), +}).passthrough(); +export type GeminiImageConfigSchemaType = typeof GeminiImageConfigSchema; +export type GeminiImageConfig = z.infer; + // This contains all the Gemini config schema types -type ConfigSchemaType = GeminiConfigSchemaType; +type ConfigSchemaType = GeminiConfigSchemaType | GeminiImageConfigSchemaType; +type ConfigSchema = z.infer; function commonRef( name: string, @@ -381,9 +407,14 @@ function commonRef( }); } -export const GENERIC_MODEL = commonRef('gemini'); +const GENERIC_MODEL = commonRef('gemini'); +const GENERIC_IMAGE_MODEL = commonRef( + 'gemini-image', + undefined, + GeminiImageConfigSchema +); -export const KNOWN_MODELS = { +export const KNOWN_GEMINI_MODELS = { 'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'), 'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'), 'gemini-2.5-pro': commonRef('gemini-2.5-pro'), @@ -393,21 +424,58 @@ export const KNOWN_MODELS = { 'gemini-2.0-flash-lite': commonRef('gemini-2.0-flash-lite'), 'gemini-2.0-flash-lite-001': commonRef('gemini-2.0-flash-lite-001'), } as const; -export type KnownModels = keyof typeof KNOWN_MODELS; +export type KnownGeminiModels = keyof typeof KNOWN_GEMINI_MODELS; export type GeminiModelName = `gemini-${string}`; export function isGeminiModelName(value?: string): value is GeminiModelName { - return !!value?.startsWith('gemini-') && !value.includes('embedding'); + return !!( + value?.startsWith('gemini-') && + !value.includes('embedding') && + !value.includes('-image') + ); } +export const KNOWN_IMAGE_MODELS = { + 'gemini-3-pro-image-preview': commonRef( + 'gemini-3-pro-image-preview', + { ...GENERIC_IMAGE_MODEL.info }, + GeminiImageConfigSchema + ), + 'gemini-2.5-flash-image': commonRef( + 'gemini-2.5-flash-image', + undefined, + GeminiImageConfigSchema + ), +} as const; +export type KnownImageModels = keyof typeof KNOWN_IMAGE_MODELS; +export type ImageModelName = `gemini-${string}-image${string}`; +export function isImageModelName(value?: string): value is ImageModelName { + return !!(value?.startsWith('gemini-') && value.includes('-image')); +} + +const KNOWN_MODELS = { + ...KNOWN_GEMINI_MODELS, + ...KNOWN_IMAGE_MODELS, +}; +export type KnownModels = keyof typeof KNOWN_MODELS; + export function model( version: string, - options: GeminiConfig = {} -): ModelReference { + config: ConfigSchema = {} +): ModelReference { const name = checkModelName(version); + if (isImageModelName(name)) { + return modelRef({ + name: `vertexai/${name}`, + config, + configSchema: GeminiImageConfigSchema, + info: { ...GENERIC_IMAGE_MODEL.info }, + }); + } + return modelRef({ name: `vertexai/${name}`, - config: options, + config, configSchema: GeminiConfigSchema, info: { ...GENERIC_MODEL.info, @@ -426,7 +494,8 @@ export function listActions(models: Model[]): ActionMetadata[] { return models .filter( (m) => - isGeminiModelName(modelName(m.name)) && + (isGeminiModelName(modelName(m.name)) || + isImageModelName(modelName(m.name))) && !KNOWN_DECOMISSIONED_MODELS.includes(modelName(m.name) || '') ) .map((m) => { @@ -502,7 +571,7 @@ export function defineModel( systemInstruction = toGeminiSystemInstruction(systemMessage); } - const requestConfig = { ...request.config }; + const requestConfig: ConfigSchema = { ...request.config }; const { apiKey: apiKeyFromConfig, @@ -725,4 +794,8 @@ export function defineModel( ); } -export const TEST_ONLY = { KNOWN_MODELS }; +export const TEST_ONLY = { + KNOWN_GEMINI_MODELS, + KNOWN_IMAGE_MODELS, + KNOWN_MODELS, +}; diff --git a/js/plugins/google-genai/src/vertexai/index.ts b/js/plugins/google-genai/src/vertexai/index.ts index 6b061f12ad..d7cbea43e3 100644 --- a/js/plugins/google-genai/src/vertexai/index.ts +++ b/js/plugins/google-genai/src/vertexai/index.ts @@ -124,7 +124,11 @@ function vertexAIPlugin(options?: VertexPluginOptions): GenkitPluginV2 { export type VertexAIPlugin = { (pluginOptions?: VertexPluginOptions): GenkitPluginV2; model( - name: gemini.KnownModels | (gemini.GeminiModelName & {}), + name: gemini.KnownImageModels | (gemini.ImageModelName & {}), + config?: gemini.GeminiImageConfig + ): ModelReference; + model( + name: gemini.KnownGeminiModels | (gemini.GeminiModelName & {}), config?: gemini.GeminiConfig ): ModelReference; model( @@ -166,7 +170,7 @@ export const vertexAI = vertexAIPlugin as VertexAIPlugin; if (veo.isVeoModelName(name)) { return veo.model(name, config); } - // gemini and unknown model families + // gemini, image and unknown model families return gemini.model(name, config); }; vertexAI.embedder = ( diff --git a/js/plugins/google-genai/tests/googleai/gemini_test.ts b/js/plugins/google-genai/tests/googleai/gemini_test.ts index 31de1081a9..0282b4665f 100644 --- a/js/plugins/google-genai/tests/googleai/gemini_test.ts +++ b/js/plugins/google-genai/tests/googleai/gemini_test.ts @@ -21,6 +21,7 @@ import { afterEach, beforeEach, describe, it } from 'node:test'; import * as sinon from 'sinon'; import { GeminiConfigSchema, + GeminiImageConfigSchema, GeminiTtsConfigSchema, defineModel, model, @@ -349,6 +350,34 @@ describe('Google AI Gemini', () => { }, }); }); + + it('passes imageConfig to the API', async () => { + const model = defineModel( + 'gemini-2.5-flash-image', + defaultPluginOptions + ); + mockFetchResponse(defaultApiResponse); + const request: GenerateRequest = { + ...minimalRequest, + config: { + imageConfig: { + aspectRatio: '16:9', + imageSize: '2K', + }, + }, + }; + await model.run(request); + + const apiRequest: GenerateContentRequest = JSON.parse( + fetchStub.lastCall.args[1].body + ); + assert.deepStrictEqual(apiRequest.generationConfig, { + imageConfig: { + aspectRatio: '16:9', + imageSize: '2K', + }, + }); + }); }); describe('Error Handling', () => { @@ -415,7 +444,7 @@ describe('Google AI Gemini', () => { const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); - assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); + assert.strictEqual(modelRef.configSchema, GeminiImageConfigSchema); }); it('returns a ModelReference for an unknown model string', () => { diff --git a/js/plugins/google-genai/tests/googleai/index_test.ts b/js/plugins/google-genai/tests/googleai/index_test.ts index ab7a629a1e..50f6f68fea 100644 --- a/js/plugins/google-genai/tests/googleai/index_test.ts +++ b/js/plugins/google-genai/tests/googleai/index_test.ts @@ -24,6 +24,7 @@ import { import { TEST_ONLY as GEMINI_TEST_ONLY, GeminiConfigSchema, + GeminiImageConfigSchema, GeminiTtsConfigSchema, GemmaConfigSchema, } from '../../src/googleai/gemini.js'; @@ -316,6 +317,54 @@ describe('GoogleAI Plugin', () => { ); }); + it('should return an image model reference for new models', () => { + const modelRef = googleAI.model('gemini-new-image-foo'); + assert.strictEqual( + modelRef.configSchema, + GeminiImageConfigSchema, + 'Should have GeminiImageConfigSchema' + ); + assert.ok( + modelRef.info?.supports?.multiturn, + 'Gemini Image model should support multiturn' + ); + }); + + it('should return an Image model reference with correct schema', () => { + const modelRef = googleAI.model('gemini-2.5-flash-image'); + assert.strictEqual( + modelRef.configSchema, + GeminiImageConfigSchema, + 'Should have GeminiImageConfigSchema' + ); + assert.ok( + modelRef.info?.supports?.multiturn, + 'Gemini Image model should support multiturn' + ); + }); + + it('should have config values for image model', () => { + const modelRef = googleAI.model('gemini-2.5-flash-image', { + imageConfig: { + aspectRatio: '16:9', + imageSize: '1K', + }, + }); + assert.strictEqual( + modelRef.configSchema, + GeminiImageConfigSchema, + 'Should have GeminiImageConfigSchema' + ); + assert.deepStrictEqual( + modelRef.config?.imageConfig, + { + aspectRatio: '16:9', + imageSize: '1K', + }, + 'should have correct imageConfig' + ); + }); + it('should return a Veo model reference with correct schema', () => { const modelRef = googleAI.model('veo-new-model'); assert.strictEqual( diff --git a/js/plugins/google-genai/tests/vertexai/gemini_test.ts b/js/plugins/google-genai/tests/vertexai/gemini_test.ts index affcdfd19e..290bafab75 100644 --- a/js/plugins/google-genai/tests/vertexai/gemini_test.ts +++ b/js/plugins/google-genai/tests/vertexai/gemini_test.ts @@ -25,6 +25,7 @@ import { getGenkitClientHeader } from '../../src/common/utils.js'; import { getVertexAIUrl } from '../../src/vertexai/client.js'; import { GeminiConfigSchema, + GeminiImageConfigSchema, defineModel, model, } from '../../src/vertexai/gemini.js'; @@ -122,7 +123,7 @@ describe('Vertex AI Gemini', () => { candidates: [mockCandidate], }; - describe('gemini() function', () => { + describe('model() function', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; const modelRef: ModelReference = model(name); @@ -139,6 +140,22 @@ describe('Vertex AI Gemini', () => { assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); }); + it('returns a ModelReference for a known image model string', () => { + const name = 'gemini-3-pro-image-preview'; + const modelRef = model(name); + assert.strictEqual(modelRef.name, `vertexai/${name}`); + assert.ok(modelRef.info?.supports?.multiturn); + assert.strictEqual(modelRef.configSchema, GeminiImageConfigSchema); + }); + + it('returns a ModelReference for an unknown image model string', () => { + const name = 'gemini-new-image-model'; + const modelRef = model(name); + assert.strictEqual(modelRef.name, `vertexai/${name}`); + assert.ok(modelRef.info?.supports?.multiturn); + assert.strictEqual(modelRef.configSchema, GeminiImageConfigSchema); + }); + it('applies options to the ModelReference', () => { const options = { temperature: 0.9, topK: 20 }; const modelRef: ModelReference = model( @@ -147,6 +164,12 @@ describe('Vertex AI Gemini', () => { ); assert.deepStrictEqual(modelRef.config, options); }); + + it('applies image config options to the ModelReference', () => { + const options = { imageConfig: { imageSize: '4K' } }; + const modelRef = model('gemini-3-pro-image-preview', options); + assert.deepStrictEqual(modelRef.config, options); + }); }); function runCommonTests(clientOptions: ClientOptions) { @@ -356,6 +379,32 @@ describe('Vertex AI Gemini', () => { }); }); + it('handles imageConfig', async () => { + mockFetchResponse(defaultApiResponse); + const request: GenerateRequest = { + ...minimalRequest, + config: { + imageConfig: { + aspectRatio: '16:9', + imageSize: '2K', + }, + }, + }; + const model = defineModel('gemini-3-pro-image-preview', clientOptions); + await model.run(request); + + const apiRequest: GenerateContentRequest = JSON.parse( + fetchStub.lastCall.args[1].body + ); + assert.deepStrictEqual( + (apiRequest.generationConfig as any)?.imageConfig, + { + aspectRatio: '16:9', + imageSize: '2K', + } + ); + }); + it('sends labels when provided in config', async () => { mockFetchResponse(defaultApiResponse); const myLabels = { env: 'test', version: '1' }; diff --git a/js/plugins/google-genai/tests/vertexai/index_test.ts b/js/plugins/google-genai/tests/vertexai/index_test.ts index 05b0f45b23..86a26d2a24 100644 --- a/js/plugins/google-genai/tests/vertexai/index_test.ts +++ b/js/plugins/google-genai/tests/vertexai/index_test.ts @@ -26,6 +26,7 @@ import { import { TEST_ONLY as GEMINI_TEST_ONLY, GeminiConfigSchema, + GeminiImageConfigSchema, } from '../../src/vertexai/gemini.js'; import { TEST_ONLY as IMAGEN_TEST_ONLY, @@ -81,7 +82,7 @@ describe('VertexAI Plugin', () => { describe('Initializer', () => { it('should pre-register flagship Gemini models', async () => { - const model1Name = Object.keys(GEMINI_TEST_ONLY.KNOWN_MODELS)[0]; + const model1Name = Object.keys(GEMINI_TEST_ONLY.KNOWN_GEMINI_MODELS)[0]; const model1Path = `/model/vertexai/${model1Name}`; const expectedBaseName = `vertexai/${model1Name}`; const model1 = await ai.registry.lookupAction(model1Path); @@ -90,7 +91,17 @@ describe('VertexAI Plugin', () => { }); it('should register all known Gemini models', async () => { - for (const modelName in GEMINI_TEST_ONLY.KNOWN_MODELS) { + for (const modelName in GEMINI_TEST_ONLY.KNOWN_GEMINI_MODELS) { + const modelPath = `/model/vertexai/${modelName}`; + const expectedBaseName = `vertexai/${modelName}`; + const model = await ai.registry.lookupAction(modelPath); + assert.ok(model, `${modelName} should be registered at ${modelPath}`); + assert.strictEqual(model?.__action.name, expectedBaseName); + } + }); + + it('should register all known Image models', async () => { + for (const modelName in GEMINI_TEST_ONLY.KNOWN_IMAGE_MODELS) { const modelPath = `/model/vertexai/${modelName}`; const expectedBaseName = `vertexai/${modelName}`; const model = await ai.registry.lookupAction(modelPath); @@ -203,6 +214,25 @@ describe('VertexAI Plugin', () => { ); }); + it('vertexAI.model should return a ModelReference for Gemini Image model with correct schema', () => { + const modelName = 'gemini-3-pro-image-preview'; + const modelRef = vertexAI.model(modelName); + assert.strictEqual( + modelRef.name, + `vertexai/${modelName}`, + 'Name should be prefixed' + ); + assert.ok( + modelRef.info?.supports?.multiturn, + 'Gemini model should support multiturn' + ); + assert.strictEqual( + modelRef.configSchema, + GeminiImageConfigSchema, + 'Should have GeminiImageConfigSchema' + ); + }); + it('vertexAI.model should return a ModelReference for Imagen with correct schema', () => { const modelName = 'imagen-3.0-generate-002'; const modelRef = vertexAI.model(modelName); diff --git a/js/testapps/basic-gemini/src/index-vertexai.ts b/js/testapps/basic-gemini/src/index-vertexai.ts index b182ec3a90..93974eb800 100644 --- a/js/testapps/basic-gemini/src/index-vertexai.ts +++ b/js/testapps/basic-gemini/src/index-vertexai.ts @@ -400,6 +400,23 @@ ai.defineFlow('gemini-image-editing', async (_) => { return media; }); +// Nano banana pro config +ai.defineFlow('nano-banana-pro', async (_) => { + const { media } = await ai.generate({ + model: vertexAI.model('gemini-3-pro-image-preview'), + prompt: 'Generate a picture of a sunset in the mountains by a lake', + config: { + responseModalities: ['TEXT', 'IMAGE'], + imageConfig: { + aspectRatio: '21:9', + imageSize: '4K', + }, + }, + }); + + return media; +}); + // A simple example of image generation with Gemini. ai.defineFlow('imagen-image-generation', async (_) => { const { media } = await ai.generate({ diff --git a/js/testapps/basic-gemini/src/index.ts b/js/testapps/basic-gemini/src/index.ts index c359715b80..69e66c8395 100644 --- a/js/testapps/basic-gemini/src/index.ts +++ b/js/testapps/basic-gemini/src/index.ts @@ -410,6 +410,26 @@ ai.defineFlow('gemini-image-editing', async (_) => { ], config: { responseModalities: ['TEXT', 'IMAGE'], + imageConfig: { + aspectRatio: '1:1', + }, + }, + }); + + return media; +}); + +// Nano banana pro config +ai.defineFlow('nano-banana-pro', async (_) => { + const { media } = await ai.generate({ + model: googleAI.model('gemini-3-pro-image-preview'), + prompt: 'Generate a picture of a sunset in the mountains by a lake', + config: { + responseModalities: ['TEXT', 'IMAGE'], + imageConfig: { + aspectRatio: '3:4', + imageSize: '1K', + }, }, });