Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 81 additions & 5 deletions js/plugins/google-genai/src/googleai/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,31 @@ export const GeminiTtsConfigSchema = GeminiConfigSchema.extend({
export type GeminiTtsConfigSchemaType = typeof GeminiTtsConfigSchema;
export type GeminiTtsConfig = z.infer<GeminiTtsConfigSchemaType>;

export const GeminiImageConfigSchema = GeminiConfigSchema.extend({
imageConfig: z
.object({
aspectRatio: z
.enum([
'1:1',
'2:3',
'3:2',
'3:4',
'4:3',
'4:5',
'5:4',
'9:16',
'16:9',
'21:9',
])
.optional(),
imageSize: z.enum(['1K', '2K', '4K']).optional(),
})
.passthrough()
.optional(),
}).passthrough();
export type GeminiImageConfigSchemaType = typeof GeminiImageConfigSchema;
export type GeminiImageConfig = z.infer<GeminiImageConfigSchemaType>;

export const GemmaConfigSchema = GeminiConfigSchema.extend({
temperature: z
.number()
Expand All @@ -331,7 +356,9 @@ export type GemmaConfig = z.infer<GemmaConfigSchemaType>;
type ConfigSchemaType =
| GeminiConfigSchemaType
| GeminiTtsConfigSchemaType
| GeminiImageConfigSchemaType
| GemmaConfigSchemaType;
type ConfigSchema = z.infer<ConfigSchemaType>;

function commonRef(
name: string,
Expand Down Expand Up @@ -370,6 +397,20 @@ const GENERIC_TTS_MODEL = commonRef(
},
GeminiTtsConfigSchema
);
const GENERIC_IMAGE_MODEL = commonRef(
'gemini-image',
{
supports: {
multiturn: true,
media: true,
tools: true,
toolChoice: true,
systemRole: true,
constrained: 'no-tools',
},
},
GeminiImageConfigSchema
);
const GENERIC_GEMMA_MODEL = commonRef(
'gemma-generic',
undefined,
Expand All @@ -381,15 +422,17 @@ const KNOWN_GEMINI_MODELS = {
'gemini-2.5-pro': commonRef('gemini-2.5-pro'),
'gemini-2.5-flash': commonRef('gemini-2.5-flash'),
'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'),
'gemini-2.5-flash-image-preview': commonRef('gemini-2.5-flash-image-preview'),
'gemini-2.5-flash-image': commonRef('gemini-2.5-flash-image'),
'gemini-2.0-flash': commonRef('gemini-2.0-flash'),
'gemini-2.0-flash-lite': commonRef('gemini-2.0-flash-lite'),
};
export type KnownGeminiModels = keyof typeof KNOWN_GEMINI_MODELS;
export type GeminiModelName = `gemini-${string}`;
export function isGeminiModelName(value: string): value is GeminiModelName {
return value.startsWith('gemini-') && !value.endsWith('-tts');
return (
value.startsWith('gemini-') &&
!value.endsWith('-tts') &&
!value.includes('-image')
);
}

const KNOWN_TTS_MODELS = {
Expand All @@ -410,6 +453,29 @@ export function isTTSModelName(value: string): value is TTSModelName {
return value.startsWith('gemini-') && value.endsWith('-tts');
}

const KNOWN_IMAGE_MODELS = {
'gemini-3-pro-image-preview': commonRef(
'gemini-3-pro-image-preview',
{ ...GENERIC_IMAGE_MODEL.info },
GeminiImageConfigSchema
),
'gemini-2.5-flash-image-preview': commonRef(
'gemini-2.5-flash-image-preview',
{ ...GENERIC_IMAGE_MODEL.info },
GeminiImageConfigSchema
),
'gemini-2.5-flash-image': commonRef(
'gemini-2.5-flash-image',
{ ...GENERIC_IMAGE_MODEL.info },
GeminiImageConfigSchema
),
} as const;
export type KnownImageModels = keyof typeof KNOWN_IMAGE_MODELS;
export type ImageModelName = `gemini-${string}-image${string}`;
export function isImageModelName(value: string): value is ImageModelName {
return value.startsWith('gemini-') && value.includes('-image');
}

const KNOWN_GEMMA_MODELS = {
'gemma-3-12b-it': commonRef('gemma-3-12b-it', undefined, GemmaConfigSchema),
'gemma-3-1b-it': commonRef('gemma-3-1b-it', undefined, GemmaConfigSchema),
Expand All @@ -426,12 +492,13 @@ export function isGemmaModelName(value: string): value is GemmaModelName {
const KNOWN_MODELS = {
...KNOWN_GEMINI_MODELS,
...KNOWN_TTS_MODELS,
...KNOWN_IMAGE_MODELS,
...KNOWN_GEMMA_MODELS,
};

export function model(
version: string,
config: GeminiConfig | GeminiTtsConfig | GemmaConfig = {}
config: ConfigSchema = {}
): ModelReference<ConfigSchemaType> {
const name = checkModelName(version);

Expand All @@ -444,6 +511,15 @@ export function model(
});
}

if (isImageModelName(name)) {
return modelRef({
name: `googleai/${name}`,
config,
configSchema: GeminiImageConfigSchema,
info: { ...GENERIC_IMAGE_MODEL.info },
});
}

if (isGemmaModelName(name)) {
return modelRef({
name: `googleai/${name}`,
Expand Down Expand Up @@ -561,7 +637,7 @@ export function defineModel(
});
}

const requestOptions: z.infer<ConfigSchemaType> = {
const requestOptions: ConfigSchema = {
...request.config,
};
const {
Expand Down
8 changes: 6 additions & 2 deletions js/plugins/google-genai/src/googleai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ async function resolver(
} else if (imagen.isImagenModelName(actionName)) {
return await imagen.defineModel(actionName, options);
} else {
// gemini, tts, gemma, unknown models
// gemini, tts, image, gemma, unknown models
return await gemini.defineModel(actionName, options);
}
break;
Expand Down Expand Up @@ -129,6 +129,10 @@ export type GoogleAIPlugin = {
name: gemini.KnownTtsModels | (gemini.TTSModelName & {}),
config: gemini.GeminiTtsConfig
): ModelReference<gemini.GeminiTtsConfigSchemaType>;
model(
name: gemini.KnownImageModels | (gemini.ImageModelName & {}),
config: gemini.GeminiImageConfig
): ModelReference<gemini.GeminiImageConfigSchemaType>;
model(
name: gemini.KnownGeminiModels | (gemini.GeminiModelName & {}),
config?: gemini.GeminiConfig
Expand Down Expand Up @@ -163,7 +167,7 @@ export const googleAI = googleAIPlugin as GoogleAIPlugin;
if (imagen.isImagenModelName(name)) {
return imagen.model(name, config);
}
// gemma, tts, gemini and unknown model families.
// gemma, tts, image, gemini and unknown model families.
return gemini.model(name, config);
};
googleAI.embedder = (
Expand Down
95 changes: 84 additions & 11 deletions js/plugins/google-genai/src/vertexai/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,34 @@ export type GeminiConfigSchemaType = typeof GeminiConfigSchema;
*/
export type GeminiConfig = z.infer<GeminiConfigSchemaType>;

export const GeminiImageConfigSchema = GeminiConfigSchema.extend({
imageConfig: z
.object({
aspectRatio: z
.enum([
'1:1',
'2:3',
'3:2',
'3:4',
'4:3',
'4:5',
'5:4',
'9:16',
'16:9',
'21:9',
])
.optional(),
imageSize: z.enum(['1K', '2K', '4K']).optional(),
})
.passthrough()
.optional(),
}).passthrough();
export type GeminiImageConfigSchemaType = typeof GeminiImageConfigSchema;
export type GeminiImageConfig = z.infer<GeminiImageConfigSchemaType>;

// This contains all the Gemini config schema types
type ConfigSchemaType = GeminiConfigSchemaType;
type ConfigSchemaType = GeminiConfigSchemaType | GeminiImageConfigSchemaType;
type ConfigSchema = z.infer<ConfigSchemaType>;

function commonRef(
name: string,
Expand All @@ -381,9 +407,14 @@ function commonRef(
});
}

export const GENERIC_MODEL = commonRef('gemini');
const GENERIC_MODEL = commonRef('gemini');
const GENERIC_IMAGE_MODEL = commonRef(
'gemini-image',
undefined,
GeminiImageConfigSchema
);

export const KNOWN_MODELS = {
export const KNOWN_GEMINI_MODELS = {
'gemini-3-pro-preview': commonRef('gemini-3-pro-preview'),
'gemini-2.5-flash-lite': commonRef('gemini-2.5-flash-lite'),
'gemini-2.5-pro': commonRef('gemini-2.5-pro'),
Expand All @@ -393,21 +424,58 @@ export const KNOWN_MODELS = {
'gemini-2.0-flash-lite': commonRef('gemini-2.0-flash-lite'),
'gemini-2.0-flash-lite-001': commonRef('gemini-2.0-flash-lite-001'),
} as const;
export type KnownModels = keyof typeof KNOWN_MODELS;
export type KnownGeminiModels = keyof typeof KNOWN_GEMINI_MODELS;
export type GeminiModelName = `gemini-${string}`;
export function isGeminiModelName(value?: string): value is GeminiModelName {
return !!value?.startsWith('gemini-') && !value.includes('embedding');
return !!(
value?.startsWith('gemini-') &&
!value.includes('embedding') &&
!value.includes('-image')
);
}

export const KNOWN_IMAGE_MODELS = {
'gemini-3-pro-image-preview': commonRef(
'gemini-3-pro-image-preview',
{ ...GENERIC_IMAGE_MODEL.info },
GeminiImageConfigSchema
),
'gemini-2.5-flash-image': commonRef(
'gemini-2.5-flash-image',
undefined,
GeminiImageConfigSchema
),
} as const;
export type KnownImageModels = keyof typeof KNOWN_IMAGE_MODELS;
export type ImageModelName = `gemini-${string}-image${string}`;
export function isImageModelName(value?: string): value is ImageModelName {
return !!(value?.startsWith('gemini-') && value.includes('-image'));
}

const KNOWN_MODELS = {
...KNOWN_GEMINI_MODELS,
...KNOWN_IMAGE_MODELS,
};
export type KnownModels = keyof typeof KNOWN_MODELS;

export function model(
version: string,
options: GeminiConfig = {}
): ModelReference<typeof GeminiConfigSchema> {
config: ConfigSchema = {}
): ModelReference<ConfigSchemaType> {
const name = checkModelName(version);

if (isImageModelName(name)) {
return modelRef({
name: `vertexai/${name}`,
config,
configSchema: GeminiImageConfigSchema,
info: { ...GENERIC_IMAGE_MODEL.info },
});
}

return modelRef({
name: `vertexai/${name}`,
config: options,
config,
configSchema: GeminiConfigSchema,
info: {
...GENERIC_MODEL.info,
Expand All @@ -426,7 +494,8 @@ export function listActions(models: Model[]): ActionMetadata[] {
return models
.filter(
(m) =>
isGeminiModelName(modelName(m.name)) &&
(isGeminiModelName(modelName(m.name)) ||
isImageModelName(modelName(m.name))) &&
!KNOWN_DECOMISSIONED_MODELS.includes(modelName(m.name) || '')
)
.map((m) => {
Expand Down Expand Up @@ -502,7 +571,7 @@ export function defineModel(
systemInstruction = toGeminiSystemInstruction(systemMessage);
}

const requestConfig = { ...request.config };
const requestConfig: ConfigSchema = { ...request.config };

const {
apiKey: apiKeyFromConfig,
Expand Down Expand Up @@ -725,4 +794,8 @@ export function defineModel(
);
}

export const TEST_ONLY = { KNOWN_MODELS };
export const TEST_ONLY = {
KNOWN_GEMINI_MODELS,
KNOWN_IMAGE_MODELS,
KNOWN_MODELS,
};
8 changes: 6 additions & 2 deletions js/plugins/google-genai/src/vertexai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,11 @@ function vertexAIPlugin(options?: VertexPluginOptions): GenkitPluginV2 {
export type VertexAIPlugin = {
(pluginOptions?: VertexPluginOptions): GenkitPluginV2;
model(
name: gemini.KnownModels | (gemini.GeminiModelName & {}),
name: gemini.KnownImageModels | (gemini.ImageModelName & {}),
config?: gemini.GeminiImageConfig
): ModelReference<gemini.GeminiImageConfigSchemaType>;
model(
name: gemini.KnownGeminiModels | (gemini.GeminiModelName & {}),
config?: gemini.GeminiConfig
): ModelReference<gemini.GeminiConfigSchemaType>;
model(
Expand Down Expand Up @@ -166,7 +170,7 @@ export const vertexAI = vertexAIPlugin as VertexAIPlugin;
if (veo.isVeoModelName(name)) {
return veo.model(name, config);
}
// gemini and unknown model families
// gemini, image and unknown model families
return gemini.model(name, config);
};
vertexAI.embedder = (
Expand Down
Loading