Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 28 additions & 10 deletions src/core/ai/gateway.ts
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,33 @@ const _warnedRecipes = new Set<string>();
* The warning calls that out before production traffic hits it, while avoiding
* unrelated startup noise from recipes the current brain is not using.
*/
/**
* Pure predicate: would this recipe trip the missing-max_batch_tokens startup
* warning? A recipe warns when it declares an embedding touchpoint that
* (a) has no `max_batch_tokens`, (b) is not the OpenAI canonical fast-path
* recipe, and (c) has not explicitly opted out via `no_batch_cap: true`.
*
* Exported for testability. As of the google/gemini cap, every SHIPPED recipe
* is either capped or opts out, so the warning has no live fixture left in the
* registry. Tests exercise the guardrail through this predicate with a
* synthetic capless recipe — the mechanism that catches a FUTURE recipe which
* inherits the embedding touchpoint but forgets the cap stays covered without
* depending on any real recipe being left uncapped.
*/
export function shouldWarnMissingBatchTokens(recipe: Recipe): boolean {
const embedding = recipe.touchpoints?.embedding;
if (!embedding || embedding.max_batch_tokens !== undefined) return false;
// OpenAI is the canonical "no cap declared, fast path is intentional"
// recipe; suppress the warning for it. Every other recipe missing the
// field is suspicious.
if (recipe.id === 'openai') return false;
// v0.32 (#779): explicit opt-out for dynamic-cap recipes (Ollama,
// LiteLLM proxy, llama-server) — they ship without a static cap because
// the cap depends on a user-launched server. Warning is noise for them.
if (embedding.no_batch_cap === true) return false;
return true;
}

function warnRecipesMissingBatchTokens(): void {
const configuredProviderIds = new Set<string>();
for (const model of [_config?.embedding_model, _config?.embedding_multimodal_model]) {
Expand All @@ -519,16 +546,7 @@ function warnRecipesMissingBatchTokens(): void {

for (const recipe of listRecipes()) {
if (!configuredProviderIds.has(recipe.id)) continue;
const embedding = recipe.touchpoints?.embedding;
if (!embedding || embedding.max_batch_tokens !== undefined) continue;
// OpenAI is the canonical "no cap declared, fast path is intentional"
// recipe; suppress the warning for it. Every other recipe missing the
// field is suspicious.
if (recipe.id === 'openai') continue;
// v0.32 (#779): explicit opt-out for dynamic-cap recipes (Ollama,
// LiteLLM proxy, llama-server) — they ship without a static cap because
// the cap depends on a user-launched server. Warning is noise for them.
if (embedding.no_batch_cap === true) continue;
if (!shouldWarnMissingBatchTokens(recipe)) continue;
if (_warnedRecipes.has(recipe.id)) continue;
_warnedRecipes.add(recipe.id);
// eslint-disable-next-line no-console
Expand Down
13 changes: 13 additions & 0 deletions src/core/ai/recipes/google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,19 @@ export const google: Recipe = {
dims_options: [768, 1536, 3072],
cost_per_1m_tokens_usd: 0.15,
price_last_verified: '2026-04-20',
// gemini-embedding-001 caps each input at 2048 tokens and — unlike most
// providers — effectively accepts only ONE text per request (the
// batchEmbedContents batch size is 1; excess tokens error when
// AUTO_TRUNCATE=false). gbrain batches by token budget, not item count,
// so this can't enforce the 1-item rule outright, but a conservative
// 2048-token cap (1 char ≈ 1 token dense content, 0.5 utilization —
// same assumption as the voyage recipe) keeps any single request under
// the per-text limit and arms the gateway's recursive-halving safety
// net. Sources: https://ai.google.dev/api/embeddings ;
// langchain-ai/langchainjs#8490 (single-input batch limit).
max_batch_tokens: 2_048,
chars_per_token: 1,
safety_factor: 0.5,
},
expansion: {
models: ['gemini-2.0-flash', 'gemini-2.0-flash-lite'],
Expand Down
49 changes: 37 additions & 12 deletions test/ai/adaptive-embed-batch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@ import {
embed,
splitByTokenBudget,
isTokenLimitError,
shouldWarnMissingBatchTokens,
__setEmbedTransportForTests,
__getShrinkStateForTests,
} from '../../src/core/ai/gateway.ts';
import { AIConfigError, AITransientError } from '../../src/core/ai/errors.ts';
import type { EmbeddingTouchpoint, Recipe } from '../../src/core/ai/types.ts';

// --------- Test helpers ---------

Expand Down Expand Up @@ -383,34 +385,57 @@ describe('shrink-on-miss adaptive cache', () => {
describe('startup warning for recipes missing max_batch_tokens', () => {
beforeEach(() => resetGateway());

test('configured missing-cap recipe warns once; unrelated recipes stay quiet', () => {
test('every shipped embedding recipe declares a batch cap or opts out (no startup warnings)', () => {
const warnings: string[] = [];
const original = console.warn;
console.warn = (msg: string) => warnings.push(String(msg));
try {
// Configuring the gateway walks the whole recipe registry once. As of
// the google max_batch_tokens fix, EVERY built-in embedding recipe
// either declares max_batch_tokens (voyage, openai, google, …) or sets
// no_batch_cap: true (litellm-proxy, llama-server, ollama), so the
// missing-cap guardrail must stay silent. Reconfiguring must not
// resurrect a warning either.
configureOpenAI();
expect(warnings.length).toBe(0);
configureGoogle();
const firstCallCount = warnings.length;
// Reconfigure: the warning should NOT re-fire for the same recipes
// within one process (we already told the operator).
configureGoogle();
expect(warnings.length).toBe(firstCallCount);
} finally {
console.warn = original;
}

// The warning text should match the documented contract.
// No recipe should trip the missing-max_batch_tokens contract warning.
const contractMatch = warnings.filter(w =>
w.includes('[ai.gateway]') && w.includes('declares an embedding touchpoint'),
);
expect(contractMatch.length).toBe(1);
expect(contractMatch.length).toBe(0);

// Voyage declares max_batch_tokens → suppressed. OpenAI is the
// canonical fast-path recipe → also suppressed by id. Both must be
// absent from the warnings.
// google previously warned (it was the sole capless recipe); it now
// declares max_batch_tokens, so it must be suppressed like the rest.
expect(warnings.find(w => w.includes('"google"'))).toBeUndefined();
expect(warnings.find(w => w.includes('"voyage"'))).toBeUndefined();
expect(warnings.find(w => w.includes('"openai"'))).toBeUndefined();
expect(warnings.find(w => w.includes('"google"'))).toBeDefined();
});

test('the guardrail still detects a recipe that forgets the cap (synthetic fixture)', () => {
// The invariant above proves no SHIPPED recipe warns. This proves the
// mechanism that WOULD warn is still armed for a future recipe that
// inherits the embedding touchpoint but forgets max_batch_tokens — the
// "warns once" coverage that used to live on the (now-capped) google
// recipe, now decoupled from any real recipe being left uncapped.
const base: EmbeddingTouchpoint = { models: ['synthetic-embed-001'], default_dims: 768 };
const mk = (id: string, embedding: EmbeddingTouchpoint | undefined): Recipe => ({
id,
name: `synthetic ${id}`,
tier: 'openai-compat',
implementation: 'openai-compatible',
touchpoints: { embedding },
});
// Forgets the cap → the guardrail fires.
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base }))).toBe(true);
// Suppression paths stay silent.
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base, max_batch_tokens: 4096 }))).toBe(false);
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base, no_batch_cap: true }))).toBe(false);
expect(shouldWarnMissingBatchTokens(mk('openai', { ...base }))).toBe(false);
expect(shouldWarnMissingBatchTokens(mk('chat-only', undefined))).toBe(false);
});
});
50 changes: 45 additions & 5 deletions test/ai/no-batch-cap-suppression.serial.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,24 @@
* Coverage:
* - Recipes with `embedding.no_batch_cap: true` suppress the
* missing-max_batch_tokens startup warning (#779)
* - Real-provider recipes without the flag still warn (regression guard)
* - The warning MECHANISM still fires for a recipe that declares an
* embedding touchpoint but forgets the cap (regression guard). Every
* SHIPPED recipe is now capped or opts out (google/gemini included), so
* this is exercised through `shouldWarnMissingBatchTokens` with a
* synthetic capless recipe rather than a real one.
* - google now declares max_batch_tokens → stays silent even when configured.
* - listRecipes returns expected dynamic-cap recipes (ollama, litellm,
* llama-server) all flagged
*/

import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test';
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
import {
configureGateway,
resetGateway,
shouldWarnMissingBatchTokens,
} from '../../src/core/ai/gateway.ts';
import { listRecipes, getRecipe } from '../../src/core/ai/recipes/index.ts';
import type { EmbeddingTouchpoint, Recipe } from '../../src/core/ai/types.ts';

describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warning', () => {
let warnSpy: ReturnType<typeof mock>;
Expand Down Expand Up @@ -52,7 +62,34 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
}
});

test('configureGateway warns for google only when google embedding is configured', () => {
test('warning mechanism still fires for a synthetic capless recipe (guardrail armed)', () => {
// Every SHIPPED recipe is now capped or opts out (google/gemini included),
// so the registry has no live capless fixture. Exercise the guardrail
// directly with a synthetic recipe that inherits the embedding touchpoint
// but forgets max_batch_tokens — the exact v0.27 Voyage-backfill footgun
// this warning exists to catch — plus the suppression paths.
const base: EmbeddingTouchpoint = { models: ['synthetic-embed-001'], default_dims: 768 };
const mk = (id: string, embedding: EmbeddingTouchpoint | undefined): Recipe => ({
id,
name: `synthetic ${id}`,
tier: 'openai-compat',
implementation: 'openai-compatible',
touchpoints: { embedding },
});

// Forgets the cap → warns.
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base }))).toBe(true);
// Declares a cap → silent.
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base, max_batch_tokens: 8192 }))).toBe(false);
// Explicit dynamic-cap opt-out → silent.
expect(shouldWarnMissingBatchTokens(mk('acme-embed', { ...base, no_batch_cap: true }))).toBe(false);
// OpenAI canonical fast path → silent even without a cap.
expect(shouldWarnMissingBatchTokens(mk('openai', { ...base }))).toBe(false);
// No embedding touchpoint at all → silent.
expect(shouldWarnMissingBatchTokens(mk('chat-only', undefined))).toBe(false);
});

test('google now declares max_batch_tokens → no missing-cap warning even when configured', () => {
warnSpy.mockClear();
resetGateway();
configureGateway({ env: {} });
Expand All @@ -62,6 +99,9 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
'google should not warn while OpenAI default is configured',
).toBe(false);

// gemini-embedding-001 has a real per-text cap (2048 tokens, ~1 text/req);
// the google recipe now declares max_batch_tokens, so configuring it must
// NOT trip the guardrail. Regression guard on the cap itself.
warnSpy.mockClear();
resetGateway();
configureGateway({
Expand All @@ -72,8 +112,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
messages = warnSpy.mock.calls.map(c => String(c[0] ?? ''));
expect(
messages.some(m => m.includes('"google"') && m.includes('without max_batch_tokens')),
'google should warn when configured because it has fixed-cap models',
).toBe(true);
'google must stay silent now that it declares max_batch_tokens',
).toBe(false);
});

test('every recipe with empty models[] declares user_provided_models OR has openai-fast-path', () => {
Expand Down
Loading