From 4b5371699aecfea77d0b0dc3ca30bbdf8e5e4fa9 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 13 May 2026 09:39:27 +0000 Subject: [PATCH] fix: deduplicate concurrent models.dev fetch requests fetchModelData() cached the resolved result but not the in-flight promise, so concurrent calls before the first resolved each fired a separate HTTP request. Store the pending promise in a module-level variable and return it for subsequent callers until settlement. --- packages/gateway/src/worker-model.ts | 79 ++++++++++++---------- packages/gateway/test/worker-model.test.ts | 34 ++++++++++ 2 files changed, 79 insertions(+), 34 deletions(-) diff --git a/packages/gateway/src/worker-model.ts b/packages/gateway/src/worker-model.ts index d761162..557e04b 100644 --- a/packages/gateway/src/worker-model.ts +++ b/packages/gateway/src/worker-model.ts @@ -30,6 +30,7 @@ const MODELS_DEV_API = "https://models.dev/api.json"; /** Cached models.dev data: model entries for all supported providers. */ let cachedModelData: Map | null = null; let cachedModelDataAt = 0; +let inflightFetch: Promise> | null = null; const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour /** Providers to fetch pricing data for from models.dev. */ @@ -99,53 +100,62 @@ function fallbackEntry(modelID: string): ModelsDevEntry { * Single HTTP request, cached for 1 hour. Returns a map of * modelID → entry with cost and limit data across all supported providers. */ -export async function fetchModelData(): Promise> { +export function fetchModelData(): Promise> { // Return cache if fresh if (cachedModelData && Date.now() - cachedModelDataAt < CACHE_TTL_MS) { - return cachedModelData; + return Promise.resolve(cachedModelData); } - try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10_000); + // Deduplicate concurrent calls: return the in-flight promise if one exists + if (inflightFetch) return inflightFetch; - const response = await fetch(MODELS_DEV_API, { signal: controller.signal }); - clearTimeout(timeout); + inflightFetch = (async () => { + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 10_000); - if (!response.ok) { - log.warn(`models.dev API failed: ${response.status} ${response.statusText}`); - return cachedModelData ?? new Map(); - } - - const data = (await response.json()) as ModelsDevResponse; - const modelData = new Map(); + const response = await fetch(MODELS_DEV_API, { signal: controller.signal }); + clearTimeout(timeout); - for (const providerName of SUPPORTED_PROVIDERS) { - const providerModels = data[providerName]?.models; - if (!providerModels) { - log.warn(`models.dev API: no ${providerName} provider found`); - continue; + if (!response.ok) { + log.warn(`models.dev API failed: ${response.status} ${response.statusText}`); + return cachedModelData ?? new Map(); } - for (const [modelId, entry] of Object.entries(providerModels)) { - const e: ModelsDevEntry = { ...entry, id: modelId }; - // Compute cache_write cost if not provided (typically 1.25× input price) - if (e.cost && e.cost.cache_write == null && e.cost.input != null) { - e.cost.cache_write = e.cost.input * 1.25; + const data = (await response.json()) as ModelsDevResponse; + const modelData = new Map(); + + for (const providerName of SUPPORTED_PROVIDERS) { + const providerModels = data[providerName]?.models; + if (!providerModels) { + log.warn(`models.dev API: no ${providerName} provider found`); + continue; + } + + for (const [modelId, entry] of Object.entries(providerModels)) { + const e: ModelsDevEntry = { ...entry, id: modelId }; + // Compute cache_write cost if not provided (typically 1.25× input price) + if (e.cost && e.cost.cache_write == null && e.cost.input != null) { + e.cost.cache_write = e.cost.input * 1.25; + } + modelData.set(modelId, e); } - modelData.set(modelId, e); } - } - cachedModelData = modelData; - cachedModelDataAt = Date.now(); + cachedModelData = modelData; + cachedModelDataAt = Date.now(); - log.info(`models.dev: loaded data for ${modelData.size} models across ${SUPPORTED_PROVIDERS.join(", ")}`); - return modelData; - } catch (e) { - log.warn("models.dev API error:", e); - return cachedModelData ?? new Map(); - } + log.info(`models.dev: loaded data for ${modelData.size} models across ${SUPPORTED_PROVIDERS.join(", ")}`); + return modelData; + } catch (e) { + log.warn("models.dev API error:", e); + return cachedModelData ?? new Map(); + } finally { + inflightFetch = null; + } + })(); + + return inflightFetch; } /** @@ -206,6 +216,7 @@ export function getModelEntrySync(modelID: string): ModelsDevEntry { export function clearModelDataCache(): void { cachedModelData = null; cachedModelDataAt = 0; + inflightFetch = null; } // --------------------------------------------------------------------------- diff --git a/packages/gateway/test/worker-model.test.ts b/packages/gateway/test/worker-model.test.ts index 97baf05..1e13d76 100644 --- a/packages/gateway/test/worker-model.test.ts +++ b/packages/gateway/test/worker-model.test.ts @@ -151,6 +151,40 @@ describe("fetchModelData", () => { expect(data.size).toBe(0); }); + test("deduplicates concurrent in-flight requests", async () => { + let callCount = 0; + globalThis.fetch = mock(() => { + callCount++; + return Promise.resolve( + new Response(JSON.stringify(buildModelsDevResponse(DEFAULT_MODELS)), { status: 200 }), + ); + }) as unknown as typeof fetch; + + const [a, b, c] = await Promise.all([ + fetchModelData(), + fetchModelData(), + fetchModelData(), + ]); + + expect(callCount).toBe(1); + expect(a).toBe(b); + expect(b).toBe(c); + }); + + test("deduplicates concurrent calls even on network error", async () => { + let callCount = 0; + globalThis.fetch = mock(() => { + callCount++; + return Promise.reject(new Error("Network error")); + }) as unknown as typeof fetch; + + const [a, b] = await Promise.all([fetchModelData(), fetchModelData()]); + + expect(callCount).toBe(1); + expect(a).toBe(b); + expect(a.size).toBe(0); + }); + test("handles missing providers gracefully", async () => { globalThis.fetch = mock(() => Promise.resolve(