fix(provider): improve prompt caching

package.json

@@ -155,6 +155,11 @@
     "@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
     "@modelcontextprotocol/sdk@1.29.0": "patches/@modelcontextprotocol%2Fsdk@1.29.0.patch",
     "@tanstack/virtual-core@3.17.0": "patches/@tanstack%2Fvirtual-core@3.17.0.patch",
-    "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch"
+    "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch",
+    "@ai-sdk/mistral@3.0.34": "patches/@ai-sdk%2Fmistral@3.0.34.patch",
+    "ai-gateway-provider@3.1.2": "patches/ai-gateway-provider@3.1.2.patch",
+    "@ai-sdk/cohere@3.0.27": "patches/@ai-sdk%2Fcohere@3.0.27.patch",
+    "@ai-sdk/groq@3.0.31": "patches/@ai-sdk%2Fgroq@3.0.31.patch",
+    "@ai-sdk/openai-compatible@2.0.37": "patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch"
   }
 }

packages/core/package.json

@@ -65,14 +65,14 @@
     "@ai-sdk/amazon-bedrock": "4.0.112",
     "@ai-sdk/anthropic": "3.0.82",
     "@ai-sdk/azure": "3.0.49",
-    "@ai-sdk/cerebras": "2.0.41",
+    "@ai-sdk/cerebras": "2.0.60",
     "@ai-sdk/cohere": "3.0.27",
     "@ai-sdk/deepinfra": "2.0.41",
     "@ai-sdk/gateway": "3.0.104",
     "@ai-sdk/google": "3.0.73",
     "@ai-sdk/google-vertex": "4.0.128",
     "@ai-sdk/groq": "3.0.31",
-    "@ai-sdk/mistral": "3.0.27",
+    "@ai-sdk/mistral": "3.0.34",
     "@ai-sdk/openai": "3.0.53",
     "@ai-sdk/openai-compatible": "2.0.41",
     "@ai-sdk/perplexity": "3.0.26",

packages/core/test/provider-cache-key.test.ts

@@ -0,0 +1,91 @@
+import { createCerebras } from "@ai-sdk/cerebras"
+import { createDeepInfra } from "@ai-sdk/deepinfra"
+import { createMistral } from "@ai-sdk/mistral"
+import { createOpenAI } from "@ai-sdk/openai"
+import { describe, expect, test } from "bun:test"
+
+test("Mistral sends promptCacheKey as prompt_cache_key", async () => {
+  let body: Record<string, unknown> | undefined
+  const mockFetch = Object.assign(
+    async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
+      body = JSON.parse(String(init?.body))
+      return Response.json({
+        id: "response-1",
+        created: 0,
+        model: "mistral-large-latest",
+        object: "chat.completion",
+        choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }],
+        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+      })
+    },
+    { preconnect: fetch.preconnect },
+  )
+  const model = createMistral({ apiKey: "test", fetch: mockFetch })("mistral-large-latest")
+
+  await model.doGenerate({
+    prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
+    providerOptions: { mistral: { promptCacheKey: "session-123" } },
+  })
+
+  expect(body?.prompt_cache_key).toBe("session-123")
+})
+
+test("OpenAI Responses sends promptCacheKey as prompt_cache_key", async () => {
+  let body: Record<string, unknown> | undefined
+  const mockFetch = Object.assign(
+    async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
+      body = JSON.parse(String(init?.body))
+      return Response.json({
+        id: "response-1",
+        created_at: 0,
+        model: "gpt-5",
+        object: "response",
+        output: [],
+        usage: { input_tokens: 1, output_tokens: 0 },
+        status: "completed",
+      })
+    },
+    { preconnect: fetch.preconnect },
+  )
+  const model = createOpenAI({ apiKey: "test", fetch: mockFetch }).responses("gpt-5")
+
+  await model.doGenerate({
+    prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
+    providerOptions: { openai: { promptCacheKey: "session-123" } },
+  })
+
+  expect(body?.prompt_cache_key).toBe("session-123")
+})
+
+describe("OpenAI-compatible provider cache keys", () => {
+  for (const provider of [
+    { name: "Cerebras", create: createCerebras, namespace: "cerebras" },
+    { name: "DeepInfra", create: createDeepInfra, namespace: "deepinfra" },
+  ]) {
+    test(`${provider.name} passes prompt_cache_key through`, async () => {
+      let body: Record<string, unknown> | undefined
+      const mockFetch = Object.assign(
+        async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
+          body = JSON.parse(String(init?.body))
+          return Response.json({
+            id: "response-1",
+            created: 0,
+            model: "test-model",
+            object: "chat.completion",
+            choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }],
+            usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+          })
+        },
+        { preconnect: fetch.preconnect },
+      )
+      const model = provider.create({ apiKey: "test", fetch: mockFetch })("test-model")
+
+      await model.doGenerate({
+        prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
+        providerOptions: { [provider.namespace]: { prompt_cache_key: "session-123" } },
+      })
+
+      expect(body?.prompt_cache_key).toBe("session-123")
+    })
+  }
+})

packages/core/test/provider-cache-usage.test.ts

@@ -0,0 +1,69 @@
+import { createCohere } from "@ai-sdk/cohere"
+import { createGroq } from "@ai-sdk/groq"
+import { createTogetherAI } from "@ai-sdk/togetherai"
+import { describe, expect, test } from "bun:test"
+
+const prompt = [{ role: "user" as const, content: [{ type: "text" as const, text: "Hello" }] }]
+
+describe("provider cache usage", () => {
+  test("Cohere reports cached input tokens", async () => {
+    const model = createCohere({
+      apiKey: "test",
+      fetch: mockFetch({
+        generation_id: "response-1",
+        message: { role: "assistant", content: [{ type: "text", text: "Hello" }] },
+        finish_reason: "COMPLETE",
+        usage: {
+          billed_units: { input_tokens: 500, output_tokens: 1 },
+          tokens: { input_tokens: 500, output_tokens: 1, cached_tokens: 400 },
+        },
+      }),
+    })("command-r")
+
+    const result = await model.doGenerate({ prompt })
+    expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined })
+  })
+
+  test("Groq reports cached input tokens", async () => {
+    const model = createGroq({
+      apiKey: "test",
+      fetch: mockFetch({
+        id: "response-1",
+        created: 0,
+        model: "openai/gpt-oss-20b",
+        object: "chat.completion",
+        choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }],
+        usage: {
+          prompt_tokens: 500,
+          completion_tokens: 1,
+          total_tokens: 501,
+          prompt_tokens_details: { cached_tokens: 400 },
+        },
+      }),
+    })("openai/gpt-oss-20b")
+
+    const result = await model.doGenerate({ prompt })
+    expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined })
+  })
+
+  test("Together AI reports flat cached input tokens", async () => {
+    const model = createTogetherAI({
+      apiKey: "test",
+      fetch: mockFetch({
+        id: "response-1",
+        created: 0,
+        model: "moonshotai/Kimi-K2.6",
+        object: "chat.completion",
+        choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }],
+        usage: { prompt_tokens: 500, completion_tokens: 1, total_tokens: 501, cached_tokens: 400 },
+      }),
+    })("moonshotai/Kimi-K2.6")
+
+    const result = await model.doGenerate({ prompt })
+    expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined })
+  })
+})
+
+function mockFetch(response: unknown) {
+  return Object.assign(async () => Response.json(response), { preconnect: fetch.preconnect })
+}

packages/llm/src/protocols/bedrock-converse.ts

@@ -436,21 +436,19 @@ const mapFinishReason = (reason: string): FinishReason => {
   return "unknown"
 }
 
-// AWS Bedrock Converse reports `inputTokens` (inclusive total) with
-// `cacheReadInputTokens` and `cacheWriteInputTokens` as subsets. Pass
-// the total through and derive the non-cached breakdown. Bedrock does
-// not break reasoning out of `outputTokens` for any current model.
+// AWS reports inputTokens separately from cache reads and writes.
+// Bedrock does not break reasoning out of outputTokens for current models.
 const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
   if (!usage) return undefined
   const cacheTotal = (usage.cacheReadInputTokens ?? 0) + (usage.cacheWriteInputTokens ?? 0)
-  const nonCached = ProviderShared.subtractTokens(usage.inputTokens, cacheTotal)
+  const inputTokens = usage.inputTokens === undefined ? undefined : usage.inputTokens + cacheTotal
   return new Usage({
-    inputTokens: usage.inputTokens,
+    inputTokens,
     outputTokens: usage.outputTokens,
-    nonCachedInputTokens: nonCached,
+    nonCachedInputTokens: usage.inputTokens,
     cacheReadInputTokens: usage.cacheReadInputTokens,
     cacheWriteInputTokens: usage.cacheWriteInputTokens,
-    totalTokens: ProviderShared.totalTokens(usage.inputTokens, usage.outputTokens, usage.totalTokens),
+    totalTokens: ProviderShared.totalTokens(inputTokens, usage.outputTokens, usage.totalTokens),
     providerMetadata: { bedrock: usage },
   })
 }

packages/llm/test/provider/bedrock-converse.test.ts

@@ -269,6 +269,39 @@ describe("Bedrock Converse route", () => {
     }),
   )
 
+  it.effect("adds cache reads and writes to Bedrock input usage", () =>
+    Effect.gen(function* () {
+      const body = eventStreamBody(
+        ["messageStart", { role: "assistant" }],
+        ["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "Hello" } }],
+        ["contentBlockStop", { contentBlockIndex: 0 }],
+        ["messageStop", { stopReason: "end_turn" }],
+        [
+          "metadata",
+          {
+            usage: {
+              inputTokens: 5,
+              outputTokens: 2,
+              totalTokens: 12,
+              cacheReadInputTokens: 3,
+              cacheWriteInputTokens: 2,
+            },
+          },
+        ],
+      )
+      const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
+
+      expect(response.usage).toMatchObject({
+        inputTokens: 10,
+        nonCachedInputTokens: 5,
+        cacheReadInputTokens: 3,
+        cacheWriteInputTokens: 2,
+        outputTokens: 2,
+        totalTokens: 12,
+      })
+    }),
+  )
+
   it.effect("assembles streamed tool call input", () =>
     Effect.gen(function* () {
       const body = eventStreamBody(

packages/opencode/package.json

@@ -66,7 +66,7 @@
     "@ai-sdk/google": "3.0.73",
     "@ai-sdk/google-vertex": "4.0.128",
     "@ai-sdk/groq": "3.0.31",
-    "@ai-sdk/mistral": "3.0.27",
+    "@ai-sdk/mistral": "3.0.34",
     "@ai-sdk/openai": "3.0.53",
     "@ai-sdk/openai-compatible": "2.0.41",
     "@ai-sdk/perplexity": "3.0.26",