From 439a5a52817b389be9349698f63335bbc7e8aac2 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 8 Jul 2026 19:22:45 -0500 Subject: [PATCH] fix(provider): improve prompt caching --- bun.lock | 47 ++++--- package.json | 7 +- packages/core/package.json | 4 +- packages/core/test/provider-cache-key.test.ts | 91 +++++++++++++ .../core/test/provider-cache-usage.test.ts | 69 ++++++++++ .../llm/src/protocols/bedrock-converse.ts | 14 +- .../test/provider/bedrock-converse.test.ts | 33 +++++ packages/opencode/package.json | 2 +- packages/opencode/src/provider/transform.ts | 86 ++++++++---- packages/opencode/src/session/llm/request.ts | 2 +- packages/opencode/src/session/session.ts | 3 +- .../test/provider/cf-ai-gateway-e2e.test.ts | 19 ++- .../opencode/test/provider/transform.test.ts | 107 +++++++++++++++ .../opencode/test/session/compaction.test.ts | 19 +++ patches/@ai-sdk%2Fcohere@3.0.27.patch | 126 ++++++++++++++++++ patches/@ai-sdk%2Fgroq@3.0.31.patch | 67 ++++++++++ patches/@ai-sdk%2Fmistral@3.0.34.patch | 84 ++++++++++++ .../@ai-sdk%2Fopenai-compatible@2.0.37.patch | 101 ++++++++++++++ patches/ai-gateway-provider@3.1.2.patch | 34 +++++ 19 files changed, 857 insertions(+), 58 deletions(-) create mode 100644 packages/core/test/provider-cache-key.test.ts create mode 100644 packages/core/test/provider-cache-usage.test.ts create mode 100644 patches/@ai-sdk%2Fcohere@3.0.27.patch create mode 100644 patches/@ai-sdk%2Fgroq@3.0.31.patch create mode 100644 patches/@ai-sdk%2Fmistral@3.0.34.patch create mode 100644 patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch create mode 100644 patches/ai-gateway-provider@3.1.2.patch diff --git a/bun.lock b/bun.lock index 2c91b45ed6de..b0d5b2c54b90 100644 --- a/bun.lock +++ b/bun.lock @@ -293,14 +293,14 @@ "@ai-sdk/amazon-bedrock": "4.0.112", "@ai-sdk/anthropic": "3.0.82", "@ai-sdk/azure": "3.0.49", - "@ai-sdk/cerebras": "2.0.41", + "@ai-sdk/cerebras": "2.0.60", "@ai-sdk/cohere": "3.0.27", "@ai-sdk/deepinfra": "2.0.41", "@ai-sdk/gateway": "3.0.104", "@ai-sdk/google": "3.0.73", "@ai-sdk/google-vertex": "4.0.128", "@ai-sdk/groq": "3.0.31", - "@ai-sdk/mistral": "3.0.27", + "@ai-sdk/mistral": "3.0.34", "@ai-sdk/openai": "3.0.53", "@ai-sdk/openai-compatible": "2.0.41", "@ai-sdk/perplexity": "3.0.26", @@ -575,7 +575,7 @@ "@ai-sdk/google": "3.0.73", "@ai-sdk/google-vertex": "4.0.128", "@ai-sdk/groq": "3.0.31", - "@ai-sdk/mistral": "3.0.27", + "@ai-sdk/mistral": "3.0.34", "@ai-sdk/openai": "3.0.53", "@ai-sdk/openai-compatible": "2.0.41", "@ai-sdk/perplexity": "3.0.26", @@ -1063,19 +1063,24 @@ "tree-sitter-bash", ], "patchedDependencies": { + "ai-gateway-provider@3.1.2": "patches/ai-gateway-provider@3.1.2.patch", + "@ai-sdk/openai-compatible@2.0.37": "patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch", "@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch", "@ai-sdk/xai@3.0.102": "patches/@ai-sdk%2Fxai@3.0.102.patch", "@modelcontextprotocol/sdk@1.29.0": "patches/@modelcontextprotocol%2Fsdk@1.29.0.patch", "gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch", + "@ai-sdk/cohere@3.0.27": "patches/@ai-sdk%2Fcohere@3.0.27.patch", "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch", "@npmcli/agent@4.0.2": "patches/@npmcli%2Fagent@4.0.2.patch", "@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch", "@tanstack/solid-virtual@3.13.28": "patches/@tanstack%2Fsolid-virtual@3.13.28.patch", "solid-js@1.9.10": "patches/solid-js@1.9.10.patch", + "@ai-sdk/groq@3.0.31": "patches/@ai-sdk%2Fgroq@3.0.31.patch", "@ai-sdk/google@3.0.73": "patches/@ai-sdk%2Fgoogle@3.0.73.patch", "@tanstack/virtual-core@3.17.0": "patches/@tanstack%2Fvirtual-core@3.17.0.patch", "pacote@21.5.0": "patches/pacote@21.5.0.patch", + "@ai-sdk/mistral@3.0.34": "patches/@ai-sdk%2Fmistral@3.0.34.patch", }, "overrides": { "@opentui/core": "catalog:", @@ -1173,7 +1178,7 @@ "@ai-sdk/azure": ["@ai-sdk/azure@3.0.49", "", { "dependencies": { "@ai-sdk/openai": "3.0.48", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-wskgAL+OmrHG7by/iWIxEBQCEdc1mDudha/UZav46i0auzdFfsDB/k2rXZaC4/3nWSgMZkxr0W3ncyouEGX/eg=="], - "@ai-sdk/cerebras": ["@ai-sdk/cerebras@2.0.41", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.37", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-kDMEpjaRdRXIUi1EH8WHwLRahyDTYv9SAJnP6VCCeq8X+tVqZbMLCqqxSG5dRknrI65ucjvzQt+FiDKTAa7AHg=="], + "@ai-sdk/cerebras": ["@ai-sdk/cerebras@2.0.60", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.54", "@ai-sdk/provider": "3.0.12", "@ai-sdk/provider-utils": "4.0.33" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Rnok3cThg6awBwaDSyiZpgRpbV7pqxGYrA89LODCo5cuEHeP2h0AM0lLHP7zIkclAdXfOm4wldKi/S2T/DGCOw=="], "@ai-sdk/cohere": ["@ai-sdk/cohere@3.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-OqcCq2PiFY1dbK/0Ck45KuvE8jfdxRuuAE9Y5w46dAk6U+9vPOeg1CDcmR+ncqmrYrhRl3nmyDttyDahyjCzAw=="], @@ -1195,7 +1200,7 @@ "@ai-sdk/groq": ["@ai-sdk/groq@3.0.31", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-XbbugpnFmXGu2TlXiq8KUJskP6/VVbuFcnFIGDzDIB/Chg6XHsNnqrTF80Zxkh0Pd3+NvbM+2Uqrtsndk6bDAg=="], - "@ai-sdk/mistral": ["@ai-sdk/mistral@3.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ZXe7nZQgliDdjz5ufH5RKpHWxbN72AzmzzKGbF/z+0K9GN5tUCnftrQRvTRFHA5jAzTapcm2BEevmGLVbMkW+A=="], + "@ai-sdk/mistral": ["@ai-sdk/mistral@3.0.34", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-HpK28sWGdIfg1vTSScJNtzVdvNRfA4mfCmPmPR+j/MGJ0oAuEJMqxWkL96ZnGPdhZt5KdW09aKovdIe+q2zQ7A=="], "@ai-sdk/openai": ["@ai-sdk/openai@3.0.48", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ALmj/53EXpcRqMbGpPJPP4UOSWw0q4VGpnDo7YctvsynjkrKDmoneDG/1a7VQnSPYHnJp6tTRMf5ZdxZ5whulg=="], @@ -5641,7 +5646,11 @@ "@ai-sdk/azure/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], - "@ai-sdk/cerebras/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], + "@ai-sdk/cerebras/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.54", "", { "dependencies": { "@ai-sdk/provider": "3.0.12", "@ai-sdk/provider-utils": "4.0.33" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-OyXt0zK8y2/ZIyWlbxTv2r1M7AK227S+Gl4BYOEF42q0wz1n5m4fwR8L4Fy/MQ4Ho6xje47MPsFcRdIqIyP6Rw=="], + + "@ai-sdk/cerebras/@ai-sdk/provider": ["@ai-sdk/provider@3.0.12", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-sj9DWTJ2Ze0WR9qsiOPqoqzNx3OxL6iMxHImbhvoe9qOspekbzxNDMiJ4TIGfYHYh9w4OmBjz3prvqhzTi96+Q=="], + + "@ai-sdk/cerebras/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.33", "", { "dependencies": { "@ai-sdk/provider": "3.0.12", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-nJ0bAfegMAIJtrzMJtbzer1cS3nb7c7DsyU1S4nrPm7ZU0Mn6SBBZv5IGZZGTbpWTJwqKTSPeZJTXalbAxt1BA=="], "@ai-sdk/cohere/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], @@ -5679,7 +5688,9 @@ "@ai-sdk/groq/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], - "@ai-sdk/mistral/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], + "@ai-sdk/mistral/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="], + + "@ai-sdk/mistral/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.26", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CsKNLKsOpvPujRlIYvoz+Ybw+kGn7J4/fIZa/58+R7iWLLfwn6ifE2G6Yq8K9XvH/I/3bzaDAJ3NhRwEMsLBKQ=="], "@ai-sdk/openai/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], @@ -6145,6 +6156,10 @@ "ai-gateway-provider/@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.78", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-0OY12G20cUt6iU6htpEA1491Oz++NVxZxlmWGX4B7rSbeZ5pnDmOu6YtW9BKzdZlNx5Gn23i6WMxyZFoMKNcgA=="], + "ai-gateway-provider/@ai-sdk/cerebras": ["@ai-sdk/cerebras@2.0.41", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.37", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-kDMEpjaRdRXIUi1EH8WHwLRahyDTYv9SAJnP6VCCeq8X+tVqZbMLCqqxSG5dRknrI65ucjvzQt+FiDKTAa7AHg=="], + + "ai-gateway-provider/@ai-sdk/mistral": ["@ai-sdk/mistral@3.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ZXe7nZQgliDdjz5ufH5RKpHWxbN72AzmzzKGbF/z+0K9GN5tUCnftrQRvTRFHA5jAzTapcm2BEevmGLVbMkW+A=="], + "ai-gateway-provider/@ai-sdk/openai": ["@ai-sdk/openai@3.0.53", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ=="], "ai-gateway-provider/@ai-sdk/xai": ["@ai-sdk/xai@3.0.82", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.41", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-A0VFMufnVf4wODcT3SPQUUzvYXiIO1VhFuXj9r6z/vP4rlo+QRDPw3WSTchcz93ROQWSfBE3I6Szqz342OHi5w=="], @@ -6369,8 +6384,6 @@ "nypm/tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], - "opencode/@ai-sdk/cerebras": ["@ai-sdk/cerebras@2.0.60", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.54", "@ai-sdk/provider": "3.0.12", "@ai-sdk/provider-utils": "4.0.33" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Rnok3cThg6awBwaDSyiZpgRpbV7pqxGYrA89LODCo5cuEHeP2h0AM0lLHP7zIkclAdXfOm4wldKi/S2T/DGCOw=="], - "opencode/@ai-sdk/openai": ["@ai-sdk/openai@3.0.53", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ=="], "opencode/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.41", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-kNAGINk71AlOXx10Dq/PXw4t/9XjdK8uxfpVElRwtSFMdeSiLVt58p9TPx4/FJD+hxZuVhvxYj9r42osxWq79g=="], @@ -6947,6 +6960,10 @@ "ai-gateway-provider/@ai-sdk/anthropic/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="], + "ai-gateway-provider/@ai-sdk/cerebras/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], + + "ai-gateway-provider/@ai-sdk/mistral/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.21", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MtFUYI1/8mgDvRmaBDjbLJPFFrMG777AvSgyIFQtZHIMzm88R/12vYBBpnk7pfiWLFE1DSZzY4WDYzGbKAcmiw=="], + "ai-gateway-provider/@ai-sdk/xai/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.41", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-kNAGINk71AlOXx10Dq/PXw4t/9XjdK8uxfpVElRwtSFMdeSiLVt58p9TPx4/FJD+hxZuVhvxYj9r42osxWq79g=="], "ajv-keywords/ajv/json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], @@ -7093,12 +7110,6 @@ "opencode-gitlab-auth/open/wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], - "opencode/@ai-sdk/cerebras/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.54", "", { "dependencies": { "@ai-sdk/provider": "3.0.12", "@ai-sdk/provider-utils": "4.0.33" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-OyXt0zK8y2/ZIyWlbxTv2r1M7AK227S+Gl4BYOEF42q0wz1n5m4fwR8L4Fy/MQ4Ho6xje47MPsFcRdIqIyP6Rw=="], - - "opencode/@ai-sdk/cerebras/@ai-sdk/provider": ["@ai-sdk/provider@3.0.12", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-sj9DWTJ2Ze0WR9qsiOPqoqzNx3OxL6iMxHImbhvoe9qOspekbzxNDMiJ4TIGfYHYh9w4OmBjz3prvqhzTi96+Q=="], - - "opencode/@ai-sdk/cerebras/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.33", "", { "dependencies": { "@ai-sdk/provider": "3.0.12", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-nJ0bAfegMAIJtrzMJtbzer1cS3nb7c7DsyU1S4nrPm7ZU0Mn6SBBZv5IGZZGTbpWTJwqKTSPeZJTXalbAxt1BA=="], - "p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "pkg-dir/find-up/locate-path": ["locate-path@5.0.0", "", { "dependencies": { "p-locate": "^4.1.0" } }, "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g=="], @@ -7363,6 +7374,10 @@ "ai-gateway-provider/@ai-sdk/anthropic/@ai-sdk/provider-utils/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "ai-gateway-provider/@ai-sdk/cerebras/@ai-sdk/provider-utils/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + + "ai-gateway-provider/@ai-sdk/mistral/@ai-sdk/provider-utils/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "ansi-align/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "app-builder-lib/@electron/get/fs-extra/universalify": ["universalify@0.1.2", "", {}, "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="], @@ -7431,8 +7446,6 @@ "js-beautify/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], - "opencode/@ai-sdk/cerebras/@ai-sdk/provider-utils/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - "pkg-dir/find-up/locate-path/p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], "pkg-up/find-up/locate-path/p-locate": ["p-locate@3.0.0", "", { "dependencies": { "p-limit": "^2.0.0" } }, "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ=="], diff --git a/package.json b/package.json index 0d394011966a..0d75b4e9288a 100644 --- a/package.json +++ b/package.json @@ -155,6 +155,11 @@ "@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch", "@modelcontextprotocol/sdk@1.29.0": "patches/@modelcontextprotocol%2Fsdk@1.29.0.patch", "@tanstack/virtual-core@3.17.0": "patches/@tanstack%2Fvirtual-core@3.17.0.patch", - "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch" + "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch", + "@ai-sdk/mistral@3.0.34": "patches/@ai-sdk%2Fmistral@3.0.34.patch", + "ai-gateway-provider@3.1.2": "patches/ai-gateway-provider@3.1.2.patch", + "@ai-sdk/cohere@3.0.27": "patches/@ai-sdk%2Fcohere@3.0.27.patch", + "@ai-sdk/groq@3.0.31": "patches/@ai-sdk%2Fgroq@3.0.31.patch", + "@ai-sdk/openai-compatible@2.0.37": "patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch" } } diff --git a/packages/core/package.json b/packages/core/package.json index 0680fac03d92..1b69c9362bad 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -65,14 +65,14 @@ "@ai-sdk/amazon-bedrock": "4.0.112", "@ai-sdk/anthropic": "3.0.82", "@ai-sdk/azure": "3.0.49", - "@ai-sdk/cerebras": "2.0.41", + "@ai-sdk/cerebras": "2.0.60", "@ai-sdk/cohere": "3.0.27", "@ai-sdk/deepinfra": "2.0.41", "@ai-sdk/gateway": "3.0.104", "@ai-sdk/google": "3.0.73", "@ai-sdk/google-vertex": "4.0.128", "@ai-sdk/groq": "3.0.31", - "@ai-sdk/mistral": "3.0.27", + "@ai-sdk/mistral": "3.0.34", "@ai-sdk/openai": "3.0.53", "@ai-sdk/openai-compatible": "2.0.41", "@ai-sdk/perplexity": "3.0.26", diff --git a/packages/core/test/provider-cache-key.test.ts b/packages/core/test/provider-cache-key.test.ts new file mode 100644 index 000000000000..279f75dea97b --- /dev/null +++ b/packages/core/test/provider-cache-key.test.ts @@ -0,0 +1,91 @@ +import { createCerebras } from "@ai-sdk/cerebras" +import { createDeepInfra } from "@ai-sdk/deepinfra" +import { createMistral } from "@ai-sdk/mistral" +import { createOpenAI } from "@ai-sdk/openai" +import { describe, expect, test } from "bun:test" + +test("Mistral sends promptCacheKey as prompt_cache_key", async () => { + let body: Record | undefined + const mockFetch = Object.assign( + async (_input: Parameters[0], init?: RequestInit) => { + body = JSON.parse(String(init?.body)) + return Response.json({ + id: "response-1", + created: 0, + model: "mistral-large-latest", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }) + }, + { preconnect: fetch.preconnect }, + ) + const model = createMistral({ apiKey: "test", fetch: mockFetch })("mistral-large-latest") + + await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }], + providerOptions: { mistral: { promptCacheKey: "session-123" } }, + }) + + expect(body?.prompt_cache_key).toBe("session-123") +}) + +test("OpenAI Responses sends promptCacheKey as prompt_cache_key", async () => { + let body: Record | undefined + const mockFetch = Object.assign( + async (_input: Parameters[0], init?: RequestInit) => { + body = JSON.parse(String(init?.body)) + return Response.json({ + id: "response-1", + created_at: 0, + model: "gpt-5", + object: "response", + output: [], + usage: { input_tokens: 1, output_tokens: 0 }, + status: "completed", + }) + }, + { preconnect: fetch.preconnect }, + ) + const model = createOpenAI({ apiKey: "test", fetch: mockFetch }).responses("gpt-5") + + await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }], + providerOptions: { openai: { promptCacheKey: "session-123" } }, + }) + + expect(body?.prompt_cache_key).toBe("session-123") +}) + +describe("OpenAI-compatible provider cache keys", () => { + for (const provider of [ + { name: "Cerebras", create: createCerebras, namespace: "cerebras" }, + { name: "DeepInfra", create: createDeepInfra, namespace: "deepinfra" }, + ]) { + test(`${provider.name} passes prompt_cache_key through`, async () => { + let body: Record | undefined + const mockFetch = Object.assign( + async (_input: Parameters[0], init?: RequestInit) => { + body = JSON.parse(String(init?.body)) + return Response.json({ + id: "response-1", + created: 0, + model: "test-model", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }) + }, + { preconnect: fetch.preconnect }, + ) + const model = provider.create({ apiKey: "test", fetch: mockFetch })("test-model") + + await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }], + providerOptions: { [provider.namespace]: { prompt_cache_key: "session-123" } }, + }) + + expect(body?.prompt_cache_key).toBe("session-123") + }) + } +}) diff --git a/packages/core/test/provider-cache-usage.test.ts b/packages/core/test/provider-cache-usage.test.ts new file mode 100644 index 000000000000..09bb3868c95a --- /dev/null +++ b/packages/core/test/provider-cache-usage.test.ts @@ -0,0 +1,69 @@ +import { createCohere } from "@ai-sdk/cohere" +import { createGroq } from "@ai-sdk/groq" +import { createTogetherAI } from "@ai-sdk/togetherai" +import { describe, expect, test } from "bun:test" + +const prompt = [{ role: "user" as const, content: [{ type: "text" as const, text: "Hello" }] }] + +describe("provider cache usage", () => { + test("Cohere reports cached input tokens", async () => { + const model = createCohere({ + apiKey: "test", + fetch: mockFetch({ + generation_id: "response-1", + message: { role: "assistant", content: [{ type: "text", text: "Hello" }] }, + finish_reason: "COMPLETE", + usage: { + billed_units: { input_tokens: 500, output_tokens: 1 }, + tokens: { input_tokens: 500, output_tokens: 1, cached_tokens: 400 }, + }, + }), + })("command-r") + + const result = await model.doGenerate({ prompt }) + expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined }) + }) + + test("Groq reports cached input tokens", async () => { + const model = createGroq({ + apiKey: "test", + fetch: mockFetch({ + id: "response-1", + created: 0, + model: "openai/gpt-oss-20b", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }], + usage: { + prompt_tokens: 500, + completion_tokens: 1, + total_tokens: 501, + prompt_tokens_details: { cached_tokens: 400 }, + }, + }), + })("openai/gpt-oss-20b") + + const result = await model.doGenerate({ prompt }) + expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined }) + }) + + test("Together AI reports flat cached input tokens", async () => { + const model = createTogetherAI({ + apiKey: "test", + fetch: mockFetch({ + id: "response-1", + created: 0, + model: "moonshotai/Kimi-K2.6", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }], + usage: { prompt_tokens: 500, completion_tokens: 1, total_tokens: 501, cached_tokens: 400 }, + }), + })("moonshotai/Kimi-K2.6") + + const result = await model.doGenerate({ prompt }) + expect(result.usage.inputTokens).toEqual({ total: 500, noCache: 100, cacheRead: 400, cacheWrite: undefined }) + }) +}) + +function mockFetch(response: unknown) { + return Object.assign(async () => Response.json(response), { preconnect: fetch.preconnect }) +} diff --git a/packages/llm/src/protocols/bedrock-converse.ts b/packages/llm/src/protocols/bedrock-converse.ts index c447a1a39d34..a09e20c22050 100644 --- a/packages/llm/src/protocols/bedrock-converse.ts +++ b/packages/llm/src/protocols/bedrock-converse.ts @@ -436,21 +436,19 @@ const mapFinishReason = (reason: string): FinishReason => { return "unknown" } -// AWS Bedrock Converse reports `inputTokens` (inclusive total) with -// `cacheReadInputTokens` and `cacheWriteInputTokens` as subsets. Pass -// the total through and derive the non-cached breakdown. Bedrock does -// not break reasoning out of `outputTokens` for any current model. +// AWS reports inputTokens separately from cache reads and writes. +// Bedrock does not break reasoning out of outputTokens for current models. const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => { if (!usage) return undefined const cacheTotal = (usage.cacheReadInputTokens ?? 0) + (usage.cacheWriteInputTokens ?? 0) - const nonCached = ProviderShared.subtractTokens(usage.inputTokens, cacheTotal) + const inputTokens = usage.inputTokens === undefined ? undefined : usage.inputTokens + cacheTotal return new Usage({ - inputTokens: usage.inputTokens, + inputTokens, outputTokens: usage.outputTokens, - nonCachedInputTokens: nonCached, + nonCachedInputTokens: usage.inputTokens, cacheReadInputTokens: usage.cacheReadInputTokens, cacheWriteInputTokens: usage.cacheWriteInputTokens, - totalTokens: ProviderShared.totalTokens(usage.inputTokens, usage.outputTokens, usage.totalTokens), + totalTokens: ProviderShared.totalTokens(inputTokens, usage.outputTokens, usage.totalTokens), providerMetadata: { bedrock: usage }, }) } diff --git a/packages/llm/test/provider/bedrock-converse.test.ts b/packages/llm/test/provider/bedrock-converse.test.ts index 46657331a344..c96a599d7625 100644 --- a/packages/llm/test/provider/bedrock-converse.test.ts +++ b/packages/llm/test/provider/bedrock-converse.test.ts @@ -269,6 +269,39 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("adds cache reads and writes to Bedrock input usage", () => + Effect.gen(function* () { + const body = eventStreamBody( + ["messageStart", { role: "assistant" }], + ["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "Hello" } }], + ["contentBlockStop", { contentBlockIndex: 0 }], + ["messageStop", { stopReason: "end_turn" }], + [ + "metadata", + { + usage: { + inputTokens: 5, + outputTokens: 2, + totalTokens: 12, + cacheReadInputTokens: 3, + cacheWriteInputTokens: 2, + }, + }, + ], + ) + const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body))) + + expect(response.usage).toMatchObject({ + inputTokens: 10, + nonCachedInputTokens: 5, + cacheReadInputTokens: 3, + cacheWriteInputTokens: 2, + outputTokens: 2, + totalTokens: 12, + }) + }), + ) + it.effect("assembles streamed tool call input", () => Effect.gen(function* () { const body = eventStreamBody( diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 401803c37cd8..794690f9e31e 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -66,7 +66,7 @@ "@ai-sdk/google": "3.0.73", "@ai-sdk/google-vertex": "4.0.128", "@ai-sdk/groq": "3.0.31", - "@ai-sdk/mistral": "3.0.27", + "@ai-sdk/mistral": "3.0.34", "@ai-sdk/openai": "3.0.53", "@ai-sdk/openai-compatible": "2.0.41", "@ai-sdk/perplexity": "3.0.26", diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts index 8b5bcab6117a..d1679d607d77 100644 --- a/packages/opencode/src/provider/transform.ts +++ b/packages/opencode/src/provider/transform.ts @@ -46,6 +46,28 @@ function sdkKey(npm: string): string | undefined { return "vertex" case "@ai-sdk/google": return "google" + case "@ai-sdk/alibaba": + return "alibaba" + case "@ai-sdk/cerebras": + return "cerebras" + case "@ai-sdk/cohere": + return "cohere" + case "@ai-sdk/deepinfra": + return "deepinfra" + case "@ai-sdk/groq": + return "groq" + case "@ai-sdk/mistral": + return "mistral" + case "@ai-sdk/perplexity": + return "perplexity" + case "@ai-sdk/togetherai": + return "togetherai" + case "@ai-sdk/vercel": + return "vercel" + case "@ai-sdk/xai": + return "xai" + case "venice-ai-sdk-provider": + return "venice" case "@ai-sdk/gateway": return "gateway" case "@openrouter/ai-sdk-provider": @@ -430,6 +452,9 @@ function mapProviderOptions( export function message(msgs: ModelMessage[], model: Provider.Model, options: Record) { msgs = unsupportedParts(msgs, model) msgs = normalizeMessages(msgs, model, options) + const usesAnthropicAutomaticCaching = + options.cacheControl !== undefined && + (model.api.npm === "@ai-sdk/anthropic" || model.api.npm === "@ai-sdk/google-vertex/anthropic") if ( (model.providerID === "anthropic" || model.providerID === "google-vertex-anthropic" || @@ -439,7 +464,8 @@ export function message(msgs: ModelMessage[], model: Provider.Model, options: Re model.id.includes("claude") || model.api.npm === "@ai-sdk/anthropic" || model.api.npm === "@ai-sdk/alibaba") && - model.api.npm !== "@ai-sdk/gateway" + model.api.npm !== "@ai-sdk/gateway" && + !usesAnthropicAutomaticCaching ) { msgs = applyCaching(msgs, model) } @@ -1077,7 +1103,7 @@ export function options(input: { sessionID: string providerOptions?: Record }): Record { - const result: Record = {} + const result: Record = cachingOptions(input) if ( input.model.api.npm === "@ai-sdk/google-vertex/anthropic" || @@ -1098,7 +1124,6 @@ export function options(input: { if (input.model.api.npm === "@ai-sdk/azure") { result["store"] = false - result["promptCacheKey"] = input.sessionID } if (input.model.api.npm === "@openrouter/ai-sdk-provider" || input.model.api.npm === "@llmgateway/ai-sdk-provider") { @@ -1127,13 +1152,6 @@ export function options(input: { } } - if ( - input.providerOptions?.setCacheKey !== false && - (input.model.providerID === "openai" || input.model.api.npm === "@ai-sdk/xai" || input.providerOptions?.setCacheKey) - ) { - result["promptCacheKey"] = input.sessionID - } - if (input.model.api.npm === "@ai-sdk/google" || input.model.api.npm === "@ai-sdk/google-vertex") { if (input.model.capabilities.reasoning) { result["thinkingConfig"] = { @@ -1209,37 +1227,55 @@ export function options(input: { result["textVerbosity"] = "low" } - if (input.model.providerID.startsWith("opencode")) { + if (input.model.providerID.startsWith("opencode") && input.providerOptions?.setCacheKey !== false) { result["promptCacheKey"] = input.sessionID result["include"] = INCLUDE_ENCRYPTED_REASONING result["reasoningSummary"] = "auto" } } - if (input.model.providerID === "venice") { - result["promptCacheKey"] = input.sessionID + return result +} + +function cachingOptions(input: { + model: Provider.Model + sessionID: string + providerOptions?: Record +}): Record { + if (input.providerOptions?.setCacheKey === false) { + if (input.model.api.npm === "@ai-sdk/gateway") return { gateway: { caching: "auto" } } + return {} } - if (input.model.providerID === "openrouter") { - result["prompt_cache_key"] = input.sessionID + if (input.model.api.npm === "@ai-sdk/deepinfra" || input.model.api.npm === "@ai-sdk/cerebras") { + return { prompt_cache_key: input.sessionID } } - if (input.model.api.npm === "@ai-sdk/gateway") { - result["gateway"] = { - caching: "auto", - } + + if ( + input.model.api.npm === "@ai-sdk/openai" || + (input.model.providerID === "openai" && input.model.api.npm !== "@ai-sdk/openai-compatible") || + input.model.api.npm === "@ai-sdk/azure" || + input.model.api.npm === "@ai-sdk/xai" || + input.model.api.npm === "@ai-sdk/mistral" || + input.model.api.npm === "venice-ai-sdk-provider" || + input.providerOptions?.setCacheKey === true + ) { + return { promptCacheKey: input.sessionID } } - return result + if (input.model.api.npm === "@ai-sdk/gateway") return { gateway: { caching: "auto" } } + return {} } -export function smallOptions(model: Provider.Model) { +export function smallOptions(model: Provider.Model, sessionID?: string, providerOptions?: Record) { const small = Object.values(model.variants ?? {})[0] ?? {} + const caching = sessionID ? cachingOptions({ model, sessionID, providerOptions }) : {} if ( model.providerID === "openai" || model.api.npm === "@ai-sdk/openai" || model.api.npm === "@ai-sdk/github-copilot" ) { - const base = { store: false } + const base = { ...caching, store: false } return mergeDeep(base, small) } if (model.providerID === "openrouter" || model.providerID === "llmgateway") { @@ -1249,11 +1285,11 @@ export function smallOptions(model: Provider.Model) { } if (model.providerID === "venice") { - if (Object.keys(small).length > 0) return small - return { veniceParameters: { disableThinking: true } } + if (Object.keys(small).length > 0) return mergeDeep(caching, small) + return mergeDeep(caching, { veniceParameters: { disableThinking: true } }) } - return small + return mergeDeep(caching, small) } // Maps model ID prefix to provider slug used in providerOptions. diff --git a/packages/opencode/src/session/llm/request.ts b/packages/opencode/src/session/llm/request.ts index 4f93411107df..ad737e059cb8 100644 --- a/packages/opencode/src/session/llm/request.ts +++ b/packages/opencode/src/session/llm/request.ts @@ -82,7 +82,7 @@ export const prepare = Effect.fn("LLMRequestPrep.prepare")(function* (input: Pre ? input.model.variants[input.user.model.variant] : {} const base = input.small - ? ProviderTransform.smallOptions(input.model) + ? ProviderTransform.smallOptions(input.model, input.sessionID, input.provider.options) : ProviderTransform.options({ model: input.model, sessionID: input.sessionID, diff --git a/packages/opencode/src/session/session.ts b/packages/opencode/src/session/session.ts index de8c3dc4cbd1..2eb6d1990a73 100644 --- a/packages/opencode/src/session/session.ts +++ b/packages/opencode/src/session/session.ts @@ -387,6 +387,7 @@ export const getUsage = (input: { model: Provider.Model; usage: Usage; metadata? ? input.model.cost.experimentalOver200K : input.model.cost) const totalNanoAiu = input.metadata?.["copilot"]?.["totalNanoAiu"] + const cacheReadCost = input.model.api.npm === "@ai-sdk/cerebras" ? costInfo?.input : costInfo?.cache?.read return { cost: typeof totalNanoAiu === "number" && Number.isFinite(totalNanoAiu) && totalNanoAiu >= 0 @@ -395,7 +396,7 @@ export const getUsage = (input: { model: Provider.Model; usage: Usage; metadata? new Decimal(0) .add(new Decimal(tokens.input).mul(costInfo?.input ?? 0).div(1_000_000)) .add(new Decimal(tokens.output).mul(costInfo?.output ?? 0).div(1_000_000)) - .add(new Decimal(tokens.cache.read).mul(costInfo?.cache?.read ?? 0).div(1_000_000)) + .add(new Decimal(tokens.cache.read).mul(cacheReadCost ?? 0).div(1_000_000)) .add(new Decimal(tokens.cache.write).mul(costInfo?.cache?.write ?? 0).div(1_000_000)) // TODO: update models.dev to have better pricing model, for now: // charge reasoning tokens at the same rate as output tokens diff --git a/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts b/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts index f062868c427a..750ea46de774 100644 --- a/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts +++ b/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts @@ -16,7 +16,7 @@ import type * as Provider from "@/provider/provider" import { ProviderV2 } from "@opencode-ai/core/provider" import { ModelV2 } from "@opencode-ai/core/model" -type Captured = { url: string; outerBody: unknown } +type Captured = { url: string; headers: Headers; outerBody: unknown } type ProviderOptions = Record> const realFetch = globalThis.fetch @@ -32,7 +32,7 @@ beforeEach(() => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url if (url.startsWith("https://gateway.ai.cloudflare.com/")) { const bodyText = typeof init?.body === "string" ? init.body : "" - captured = { url, outerBody: bodyText ? JSON.parse(bodyText) : null } + captured = { url, headers: new Headers(init?.headers), outerBody: bodyText ? JSON.parse(bodyText) : null } return new Response( JSON.stringify({ id: "chatcmpl-test", @@ -129,4 +129,19 @@ describe("cf-ai-gateway end-to-end (regression: #24432)", () => { }) expect(upstream?.reasoning_effort).toBeUndefined() }) + + test("uses current Cloudflare cache headers", async () => { + const aigateway = createAiGateway({ + accountId: "test", + gateway: "test", + apiKey: "test", + options: { cacheTtl: 300, skipCache: true }, + }) + await generateText({ model: aigateway(createUnified()("openai/gpt-5.4")), prompt: "hi" }) + + expect(captured?.headers.get("cf-aig-cache-ttl")).toBe("300") + expect(captured?.headers.get("cf-aig-skip-cache")).toBe("true") + expect(captured?.headers.get("cf-cache-ttl")).toBeNull() + expect(captured?.headers.get("cf-skip-cache")).toBeNull() + }) }) diff --git a/packages/opencode/test/provider/transform.test.ts b/packages/opencode/test/provider/transform.test.ts index 8f2be02a6747..1755c084d4e4 100644 --- a/packages/opencode/test/provider/transform.test.ts +++ b/packages/opencode/test/provider/transform.test.ts @@ -87,6 +87,32 @@ describe("ProviderTransform.options - setCacheKey", () => { expect(result.promptCacheKey).toBe(sessionID) }) + test("should set promptCacheKey for the OpenAI SDK regardless of provider ID", () => { + const result = ProviderTransform.options({ + model: { + ...mockModel, + providerID: "custom-openai", + api: { id: "gpt-5", url: "https://example.com", npm: "@ai-sdk/openai" }, + }, + sessionID, + providerOptions: {}, + }) + expect(result.promptCacheKey).toBe(sessionID) + }) + + test("should not set promptCacheKey for the OpenAI-compatible SDK by provider name", () => { + const result = ProviderTransform.options({ + model: { + ...mockModel, + providerID: "openai", + api: { id: "gpt-5", url: "https://example.com", npm: "@ai-sdk/openai-compatible" }, + }, + sessionID, + providerOptions: {}, + }) + expect(result.promptCacheKey).toBeUndefined() + }) + test("should not set promptCacheKey for openai when explicitly disabled", () => { const openaiModel = { ...mockModel, @@ -171,6 +197,55 @@ describe("ProviderTransform.options - setCacheKey", () => { providerOptions: {}, }) expect(result.store).toBe(false) + expect(result.promptCacheKey).toBe(sessionID) + }) + + test("should disable the Azure cache key without disabling store=false", () => { + const result = ProviderTransform.options({ + model: { + ...mockModel, + providerID: "azure", + api: { id: "gpt-5", url: "https://azure.com", npm: "@ai-sdk/azure" }, + }, + sessionID, + providerOptions: { setCacheKey: false }, + }) + expect(result.store).toBe(false) + expect(result.promptCacheKey).toBeUndefined() + }) + + for (const npm of ["@ai-sdk/deepinfra", "@ai-sdk/cerebras"]) { + test(`should set the snake-case cache key for ${npm}`, () => { + const result = ProviderTransform.options({ + model: { ...mockModel, providerID: "custom", api: { ...mockModel.api, npm } }, + sessionID, + providerOptions: {}, + }) + expect(result.prompt_cache_key).toBe(sessionID) + expect(result.promptCacheKey).toBeUndefined() + }) + } + + test("should set promptCacheKey for the Mistral SDK", () => { + const result = ProviderTransform.options({ + model: { ...mockModel, providerID: "custom", api: { ...mockModel.api, npm: "@ai-sdk/mistral" } }, + sessionID, + providerOptions: {}, + }) + expect(result.promptCacheKey).toBe(sessionID) + }) + + test("should not send an undocumented OpenRouter prompt_cache_key", () => { + const result = ProviderTransform.options({ + model: { + ...mockModel, + providerID: "openrouter", + api: { ...mockModel.api, npm: "@openrouter/ai-sdk-provider" }, + }, + sessionID, + providerOptions: {}, + }) + expect(result.prompt_cache_key).toBeUndefined() }) }) @@ -663,6 +738,17 @@ describe("ProviderTransform.providerOptions", () => { }) }) + test("uses canonical sdk key for custom xAI models", () => { + const model = createModel({ + providerID: "my-xai", + api: { id: "grok-4", url: "https://api.x.ai", npm: "@ai-sdk/xai" }, + }) + + expect(ProviderTransform.providerOptions(model, { promptCacheKey: "session" })).toEqual({ + xai: { promptCacheKey: "session" }, + }) + }) + test("forces reasoning for custom OpenAI package models with explicit effort", () => { const model = createModel({ providerID: "meta", @@ -2924,6 +3010,20 @@ describe("ProviderTransform.message - cache control on gateway", () => { }) }) + test("does not add explicit breakpoints when Anthropic automatic caching is enabled", () => { + const model = createModel({ + providerID: "anthropic", + api: { id: "claude-sonnet-4", url: "https://api.anthropic.com", npm: "@ai-sdk/anthropic" }, + }) + const msgs = [ + { role: "system", content: "You are a helpful assistant" }, + { role: "user", content: "Hello" }, + ] as any[] + + const result = ProviderTransform.message(msgs, model, { cacheControl: { type: "ephemeral" } }) as any[] + expect(result.every((message) => message.providerOptions === undefined)).toBe(true) + }) + test("google-vertex-anthropic applies cache control", () => { const model = createModel({ providerID: "google-vertex-anthropic", @@ -4686,6 +4786,13 @@ describe("ProviderTransform.smallOptions - gpt-5 chat/search", () => { expect(ProviderTransform.smallOptions(createModel(testCase.id))).toEqual(testCase.options) }) } + + test("includes the OpenAI cache key in small requests", () => { + expect(ProviderTransform.smallOptions(createModel("gpt-5-chat-latest"), "session-123")).toEqual({ + store: false, + promptCacheKey: "session-123", + }) + }) }) test("ProviderTransform.smallOptions preserves the weakest OpenRouter reasoning effort", () => { diff --git a/packages/opencode/test/session/compaction.test.ts b/packages/opencode/test/session/compaction.test.ts index 4a4210cf08bc..dbb5e4fec0ce 100644 --- a/packages/opencode/test/session/compaction.test.ts +++ b/packages/opencode/test/session/compaction.test.ts @@ -1564,6 +1564,25 @@ describe("SessionNs.getUsage", () => { expect(result.tokens.cache.read).toBe(200) }) + test("charges Cerebras cache reads at the normal input rate", () => { + const result = SessionNs.getUsage({ + model: createModel({ + context: 100_000, + output: 32_000, + npm: "@ai-sdk/cerebras", + cost: { input: 10, output: 0, cache: { read: 0, write: 0 } }, + }), + usage: usage({ + inputTokens: 1_000_000, + outputTokens: 0, + totalTokens: 1_000_000, + cacheReadInputTokens: 200_000, + }), + }) + + expect(result.cost).toBe(10) + }) + test("handles anthropic cache write metadata", () => { const model = createModel({ context: 100_000, output: 32_000 }) const result = SessionNs.getUsage({ diff --git a/patches/@ai-sdk%2Fcohere@3.0.27.patch b/patches/@ai-sdk%2Fcohere@3.0.27.patch new file mode 100644 index 000000000000..9867116d41ae --- /dev/null +++ b/patches/@ai-sdk%2Fcohere@3.0.27.patch @@ -0,0 +1,126 @@ +diff --git a/dist/index.js b/dist/index.js +index ce8d4ae802f5909937840b856866c2d735800212..d448e921d51cb0a4f752cdad09080b21d4d8e8a9 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -139,11 +139,12 @@ function convertCohereUsage(tokens) { + } + const inputTokens = tokens.input_tokens; + const outputTokens = tokens.output_tokens; ++ const cacheReadTokens = tokens.cached_tokens ?? 0; + return { + inputTokens: { + total: inputTokens, +- noCache: inputTokens, +- cacheRead: void 0, ++ noCache: Math.max(0, inputTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || void 0, + cacheWrite: void 0 + }, + outputTokens: { +@@ -672,7 +673,8 @@ var cohereChatResponseSchema = import_v43.z.object({ + }), + tokens: import_v43.z.object({ + input_tokens: import_v43.z.number(), +- output_tokens: import_v43.z.number() ++ output_tokens: import_v43.z.number(), ++ cached_tokens: import_v43.z.number().optional() + }) + }) + }); +@@ -732,7 +734,8 @@ var cohereChatChunkSchema = import_v43.z.discriminatedUnion("type", [ + usage: import_v43.z.object({ + tokens: import_v43.z.object({ + input_tokens: import_v43.z.number(), +- output_tokens: import_v43.z.number() ++ output_tokens: import_v43.z.number(), ++ cached_tokens: import_v43.z.number().optional() + }) + }) + }) +diff --git a/dist/index.mjs b/dist/index.mjs +index 44f26c27511f9089cdcc9456590dee100b070b0a..d3f585b58208430404135060af5774a7ecfa65d0 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -126,11 +126,12 @@ function convertCohereUsage(tokens) { + } + const inputTokens = tokens.input_tokens; + const outputTokens = tokens.output_tokens; ++ const cacheReadTokens = tokens.cached_tokens ?? 0; + return { + inputTokens: { + total: inputTokens, +- noCache: inputTokens, +- cacheRead: void 0, ++ noCache: Math.max(0, inputTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || void 0, + cacheWrite: void 0 + }, + outputTokens: { +@@ -661,7 +662,8 @@ var cohereChatResponseSchema = z3.object({ + }), + tokens: z3.object({ + input_tokens: z3.number(), +- output_tokens: z3.number() ++ output_tokens: z3.number(), ++ cached_tokens: z3.number().optional() + }) + }) + }); +@@ -721,7 +723,8 @@ var cohereChatChunkSchema = z3.discriminatedUnion("type", [ + usage: z3.object({ + tokens: z3.object({ + input_tokens: z3.number(), +- output_tokens: z3.number() ++ output_tokens: z3.number(), ++ cached_tokens: z3.number().optional() + }) + }) + }) +diff --git a/src/cohere-chat-language-model.ts b/src/cohere-chat-language-model.ts +index e4971ccf255ffff77c38fd35caccf84a032ad150..93fea74a08c0f9bcc6c63689fbc9e72669cdeba1 100644 +--- a/src/cohere-chat-language-model.ts ++++ b/src/cohere-chat-language-model.ts +@@ -495,6 +495,7 @@ const cohereChatResponseSchema = z.object({ + tokens: z.object({ + input_tokens: z.number(), + output_tokens: z.number(), ++ cached_tokens: z.number().optional(), + }), + }), + }); +@@ -558,6 +559,7 @@ const cohereChatChunkSchema = z.discriminatedUnion('type', [ + tokens: z.object({ + input_tokens: z.number(), + output_tokens: z.number(), ++ cached_tokens: z.number().optional(), + }), + }), + }), +diff --git a/src/convert-cohere-usage.ts b/src/convert-cohere-usage.ts +index 0f83a63ab85ab2c2ae136edd509e1f738553041f..3913f8d221c8980916769a9890b56030f69a6070 100644 +--- a/src/convert-cohere-usage.ts ++++ b/src/convert-cohere-usage.ts +@@ -3,6 +3,7 @@ import { LanguageModelV3Usage } from '@ai-sdk/provider'; + export type CohereUsageTokens = { + input_tokens: number; + output_tokens: number; ++ cached_tokens?: number; + }; + + export function convertCohereUsage( +@@ -27,12 +28,13 @@ export function convertCohereUsage( + + const inputTokens = tokens.input_tokens; + const outputTokens = tokens.output_tokens; ++ const cacheReadTokens = tokens.cached_tokens ?? 0; + + return { + inputTokens: { + total: inputTokens, +- noCache: inputTokens, +- cacheRead: undefined, ++ noCache: Math.max(0, inputTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || undefined, + cacheWrite: undefined, + }, + outputTokens: { diff --git a/patches/@ai-sdk%2Fgroq@3.0.31.patch b/patches/@ai-sdk%2Fgroq@3.0.31.patch new file mode 100644 index 000000000000..2fc7d38f1916 --- /dev/null +++ b/patches/@ai-sdk%2Fgroq@3.0.31.patch @@ -0,0 +1,67 @@ +diff --git a/dist/index.js b/dist/index.js +index 45a104f2e0775761858eac2a82ced64bceba1f5e..36861246fa3aea50f82a233aec8c8edaef0ff6e3 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -57,13 +57,14 @@ function convertGroqUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const reasoningTokens = (_d = (_c = usage.completion_tokens_details) == null ? void 0 : _c.reasoning_tokens) != null ? _d : void 0; ++ const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const reasoningTokens = usage.completion_tokens_details?.reasoning_tokens ?? void 0; + const textTokens = reasoningTokens != null ? completionTokens - reasoningTokens : completionTokens; + return { + inputTokens: { + total: promptTokens, +- noCache: promptTokens, +- cacheRead: void 0, ++ noCache: Math.max(0, promptTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || void 0, + cacheWrite: void 0 + }, + outputTokens: { +diff --git a/dist/index.mjs b/dist/index.mjs +index c644c32235d8fa88c51c0fc6958feb1da4877c96..6aa35dd173cd67dbdd2f6f3a5fe2d901cbaafae8 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -44,13 +44,14 @@ function convertGroqUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const reasoningTokens = (_d = (_c = usage.completion_tokens_details) == null ? void 0 : _c.reasoning_tokens) != null ? _d : void 0; ++ const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const reasoningTokens = usage.completion_tokens_details?.reasoning_tokens ?? void 0; + const textTokens = reasoningTokens != null ? completionTokens - reasoningTokens : completionTokens; + return { + inputTokens: { + total: promptTokens, +- noCache: promptTokens, +- cacheRead: void 0, ++ noCache: Math.max(0, promptTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || void 0, + cacheWrite: void 0 + }, + outputTokens: { +diff --git a/src/convert-groq-usage.ts b/src/convert-groq-usage.ts +index a16809223b068bc1d682ef9156ccc82ce89ace5c..732b14508609106bd5946ba9f9323c7863ee2da5 100644 +--- a/src/convert-groq-usage.ts ++++ b/src/convert-groq-usage.ts +@@ -40,6 +40,7 @@ export function convertGroqUsage( + + const promptTokens = usage.prompt_tokens ?? 0; + const completionTokens = usage.completion_tokens ?? 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? 0; + const reasoningTokens = + usage.completion_tokens_details?.reasoning_tokens ?? undefined; + const textTokens = +@@ -50,8 +51,8 @@ export function convertGroqUsage( + return { + inputTokens: { + total: promptTokens, +- noCache: promptTokens, +- cacheRead: undefined, ++ noCache: Math.max(0, promptTokens - cacheReadTokens), ++ cacheRead: cacheReadTokens || undefined, + cacheWrite: undefined, + }, + outputTokens: { diff --git a/patches/@ai-sdk%2Fmistral@3.0.34.patch b/patches/@ai-sdk%2Fmistral@3.0.34.patch new file mode 100644 index 000000000000..1d771f4fd9f7 --- /dev/null +++ b/patches/@ai-sdk%2Fmistral@3.0.34.patch @@ -0,0 +1,84 @@ +diff --git a/dist/index.d.ts b/dist/index.d.ts +index 1ca9113bed2728a616db773a8e08d8d6957447d7..15408ec429dc210b5fa43589d81b69c93bf27b2d 100644 +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -14,6 +14,7 @@ declare const mistralLanguageModelOptions: z.ZodObject<{ + none: "none"; + high: "high"; + }>>; ++ promptCacheKey: z.ZodOptional; + }, z.core.$strip>; + type MistralLanguageModelOptions = z.infer; + +diff --git a/dist/index.js b/dist/index.js +index 45735e524aaff54ea058c99c729c5ffd3c507058..6aca5f6f13da0054ede31c1f1a692e4eaed37d34 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -268,7 +268,8 @@ var mistralLanguageModelOptions = import_v4.z.object({ + * - `'high'`: Enable reasoning + * - `'none'`: Disable reasoning + */ +- reasoningEffort: import_v4.z.enum(["high", "none"]).optional() ++ reasoningEffort: import_v4.z.enum(["high", "none"]).optional(), ++ promptCacheKey: import_v4.z.string().optional() + }); + + // src/mistral-error.ts +@@ -413,6 +414,7 @@ var MistralChatLanguageModel = class { + top_p: topP, + random_seed: seed, + reasoning_effort: options.reasoningEffort, ++ prompt_cache_key: options.promptCacheKey, + // response format: + response_format: (responseFormat == null ? void 0 : responseFormat.type) === "json" ? structuredOutputs && (responseFormat == null ? void 0 : responseFormat.schema) != null ? { + type: "json_schema", +diff --git a/dist/index.mjs b/dist/index.mjs +index 4c22df1cd78a1ba81309c8a86ceecefef4ba4aea..30cd3b1f503860109b7fa2107cd1eb17b70c96be 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -256,7 +256,8 @@ var mistralLanguageModelOptions = z.object({ + * - `'high'`: Enable reasoning + * - `'none'`: Disable reasoning + */ +- reasoningEffort: z.enum(["high", "none"]).optional() ++ reasoningEffort: z.enum(["high", "none"]).optional(), ++ promptCacheKey: z.string().optional() + }); + + // src/mistral-error.ts +@@ -403,6 +404,7 @@ var MistralChatLanguageModel = class { + top_p: topP, + random_seed: seed, + reasoning_effort: options.reasoningEffort, ++ prompt_cache_key: options.promptCacheKey, + // response format: + response_format: (responseFormat == null ? void 0 : responseFormat.type) === "json" ? structuredOutputs && (responseFormat == null ? void 0 : responseFormat.schema) != null ? { + type: "json_schema", +diff --git a/src/mistral-chat-language-model.ts b/src/mistral-chat-language-model.ts +index 480c472d534bedbe8897979673453bd1c29a70b7..e46496da94f7d4af9822897202ca6baae67dae3a 100644 +--- a/src/mistral-chat-language-model.ts ++++ b/src/mistral-chat-language-model.ts +@@ -129,6 +129,7 @@ export class MistralChatLanguageModel implements LanguageModelV3 { + top_p: topP, + random_seed: seed, + reasoning_effort: options.reasoningEffort, ++ prompt_cache_key: options.promptCacheKey, + + // response format: + response_format: +diff --git a/src/mistral-chat-options.ts b/src/mistral-chat-options.ts +index 80fff45fba2c378fa06962f071946bcd2b882a0b..b4fdfa51f3bf11a4220e010c8aca92482cb0c3db 100644 +--- a/src/mistral-chat-options.ts ++++ b/src/mistral-chat-options.ts +@@ -62,6 +62,11 @@ export const mistralLanguageModelOptions = z.object({ + * - `'none'`: Disable reasoning + */ + reasoningEffort: z.enum(['high', 'none']).optional(), ++ ++ /** ++ * A stable identifier used to route requests with shared prompt prefixes. ++ */ ++ promptCacheKey: z.string().optional(), + }); + + export type MistralLanguageModelOptions = z.infer< diff --git a/patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch b/patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch new file mode 100644 index 000000000000..a3bdd13d517a --- /dev/null +++ b/patches/@ai-sdk%2Fopenai-compatible@2.0.37.patch @@ -0,0 +1,101 @@ +diff --git a/dist/index.js b/dist/index.js +index d73573b013e61903991cad2074187a3ade8921ab..ca7923e551f8f0cb4c27e0afd5f6cb3bc3b6c1c4 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -73,7 +73,7 @@ function convertOpenAICompatibleChatUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens ?? 0; + const reasoningTokens = (_f = (_e = usage.completion_tokens_details) == null ? void 0 : _e.reasoning_tokens) != null ? _f : 0; + return { + inputTokens: { +@@ -894,6 +894,7 @@ var openaiCompatibleTokenUsageSchema = import_v43.z.looseObject({ + prompt_tokens: import_v43.z.number().nullish(), + completion_tokens: import_v43.z.number().nullish(), + total_tokens: import_v43.z.number().nullish(), ++ cached_tokens: import_v43.z.number().nullish(), + prompt_tokens_details: import_v43.z.object({ + cached_tokens: import_v43.z.number().nullish() + }).nullish(), +diff --git a/dist/index.mjs b/dist/index.mjs +index 237c9e209da9da2d4d557eb309e1adc2157c1426..fb45541149086633e27f02c89b742208222bb185 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -53,7 +53,7 @@ function convertOpenAICompatibleChatUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens ?? 0; + const reasoningTokens = (_f = (_e = usage.completion_tokens_details) == null ? void 0 : _e.reasoning_tokens) != null ? _f : 0; + return { + inputTokens: { +@@ -881,6 +881,7 @@ var openaiCompatibleTokenUsageSchema = z3.looseObject({ + prompt_tokens: z3.number().nullish(), + completion_tokens: z3.number().nullish(), + total_tokens: z3.number().nullish(), ++ cached_tokens: z3.number().nullish(), + prompt_tokens_details: z3.object({ + cached_tokens: z3.number().nullish() + }).nullish(), +diff --git a/dist/internal/index.js b/dist/internal/index.js +index 0858632db778d30b303fbc9d5d46a29582d6e5b5..bb414ac25b77cb1a492cf6ad64f24decae38dd31 100644 +--- a/dist/internal/index.js ++++ b/dist/internal/index.js +@@ -249,7 +249,7 @@ function convertOpenAICompatibleChatUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens ?? 0; + const reasoningTokens = (_f = (_e = usage.completion_tokens_details) == null ? void 0 : _e.reasoning_tokens) != null ? _f : 0; + return { + inputTokens: { +diff --git a/dist/internal/index.mjs b/dist/internal/index.mjs +index 116b2deee86bf07cf981a64c000c7a4f3e42b52e..de08e6248318a0e48866ba41e9788485d56dd5cb 100644 +--- a/dist/internal/index.mjs ++++ b/dist/internal/index.mjs +@@ -224,7 +224,7 @@ function convertOpenAICompatibleChatUsage(usage) { + } + const promptTokens = (_a = usage.prompt_tokens) != null ? _a : 0; + const completionTokens = (_b = usage.completion_tokens) != null ? _b : 0; +- const cacheReadTokens = (_d = (_c = usage.prompt_tokens_details) == null ? void 0 : _c.cached_tokens) != null ? _d : 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens ?? 0; + const reasoningTokens = (_f = (_e = usage.completion_tokens_details) == null ? void 0 : _e.reasoning_tokens) != null ? _f : 0; + return { + inputTokens: { +diff --git a/src/chat/convert-openai-compatible-chat-usage.ts b/src/chat/convert-openai-compatible-chat-usage.ts +index 7982309cd99afbf7eb84a733c37446ac3cb9b626..454f14fe45af3f7475ba0371173c3342f242ccd9 100644 +--- a/src/chat/convert-openai-compatible-chat-usage.ts ++++ b/src/chat/convert-openai-compatible-chat-usage.ts +@@ -5,6 +5,7 @@ export function convertOpenAICompatibleChatUsage( + | { + prompt_tokens?: number | null; + completion_tokens?: number | null; ++ cached_tokens?: number | null; + prompt_tokens_details?: { + cached_tokens?: number | null; + } | null; +@@ -34,7 +35,7 @@ export function convertOpenAICompatibleChatUsage( + + const promptTokens = usage.prompt_tokens ?? 0; + const completionTokens = usage.completion_tokens ?? 0; +- const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? 0; ++ const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens ?? 0; + const reasoningTokens = + usage.completion_tokens_details?.reasoning_tokens ?? 0; + +diff --git a/src/chat/openai-compatible-chat-language-model.ts b/src/chat/openai-compatible-chat-language-model.ts +index 9a803cb1173e1454facb73ae430ede0ea0a3d2de..fd2c8e78ad850e11492b6be51922a382664739bf 100644 +--- a/src/chat/openai-compatible-chat-language-model.ts ++++ b/src/chat/openai-compatible-chat-language-model.ts +@@ -723,6 +723,7 @@ const openaiCompatibleTokenUsageSchema = z + prompt_tokens: z.number().nullish(), + completion_tokens: z.number().nullish(), + total_tokens: z.number().nullish(), ++ cached_tokens: z.number().nullish(), + prompt_tokens_details: z + .object({ + cached_tokens: z.number().nullish(), diff --git a/patches/ai-gateway-provider@3.1.2.patch b/patches/ai-gateway-provider@3.1.2.patch new file mode 100644 index 000000000000..ac8b1a7deaca --- /dev/null +++ b/patches/ai-gateway-provider@3.1.2.patch @@ -0,0 +1,34 @@ +diff --git a/dist/index.js b/dist/index.js +index 6fb7ae621cf5fe350ec9295af8f3d3507d265df3..ee16ee3af60f461fe2b074eb9cab44ea586c97ee 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -283,10 +283,10 @@ function createAiGateway(options) { + function parseAiGatewayOptions(options) { + const headers = new Headers(); + if (options.skipCache === true) { +- headers.set("cf-skip-cache", "true"); ++ headers.set("cf-aig-skip-cache", "true"); + } + if (options.cacheTtl) { +- headers.set("cf-cache-ttl", options.cacheTtl.toString()); ++ headers.set("cf-aig-cache-ttl", options.cacheTtl.toString()); + } + if (options.metadata) { + headers.set("cf-aig-metadata", JSON.stringify(options.metadata)); +diff --git a/dist/index.mjs b/dist/index.mjs +index 1c00a815117b5854073eb9c01cafa5bc273a367e..11acdaa65021952eb52622ebd3d9b87633e5f8e5 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -254,10 +254,10 @@ function createAiGateway(options) { + function parseAiGatewayOptions(options) { + const headers = new Headers(); + if (options.skipCache === true) { +- headers.set("cf-skip-cache", "true"); ++ headers.set("cf-aig-skip-cache", "true"); + } + if (options.cacheTtl) { +- headers.set("cf-cache-ttl", options.cacheTtl.toString()); ++ headers.set("cf-aig-cache-ttl", options.cacheTtl.toString()); + } + if (options.metadata) { + headers.set("cf-aig-metadata", JSON.stringify(options.metadata));