From e93a2fdce0e6be0602c240e63b0522b2e29ddb9f Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 22:46:03 -0700 Subject: [PATCH] feat(ai): track real usage for BYOK provider calls callAiProvider never read the usage field from Anthropic/OpenAI's raw response body, so BYOK calls populated no provider/token/cost data in ai_usage_events even though the maintainer bills these to their own account. Normalize both providers' native usage envelopes into the same shape the free/self-host path already produces, and price tokens against a static per-model USD table (absent model -> costUsd stays undefined, never fabricated). --- src/services/ai-review.ts | 92 +++++++++++++- src/services/ai-slop.ts | 3 +- test/unit/ai-review.test.ts | 241 ++++++++++++++++++++++++++++++++++++ test/unit/ai-slop.test.ts | 41 ++++++ 4 files changed, 370 insertions(+), 7 deletions(-) diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 3cff5f476b..2d04440706 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -884,17 +884,95 @@ type ProviderReviewOutcome = { diagnostic?: AiReviewDiagnostic | undefined; }; +/** Static USD-per-million-token pricing for BYOK models. Anthropic/OpenAI responses report token counts but + * never a dollar figure, so this table is the only source for a BYOK call's `costUsd`. A model absent here + * (e.g. a maintainer-configured override this table hasn't been updated for) leaves `costUsd` undefined — + * never fabricated — matching how every other unavailable usage field already degrades in this file. */ +const BYOK_MODEL_PRICING_USD_PER_MTOK: Record< + AiReviewProviderKey["provider"], + Record +> = { + anthropic: { + "claude-opus-4-8": { input: 5, output: 25 }, + "claude-opus-4-7": { input: 5, output: 25 }, + "claude-opus-4-6": { input: 5, output: 25 }, + "claude-sonnet-5": { input: 3, output: 15 }, + "claude-sonnet-4-6": { input: 3, output: 15 }, + "claude-haiku-4-5": { input: 1, output: 5 }, + }, + openai: { + "gpt-5.5": { input: 5, output: 30 }, + "gpt-5.5-pro": { input: 30, output: 180 }, + "gpt-5.4": { input: 2.5, output: 15 }, + "gpt-5.4-mini": { input: 0.75, output: 4.5 }, + "gpt-5.4-nano": { input: 0.2, output: 1.25 }, + }, +}; + +function priceByokUsageUsd( + provider: AiReviewProviderKey["provider"], + model: string, + inputTokens: number | undefined, + outputTokens: number | undefined, +): number | undefined { + if (inputTokens === undefined || outputTokens === undefined) return undefined; + const pricing = BYOK_MODEL_PRICING_USD_PER_MTOK[provider][model]; + if (!pricing) return undefined; + return (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000; +} + +/** Normalize a BYOK provider's native usage envelope into the same shape `coerceAiUsage` produces for the + * free/self-host path. Anthropic reports `usage: {input_tokens, output_tokens}`; OpenAI reports + * `usage: {prompt_tokens, completion_tokens, total_tokens}` — both snake_case and provider-specific, unlike + * the already-camelCase envelope `coerceAiUsage` reads from `env.AI.run()`. Anthropic's `usage` can also + * carry `cache_creation_input_tokens`/`cache_read_input_tokens`, priced differently than `input_tokens` — + * intentionally not read here, since `callAiProvider` never sends `cache_control`, so Anthropic never + * populates them on this path. */ +function coerceByokUsage( + providerKey: AiReviewProviderKey, + model: string, + rawResult: unknown, +): AiReviewActualUsage | undefined { + if (!rawResult || typeof rawResult !== "object") return undefined; + const usage = (rawResult as Record).usage; + if (!usage || typeof usage !== "object" || Array.isArray(usage)) return undefined; + const record = usage as Record; + const inputTokens = + providerKey.provider === "anthropic" + ? finiteUsageInteger(record.input_tokens) + : finiteUsageInteger(record.prompt_tokens); + const outputTokens = + providerKey.provider === "anthropic" + ? finiteUsageInteger(record.output_tokens) + : finiteUsageInteger(record.completion_tokens); + const totalTokens = + providerKey.provider === "openai" ? finiteUsageInteger(record.total_tokens) : undefined; + if (inputTokens === undefined && outputTokens === undefined && totalTokens === undefined) + return undefined; + // The `?? 0` fallback below is always safe: the guard above guarantees that whenever totalTokens is + // undefined, at least one of inputTokens/outputTokens is defined. + return { + provider: providerKey.provider, + model, + inputTokens, + outputTokens, + totalTokens: totalTokens ?? (inputTokens ?? 0) + (outputTokens ?? 0), + costUsd: priceByokUsageUsd(providerKey.provider, model, inputTokens, outputTokens), + }; +} + /** - * POST to the maintainer's BYOK provider and return the raw response text (or null + a failure reason). - * Never throws. Shared by every BYOK AI path (review, slop, …) so the endpoint/timeout/error handling - * lives in one place; callers parse the returned text into their own shape. + * POST to the maintainer's BYOK provider and return the raw response text (or null + a failure reason), + * plus real usage (tokens/cost) when the response body included a parseable `usage` field. Never throws. + * Shared by every BYOK AI path (review, slop, …) so the endpoint/timeout/error/usage handling lives in one + * place; callers parse the returned text into their own shape. */ export async function callAiProvider( providerKey: AiReviewProviderKey, system: string, user: string, maxTokens: number, -): Promise<{ text: string | null; failure?: ProviderFailure }> { +): Promise<{ text: string | null; usage?: AiReviewActualUsage | undefined; failure?: ProviderFailure }> { const model = providerKey.model || PROVIDER_DEFAULT_MODEL[providerKey.provider]; try { @@ -934,7 +1012,8 @@ export async function callAiProvider( }); } if (!response.ok) return { text: null, failure: "http_error" }; - return { text: coerceAiText(await response.json()) }; + const body = await response.json(); + return { text: coerceAiText(body), usage: coerceByokUsage(providerKey, model, body) }; } catch (error) { // AbortSignal.timeout rejects with a TimeoutError; everything else is a network/parse exception. const failure: ProviderFailure = @@ -953,7 +1032,7 @@ async function runProviderReview( user: string, maxTokens: number, ): Promise { - const { text, failure } = await callAiProvider( + const { text, usage, failure } = await callAiProvider( providerKey, system, user, @@ -972,6 +1051,7 @@ async function runProviderReview( status: review ? "parsed" : textValue ? "unparseable_output" : "empty_output", responseChars: textValue.length, hasJsonObject: Boolean(textValue && extractLastJsonObject(textValue)), + usage, }, }; } diff --git a/src/services/ai-slop.ts b/src/services/ai-slop.ts index d779ebed17..3062750802 100644 --- a/src/services/ai-slop.ts +++ b/src/services/ai-slop.ts @@ -220,8 +220,9 @@ export async function runGittensoryAiSlopAdvisory(env: Env, input: AiSlopInput): let opinion: SlopOpinion | null; let usage: AiReviewActualUsage | undefined; if (input.providerKey) { - const { text } = await callAiProvider(input.providerKey, SLOP_SYSTEM_PROMPT, user, maxTokens); + const { text, usage: byokUsage } = await callAiProvider(input.providerKey, SLOP_SYSTEM_PROMPT, user, maxTokens); opinion = text ? parseSlopOpinion(text) : null; + usage = byokUsage; } else { ({ opinion, usage } = await runWorkersSlopOpinion(env, SLOP_SYSTEM_PROMPT, user, maxTokens)); } diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index accad5a09c..4f84f69b85 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -830,6 +830,247 @@ describe("BYOK provider dispatch", () => { ).model, ).toBe("claude-custom"); }); + + it("records real Anthropic BYOK usage (tokens + cost) on the durable audit row", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [{ type: "text", text: reviewJson({ assessment: "BYOK review." }) }], + usage: { input_tokens: 1000, output_tokens: 200 }, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret", model: "claude-sonnet-5" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + usage: { + provider: "anthropic", + model: "claude-sonnet-5", + inputTokens: 1000, + outputTokens: 200, + totalTokens: 1200, + costUsd: 0.006, + }, + }), + ]); + const row = await env.DB.prepare( + `select provider, input_tokens, output_tokens, total_tokens, cost_usd + from ai_usage_events where feature = ? order by rowid desc limit 1`, + ) + .bind("ai_review_pr") + .first<{ + provider: string | null; + input_tokens: number; + output_tokens: number; + total_tokens: number; + cost_usd: number; + }>(); + expect(row).toMatchObject({ + provider: "anthropic", + input_tokens: 1000, + output_tokens: 200, + total_tokens: 1200, + cost_usd: 0.006, + }); + }); + + it("records real OpenAI BYOK usage using the provider's own total_tokens", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: reviewJson({ assessment: "BYOK review." }) } }], + usage: { prompt_tokens: 800, completion_tokens: 100, total_tokens: 900 }, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "openai", key: "sk-secret", model: "gpt-5.4" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + usage: { + provider: "openai", + model: "gpt-5.4", + inputTokens: 800, + outputTokens: 100, + totalTokens: 900, + costUsd: 0.0035, + }, + }), + ]); + }); + + it("leaves BYOK costUsd undefined for a model absent from the pricing table, without dropping tokens", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [{ type: "text", text: reviewJson({ assessment: "BYOK review." }) }], + usage: { input_tokens: 50, output_tokens: 10 }, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + // No `model` override — falls back to the provider default, which this pricing table doesn't cover. + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + usage: expect.objectContaining({ + inputTokens: 50, + outputTokens: 10, + totalTokens: 60, + costUsd: undefined, + }), + }), + ]); + }); + + it("leaves BYOK usage undefined when the response's usage object has no recognized fields", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: reviewJson({ assessment: "BYOK review." }) } }], + usage: {}, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "openai", key: "sk-secret", model: "gpt-5.4" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ usage: undefined }), + ]); + }); + + it("sums a lone output_tokens toward totalTokens when input_tokens is absent", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [{ type: "text", text: reviewJson({ assessment: "BYOK review." }) }], + usage: { output_tokens: 40 }, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + usage: expect.objectContaining({ inputTokens: undefined, outputTokens: 40, totalTokens: 40 }), + }), + ]); + }); + + it("sums a lone input_tokens toward totalTokens when output_tokens is absent", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [{ type: "text", text: reviewJson({ assessment: "BYOK review." }) }], + usage: { input_tokens: 25 }, + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + usage: expect.objectContaining({ inputTokens: 25, outputTokens: undefined, totalTokens: 25 }), + }), + ]); + }); + + it("treats a non-object BYOK response body as empty output with no usage", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("null", { status: 200 }))); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ status: "empty_output", usage: undefined }), + ]); + }); }); describe("Workers AI fallback + degraded output", () => { diff --git a/test/unit/ai-slop.test.ts b/test/unit/ai-slop.test.ts index 6eee39fed0..3c94be157d 100644 --- a/test/unit/ai-slop.test.ts +++ b/test/unit/ai-slop.test.ts @@ -266,6 +266,47 @@ describe("runGittensoryAiSlopAdvisory gating + fail-safe", () => { expect(fetchMock).not.toHaveBeenCalled(); expect(run).not.toHaveBeenCalled(); }); + + it("records real BYOK usage (tokens + cost) on the durable audit row", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [{ type: "text", text: slopJson({ band: "elevated" }) }], + usage: { input_tokens: 500, output_tokens: 50 }, + }), + { status: 200 }, + ), + ), + ); + const env = enabledEnv(vi.fn()); + const result = await runGittensoryAiSlopAdvisory(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-x", model: "claude-sonnet-5" }, + }); + expect(result.status).toBe("ok"); + const row = await env.DB.prepare( + `select provider, input_tokens, output_tokens, total_tokens, cost_usd + from ai_usage_events where feature = ? order by rowid desc limit 1`, + ) + .bind("ai_slop_pr") + .first<{ + provider: string | null; + input_tokens: number; + output_tokens: number; + total_tokens: number; + cost_usd: number; + }>(); + expect(row).toMatchObject({ + provider: "anthropic", + input_tokens: 500, + output_tokens: 50, + total_tokens: 550, + cost_usd: 0.00225, + }); + }); }); describe("the AI slop advisory can never become a gate blocker", () => {