Skip to content

Commit a95cbad

Browse files
committed
fix(cli): align pi/gemini/claude-code token edge cases with ccusage
- pi: an explicit totalTokens can now only ADD tokens the parts don't account for, never shrink the parts sum (ccusage apply_total_token_fallback) — fixes both total-only records counting 0 output and a small explicit total undercounting. - gemini: treat empty/whitespace id and sessionId as absent (new nonEmptyStringField, mirroring ccusage non_empty_string) so distinct events are not collapsed and sessions are attributed correctly; a present messages[] array no longer falls through to the stats branch. - claude-code: drop the synthetic "<synthetic>" model so it does not create a bogus per-model bucket.
1 parent d39e9f1 commit a95cbad

6 files changed

Lines changed: 92 additions & 10 deletions

File tree

packages/cli/src/adapters/claude-code.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,13 @@ async function parseClaudeCodeSessionFile(
225225
continue
226226
}
227227

228-
model = stringField(message, 'model') || model
228+
// Claude Code stamps injected/synthetic assistant turns with model
229+
// "<synthetic>"; ccusage strips it so those tokens don't create a bogus
230+
// per-model bucket. Ignore it here too (keep the real running model).
231+
const parsedModel = stringField(message, 'model')
232+
if (parsedModel && parsedModel !== '<synthetic>') {
233+
model = parsedModel
234+
}
229235
// Skip the entire assistant entry when (messageId, requestId) was
230236
// already processed — applies to both the usage metrics and any
231237
// tool_use items so we don't double-emit tool.started either.

packages/cli/src/adapters/gemini.ts

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { matchesBackfillFilters } from '../lib/backfill.js'
1010
import {
1111
arrayField,
1212
isPlainObject,
13+
nonEmptyStringField,
1314
numberField,
1415
objectField,
1516
stringField,
@@ -223,16 +224,19 @@ function parseJsonRecord(
223224
fileStem: string,
224225
fallbackTs: string,
225226
): ParsedFile {
226-
const sessionId = stringField(record, 'sessionId')
227-
?? stringField(record, 'session_id')
227+
const sessionId = nonEmptyStringField(record, 'sessionId')
228+
?? nonEmptyStringField(record, 'session_id')
228229
?? fileStem
229230
const sessionTs = timestampFrom(stringField(record, 'startTime'))
230231
?? timestampFrom(stringField(record, 'lastUpdated'))
231232
?? fallbackTs
232233

233-
const messages = arrayField(record, 'messages').filter(isPlainObject)
234-
if (messages.length > 0) {
235-
const usages = messages
234+
// A `messages` array — even an empty one — marks this as a session-file record;
235+
// ccusage returns its (possibly empty) messages and never falls through to the
236+
// top-level type/stats branches, so neither do we.
237+
if (Array.isArray(record.messages)) {
238+
const usages = arrayField(record, 'messages')
239+
.filter(isPlainObject)
236240
.filter(message => stringField(message, 'type') === 'gemini')
237241
.map(message => parseDirectEvent(message, undefined, sessionTs))
238242
.filter((usage): usage is GeminiUsage => usage !== undefined)
@@ -268,7 +272,7 @@ function parseJsonlRecords(
268272
if (!record) {
269273
continue
270274
}
271-
const sid = stringField(record, 'sessionId') ?? stringField(record, 'session_id')
275+
const sid = nonEmptyStringField(record, 'sessionId') ?? nonEmptyStringField(record, 'session_id')
272276
if (sid) {
273277
sessionId = sid
274278
}
@@ -282,7 +286,9 @@ function parseJsonlRecords(
282286
if (!usage) {
283287
continue
284288
}
285-
const id = stringField(record, 'id')
289+
// An empty / whitespace id is not a real dedup key (ccusage non_empty_string
290+
// -> None); push those verbatim instead of collapsing distinct events.
291+
const id = nonEmptyStringField(record, 'id')
286292
if (id === undefined) {
287293
usages.push(usage)
288294
}

packages/cli/src/adapters/pi.ts

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,16 @@ export function piUsageFromMessage(message: Record<string, unknown>): Partial<Me
323323
const output = numberField(usage, 'output') || 0
324324
const cacheRead = numberField(usage, 'cacheRead') || 0
325325
const cacheWrite = numberField(usage, 'cacheWrite') || 0
326-
const totalTokens = numberField(usage, 'totalTokens') || (input + output + cacheRead + cacheWrite)
326+
// ccusage apply_total_token_fallback: an explicit totalTokens can only ADD tokens
327+
// the parts don't account for (folded into billable output), never shrink the
328+
// parts sum. This both counts total-only records (output would otherwise be 0)
329+
// and stops an explicit total smaller than the parts from undercounting the grand
330+
// total.
331+
const explicitTotal = numberField(usage, 'totalTokens') || 0
332+
const partsSum = input + output + cacheRead + cacheWrite
333+
const missing = Math.max(0, explicitTotal - partsSum)
334+
const billableOutput = output + missing
335+
const totalTokens = partsSum + missing
327336

328337
if (totalTokens <= 0) {
329338
return undefined
@@ -334,7 +343,7 @@ export function piUsageFromMessage(message: Record<string, unknown>): Partial<Me
334343

335344
return {
336345
tokensInput: (input + cacheRead + cacheWrite) || undefined,
337-
tokensOutput: output || undefined,
346+
tokensOutput: billableOutput || undefined,
338347
tokensCachedInput: (cacheRead + cacheWrite) || undefined,
339348
tokensCacheReadInput: cacheRead || undefined,
340349
tokensCacheCreationInput: cacheWrite || undefined,

packages/cli/src/lib/fields.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ export function stringField(object: unknown, key: string): string | undefined {
66
return typeof value === 'string' ? value : undefined
77
}
88

9+
// Like stringField but treats empty / whitespace-only strings as absent, mirroring
10+
// ccusage's non_empty_string (trim -> None). Returns the original (untrimmed) value
11+
// when non-empty, so it composes with `??` fallbacks and dedup-key checks.
12+
export function nonEmptyStringField(object: unknown, key: string): string | undefined {
13+
const value = stringField(object, key)
14+
return value && value.trim().length > 0 ? value : undefined
15+
}
16+
917
export function numberField(object: Record<string, unknown>, key: string): number | undefined {
1018
const value = object[key]
1119
return typeof value === 'number' && Number.isFinite(value) ? value : undefined

packages/cli/test/gemini.test.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,22 @@ test('parses JSONL direct events and keeps tokensInput cache-inclusive', async (
4848
assert.equal(usages[1].metrics?.tokensTotal, 240)
4949
})
5050

51+
test('empty-string ids are not a dedup key; distinct events are kept', async () => {
52+
// stringField returns '' verbatim; treating it as a real id collapsed distinct
53+
// events (last-write-wins). ccusage non_empty_string maps '' -> None -> no dedup.
54+
const dir = await mkdtemp(path.join(tmpdir(), 'gemini-'))
55+
const file = path.join(dir, 'session.jsonl')
56+
await writeLines(file, [
57+
{ type: 'gemini', model: 'g', id: '', timestamp: '2026-04-29T00:00:01.000Z', tokens: { input: 10, output: 5 } },
58+
{ type: 'gemini', model: 'g', id: ' ', timestamp: '2026-04-29T00:00:02.000Z', tokens: { input: 999, output: 5 } },
59+
])
60+
61+
const usages = usageEvents(await parse(file))
62+
assert.equal(usages.length, 2)
63+
assert.equal(usages[0].metrics?.tokensInput, 10)
64+
assert.equal(usages[1].metrics?.tokensInput, 999)
65+
})
66+
5167
test('brackets usage events with session.started and session.ended', async () => {
5268
const dir = await mkdtemp(path.join(tmpdir(), 'gemini-'))
5369
const file = path.join(dir, 'session.jsonl')

packages/cli/test/pi.test.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import assert from 'node:assert/strict'
2+
// eslint-disable-next-line test/no-import-node-test -- This repo uses node:test as the runner.
3+
import { test } from 'node:test'
4+
import { piUsageFromMessage } from '../src/adapters/pi.ts'
5+
6+
// pi mirrors ccusage apply_total_token_fallback: an explicit totalTokens can only
7+
// ADD tokens the itemized parts don't account for, never shrink the parts sum.
8+
9+
test('parity: ccusage pi falls_back_to_total_tokens_when_pi_parts_are_missing', () => {
10+
// usage with only totalTokens → the whole total is folded into billable output.
11+
const usage = piUsageFromMessage({ usage: { totalTokens: 333 } })
12+
13+
assert.ok(usage)
14+
assert.equal(usage.tokensOutput, 333)
15+
assert.equal(usage.tokensTotal, 333)
16+
})
17+
18+
test('pi never lets an explicit total shrink the parts sum', () => {
19+
// totalTokens (150) is smaller than input+output+cacheRead (2150) — e.g. a total
20+
// computed excluding cache. The grand total must not undercount the parts.
21+
const usage = piUsageFromMessage({ usage: { input: 100, output: 50, cacheRead: 2000, totalTokens: 150 } })
22+
23+
assert.ok(usage)
24+
// cache-inclusive input: 100 + 2000.
25+
assert.equal(usage.tokensInput, 2100)
26+
assert.equal(usage.tokensOutput, 50)
27+
assert.equal(usage.tokensTotal, 2150) // max(explicit 150, parts 2150)
28+
})
29+
30+
test('pi still passes through a consistent explicit total', () => {
31+
const usage = piUsageFromMessage({ usage: { input: 100, output: 50, totalTokens: 200 } })
32+
33+
assert.ok(usage)
34+
assert.equal(usage.tokensInput, 100)
35+
assert.equal(usage.tokensOutput, 100) // 50 + (200 - 150) folded
36+
assert.equal(usage.tokensTotal, 200)
37+
})

0 commit comments

Comments
 (0)