diff --git a/node_modules b/node_modules new file mode 120000 index 00000000..43c299b1 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/Users/husamsoboh/codeburn/node_modules \ No newline at end of file diff --git a/src/act/report.ts b/src/act/report.ts index 95ec6e8c..30c88120 100644 --- a/src/act/report.ts +++ b/src/act/report.ts @@ -39,16 +39,27 @@ const HONEST_FOOTER = 'Estimates are scaled to the measured window for comparability; the at-apply estimate is kept in --json. ' + 'MCP and archive realized figures are derived from per-session baselines times session counts, not independently measured. ' + 'Each fix measures only its own metric; effects are never attributed across signals. ' - + 'Guard rows are correlation, not attribution. Realized numbers are rounded down.' + + 'Guard rows are correlation, not attribution. Realized numbers are rounded down. ' + + 'Deferral rows exclude servers an MCP remove/scope row already measures.' const MCP_KINDS = new Set(['mcp-remove', 'mcp-project-scope']) +// defer-* re-enable native MCP tool deferral (part 2 of #614): the same +// prefix schema tokens mcp-remove eliminates, deferral moves out of the +// upfront prefix. Realized the same way — per-session schema tokens times the +// post-apply sessions that benefited — but "benefited" flips: instead of a +// server no longer loading, it is deferral having become active (the session +// now carries a deferred-tools inventory, the detector's own signal). +const DEFER_KINDS = new Set(['defer-enable', 'defer-alwaysload', 'defer-threshold']) const ARCHIVE_DEF_TOKENS: Partial> = { 'archive-skill': TOKENS_PER_SKILL_DEF, 'archive-agent': TOKENS_PER_AGENT_DEF, 'archive-command': TOKENS_PER_COMMAND_DEF, } -export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' +// 'pending' means the applied change has not taken effect in any post-apply +// session yet (e.g. deferral before a client restart) - distinct from +// 'reverted', which asserts the user undid it. +export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' | 'pending' export type ActReportRow = { id: string @@ -178,6 +189,28 @@ function countSessionsLoading(projects: ProjectSummary[], servers: string[]): nu return allSessions(projects).filter(s => sessionLoadsAny(s, servers)).length } +// Deferral is active in a session exactly when Claude Code emitted a +// deferred-tools inventory for it — the same signal the mcp-deferral-off +// detector uses (its absence, alongside MCP overhead, is what flags a gap). +function sessionHasDeferralActive(s: SessionSummary): boolean { + return (s.mcpInventory?.length ?? 0) > 0 +} + +// MCP servers observed loading in the window (via inventory or invocation). +// defer-enable / defer-threshold re-enable deferral for the whole MCP surface +// rather than a named set, so the affected servers are derived here. +function observedMcpServers(projects: ProjectSummary[]): string[] { + const servers = new Set() + for (const s of allSessions(projects)) { + for (const fqn of s.mcpInventory ?? []) { + const seg = fqn.split('__')[1] + if (seg) servers.add(seg) + } + for (const server of Object.keys(s.mcpBreakdown)) servers.add(server) + } + return [...servers] +} + // A kind whose realized effect is a token saving (everything except guard, // which is a dollars/yield correlation, and out-of-scope kinds). function isTokenKind(kind: ActionKind): boolean { @@ -226,6 +259,45 @@ function mcpRow( return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * savedSessions), confidence } } +function deferRow( + base: ActReportRow, sessions: SessionSummary[], + baseline: ActionBaseline, afterStart: Date, now: Date, + mcpClaimedServers: ReadonlySet, +): ActReportRow { + // Sum only the servers no MCP row claims (see mcpClaimedServers in + // computeActReport), so the same schema tokens are never realized twice. + const counted = Object.entries(baseline.metrics).filter(([server]) => !mcpClaimedServers.has(server)) + const excludedServers = Object.keys(baseline.metrics).length - counted.length + const perSessionTokens = counted.reduce((a, [, tokens]) => a + tokens, 0) + if (perSessionTokens === 0) { + return { + ...base, + note: excludedServers > 0 + ? 'not measurable: every server in this baseline is already measured by an MCP remove/scope row' + : 'not measurable: empty baseline', + } + } + if (sessions.length === 0) return { ...base, note: 'not measurable: no sessions in the window yet' } + const estimatedForWindow = Math.floor(perSessionTokens * sessions.length) + // A post-apply session realized the saving only if deferral actually became + // active in it. ENABLE_TOOL_SEARCH is read at process start, so sessions + // begun before the user restarted still run deferral-off — those aren't + // counted, and if none benefited we report it plainly rather than claim a + // saving that hasn't taken effect. + const deferredSessions = sessions.filter(sessionHasDeferralActive).length + const confidence = confidenceFor(sessions.length, baseline, afterStart, now) + if (deferredSessions === 0) { + return { + ...base, + estimatedForWindow, + status: 'pending', + confidence, + note: `not yet in effect: deferral is still inactive in ${sessions.length} post-apply session${sessions.length === 1 ? '' : 's'} (takes effect on the next session; the client may not have restarted, or the change was reverted)`, + } + } + return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * deferredSessions), confidence } +} + function archiveRow( base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[], baseline: ActionBaseline, afterStart: Date, now: Date, @@ -358,7 +430,7 @@ async function modelDefaultRow( async function computeRow( rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, - opts: ActReportOptions, modelDefaultProjectFound = true, + mcpClaimedServers: ReadonlySet, opts: ActReportOptions, modelDefaultProjectFound = true, ): Promise { const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0 const base: ActReportRow = { @@ -378,6 +450,7 @@ async function computeRow( if (!baseline) return { ...base, note: 'not measurable: no baseline captured at apply time' } if (MCP_KINDS.has(rec.kind)) return mcpRow(base, rec, sessions, baseline, afterStart, now) + if (DEFER_KINDS.has(rec.kind)) return deferRow(base, sessions, baseline, afterStart, now, mcpClaimedServers) if (rec.kind in ARCHIVE_DEF_TOKENS) return archiveRow(base, rec, sessions, baseline, afterStart, now) if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now) if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' } @@ -434,6 +507,20 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise() + for (const r of active) { + if (!MCP_KINDS.has(r.kind) || !r.baseline) continue + for (const server of Object.keys(r.baseline.metrics)) mcpClaimedServers.add(server) + } + const rows: ActReportRow[] = [] for (const rec of eligible) { const afterStart = new Date(Math.max(new Date(rec.at).getTime(), windowStart.getTime())) @@ -441,7 +528,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise r.status === 'measured' && isTokenKind(r.kind)) @@ -483,6 +570,7 @@ export function buildOptimizeAppliedHeader(report: ActReport): string | null { function realizedCell(r: ActReportRow): string { if (r.status === 'reverted') return 'reverted' + if (r.status === 'pending') return 'not yet in effect' if (r.status === 'not-measurable') return 'not measurable' if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)` if (r.kind === 'model-default') return 'correlation' @@ -585,7 +673,15 @@ function mcpServersFromApply(finding: WasteFinding): string[] { } function needsConfigBaseline(kind: ActionKind): boolean { - return MCP_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config' + return MCP_KINDS.has(kind) || DEFER_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config' +} + +// Servers whose upfront schema deferral removes from the prefix. defer-alwaysload +// names them; defer-enable / defer-threshold re-enable deferral across the whole +// observed MCP surface. +function deferServers(finding: WasteFinding, ctx: CaptureCtx): string[] { + if (finding.apply?.kind === 'defer-alwaysload') return finding.apply.servers.map(s => s.server) + return observedMcpServers(ctx.projects) } export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: CaptureCtx): ActionBaseline | undefined { @@ -608,6 +704,19 @@ export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: Ca return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics } } + if (DEFER_KINDS.has(kind)) { + const servers = deferServers(finding, ctx) + if (servers.length === 0) return undefined + const covByServer = new Map(ctx.coverage.map(c => [c.server, c])) + const metrics: Record = {} + for (const server of servers) { + const cov = covByServer.get(server) + const tools = cov && cov.toolsAvailable > 0 ? cov.toolsAvailable : TOOLS_PER_MCP_SERVER + metrics[server] = tools * TOKENS_PER_MCP_TOOL + } + return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics } + } + const defTokens = ARCHIVE_DEF_TOKENS[kind] if (defTokens !== undefined) { const names = finding.apply?.kind === 'archive' ? finding.apply.names : [] diff --git a/tests/act-report.test.ts b/tests/act-report.test.ts index d690f138..80834d0d 100644 --- a/tests/act-report.test.ts +++ b/tests/act-report.test.ts @@ -7,10 +7,12 @@ import { journalPath } from '../src/act/journal.js' import { buildActReportJson, buildOptimizeAppliedHeader, + captureBaseline, computeActReport, renderActReport, } from '../src/act/report.js' import type { ActionRecord } from '../src/act/types.js' +import type { WasteFinding } from '../src/optimize.js' import type { ClassifiedTurn, ProjectSummary } from '../src/types.js' type Session = ProjectSummary['sessions'][number] @@ -584,3 +586,179 @@ describe('json + render shape', () => { expect(out).toMatch(/scaled to the measured window/) }) }) + +// --------------------------------------------------------------------------- +// defer-* realized deltas (part 2 of #614) +// --------------------------------------------------------------------------- + +function deferRecord(over: Partial = {}): ActionRecord { + const at = daysAgo(10) + return { + id: 'd1', + at, + kind: 'defer-enable', + findingId: 'mcp-deferral-off', + description: 'Remove the ENABLE_TOOL_SEARCH=false override from settings.json', + changes: [], + status: 'applied', + // 2 servers x 2000 tokens/session = 4000 prefix tokens/session. + baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 40_000, sessions: 5, metrics: { everything: 2000, 'fs-tools': 2000 } }, + ...over, + } +} + +// NOTE: an mcpInventory on a post-apply session means the OPPOSITE for defer-* +// vs mcp-remove. For mcp-remove it is "the server loaded again" (reverted); +// for defer-* it is "deferral became active" (saved). Same fixture, inverse +// meaning — exactly the design. +const DEFERRED = { mcpInventory: ['mcp__everything__get-sum'] } + +describe('defer realized delta', () => { + it('measures savings across post-apply sessions where deferral became active', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(20_000) // 4000/session * 5 deferred sessions + expect(row.estimatedForWindow).toBe(20_000) + expect(report.totalRealizedTokens).toBe(20_000) + }) + + it('reports "not yet in effect" with zero savings when no post-apply session shows deferral', async () => { + const actionsDir = await writeJournal([deferRecord()]) + // sessions with MCP activity but NO inventory = deferral still off + const off = sessionsAt(4, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 } } }) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(off)]) }) + + const row = report.rows[0]! + expect(row.status).toBe('pending') + expect(row.realizedTokens ?? 0).toBe(0) + expect(row.note).toMatch(/not yet in effect/) + expect(report.totalRealizedTokens).toBe(0) + }) + + it('counts only the sessions where deferral actually became active (partial)', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const active = sessionsAt(3, daysAgo(5), DEFERRED) + const stillOff = sessionsAt(2, daysAgo(4)) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([...active, ...stillOff])]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(12_000) // 4000 * 3 active (2 still-off excluded) + expect(row.estimatedForWindow).toBe(20_000) // 4000 * all 5 window sessions + }) + + it('is not measurable when no post-apply sessions exist yet', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([])]) }) + expect(report.rows[0]!.note).toMatch(/no sessions in the window yet/) + }) + + it('is not measurable with an empty baseline (zero prefix tokens)', async () => { + const rec = deferRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 0, sessions: 5, metrics: { everything: 0 } } }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.note).toMatch(/empty baseline/) + }) + + it('falls back to the no-baseline note for records applied before baselines existed', async () => { + const rec = deferRecord({ baseline: undefined }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.note).toMatch(/no baseline captured at apply time/) + }) + + it('measures defer-alwaysload against its named servers', async () => { + const rec = deferRecord({ + kind: 'defer-alwaysload', + findingId: 'mcp-alwaysload-hygiene', + description: 'Unpin an alwaysLoad MCP server', + baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 30_000, sessions: 6, metrics: { 'heavy-server': 5000 } }, + }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(6, daysAgo(5), DEFERRED))]) }) + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(30_000) // 5000 * 6 + }) + + it('measures defer-threshold like the other defer kinds', async () => { + const rec = deferRecord({ kind: 'defer-threshold', findingId: 'mcp-defer-threshold', description: 'Tighten the auto threshold' }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.status).toBe('measured') + expect(report.rows[0]!.realizedTokens).toBe(20_000) + }) + + it('excludes servers an mcp-remove row already claims, so the two rows never double count', async () => { + const actionsDir = await writeJournal([ + mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 10_000, sessions: 5, metrics: { everything: 2000 } } }), + deferRecord(), + ]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), { mcpInventory: ['mcp__fs-tools__ls'] }))]) }) + + const mcpRow = report.rows.find(r => r.kind === 'mcp-remove')! + const deferRow = report.rows.find(r => r.kind === 'defer-enable')! + expect(mcpRow.status).toBe('measured') + expect(mcpRow.realizedTokens).toBe(10_000) // 2000 x 5 saved sessions + expect(deferRow.status).toBe('measured') + expect(deferRow.realizedTokens).toBe(10_000) // 'fs-tools' only; 'everything' is claimed by the mcp row + expect(report.totalRealizedTokens).toBe(20_000) // disjoint sum; without dedup it would be 30_000 + }) + + it('is not measurable when every deferred server is already claimed by an MCP row', async () => { + const actionsDir = await writeJournal([ + mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 20_000, sessions: 5, metrics: { everything: 2000, 'fs-tools': 2000 } } }), + deferRecord(), + ]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5)))]) }) + + const deferRow = report.rows.find(r => r.kind === 'defer-enable')! + expect(deferRow.status).toBe('not-measurable') + expect(deferRow.realizedTokens ?? 0).toBe(0) + expect(deferRow.note).toMatch(/already measured by an MCP remove\/scope row/) + }) + + it('surfaces the pending status to --json consumers instead of asserting a revert', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const off = sessionsAt(4, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 } } }) + const json = buildActReportJson(await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(off)]) })) as { actions: Array<{ status: string; realizedTokens: number | null; note: string }> } + expect(json.actions[0]!.status).toBe('pending') + expect(json.actions[0]!.realizedTokens).toBeNull() + expect(json.actions[0]!.note).toMatch(/not yet in effect/) + }) +}) + +describe('defer baseline capture', () => { + const finding = (apply: WasteFinding['apply']): WasteFinding => ({ + id: 'mcp-deferral-off', + title: 't', explanation: 'e', impact: 'medium', tokensSaved: 40_000, + fix: { type: 'command', label: 'l', text: 'x' }, + apply, + }) + const ctx = (projects: ProjectSummary[]) => ({ projects, coverage: [], windowDays: 14, now: NOW }) + + it('derives servers from observed MCP usage for defer-enable', () => { + const projects = [projectOf(sessionsAt(3, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 }, 'fs-tools': { calls: 1, savingsUSD: 0, costUSD: 0 } } }))] + const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects)) + expect(b).toBeDefined() + // no coverage -> 5 tools x 400 fallback per server + expect(b!.metrics.everything).toBe(2000) + expect(b!.metrics['fs-tools']).toBe(2000) + }) + + it('uses the named servers for defer-alwaysload', () => { + const projects = [projectOf(sessionsAt(2, daysAgo(5)))] + const b = captureBaseline(finding({ kind: 'defer-alwaysload', servers: [{ server: 'pinned', paths: ['/a/.mcp.json'] }] }), 'defer-alwaysload', ctx(projects)) + expect(b).toBeDefined() + expect(Object.keys(b!.metrics)).toEqual(['pinned']) + }) + + it('returns undefined when there is no observed MCP surface to defer', () => { + const projects = [projectOf(sessionsAt(3, daysAgo(5)))] // no mcpBreakdown, no inventory + const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects)) + expect(b).toBeUndefined() + }) +})