feat: track and display LLM token usage across CLI, reports, and extension

AGENTS.md

@@ -195,6 +195,7 @@ npm test                          # vitest in core/
 | `core/src/execute/runAgentLoop.ts`                             | Thin wrapper: `runAgentAttack(...)` = `runAttack(new AgentAttackDriver(...))`. Shared by the Node (`evaluatorLoop`) and browser (`runAllBrowser`) loops.                                                                                                                                                                                                                   |
 | `core/src/execute/baselineScanner.ts`                          | MCP-only pre-flight scans run before evaluator attacks (tool-poisoning, resource PII/secret leakage, etc.).                                                                                                                                                                                                                                                                |
 | `core/src/execute/runListener.ts`                              | `RunListener` observer SPI — run-level (`onRunStart/Finish/Error`) + per-attack progress hooks. CLI attaches `ConsoleProgressListener` + `JsonlEventListener` (NDJSON via `--events`).                                                                                                                                                                                     |
+| `core/src/execute/tokenTracker.ts`                             | `TokenTracker` class — lightweight accumulator for LLM token usage. Created per-run in `runAll`/`runAllBrowser`, per-evaluator child trackers aggregate into the parent. Auto-records from `withRetry` and bare `generateText` results.                                                                                                                                    |
 | `core/src/execute/aggregate.ts`                                | Folds `AttackResult`s into `EvaluatorResult` / `UnifiedRunReport` (`toEvaluatorResult`, `buildUnifiedReport`, `summarizeVerdicts`).                                                                                                                                                                                                                                        |
 | `core/src/execute/runAllBrowser.ts`                            | Browser-safe variant: takes preloaded evaluators + a pre-built `AgentTarget`, no Node-only imports                                                                                                                                                                                                                                                                         |
 | `core/src/generate/generateAttacks.ts`                         | Generates `AttackSpec[]` for one evaluator — agent-prompt or MCP tool-call shape                                                                                                                                                                                                                                                                                           |
@@ -278,6 +279,8 @@ There is no longer a separate `generate` step. `opfor run --config <file>` does
 
 **Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. When aborted, the evaluator loop finishes the in-flight attack, skips remaining evaluators/attacks, and returns a partial report with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation.
 
+**Token usage tracking.** `runAll` creates a `TokenTracker` (see `core/src/execute/tokenTracker.ts`) and threads it through the evaluator loop → attack drivers → `withRetry` / `generateText` / `chatCompletionJsonContent` call sites. Every LLM call auto-records its `usage` (input/output tokens). Per-evaluator child trackers aggregate into evaluator-level totals (`EvaluatorResult.tokenUsage`); the parent accumulates run-level totals (`UnifiedRunReport.summary.tokenUsage`). The CLI prints a summary line, the HTML report shows a stat card, and the JSON report includes the data for CI. `runAllBrowser` follows the same pattern so the extension popup can show a token count.
+
 `runAllBrowser` is the same loop in browser-safe form: takes preloaded `EvaluatorSpec[]` + a pre-built `AgentTarget` (e.g. `DomTarget`), skips disk reads.
 
 ---

core/src/evaluators/judge.ts

@@ -10,6 +10,7 @@ import { formatUpstreamSessions } from "../lib/summarizeSessionContext.js";
 import { log } from "../lib/logger.js";
 import { JUDGE_AGENT_SYSTEM } from "../prompts/judge-agent.js";
 import { withRetry, isStopError } from "../lib/llmRetry.js";
+import type { TokenTracker } from "../execute/tokenTracker.js";
 import { errorJudge, type JudgeResult, type Verdict } from "../lib/judgeTypes.js";
 import { verdictParser } from "./verdictParser.js";
 
@@ -118,7 +119,8 @@ export async function judgeResponse(
   observability?: JudgeObservabilityContext,
   conversationHistory?: ConversationTurn[],
   attackContext?: AttackContext,
-  upstreamSessions?: SessionContext[]
+  upstreamSessions?: SessionContext[],
+  tokenTracker?: TokenTracker
 ): Promise<JudgeResult> {
   const obsLines: string[] = [];
   if (observability?.propagatedTraceId?.trim()) {
@@ -204,7 +206,7 @@ export async function judgeResponse(
   try {
     const result = await withRetry(
       () => generateText({ model, system: JUDGE_SYSTEM, prompt: judgePrompt }),
-      { context: "Judge", maxRetries: 3 }
+      { context: "Judge", maxRetries: 3, tokenTracker }
     );
     return parseJudgeOutput(result.text);
   } catch (err) {

core/src/execute/agentAttackDriver.ts

@@ -19,6 +19,7 @@ import type { TelemetryConfig } from "../config/types.js";
 import type { UnifiedTargetConfig } from "./types.js";
 import { ConversationHistory } from "./conversationHistory.js";
 import type { AttackDriver } from "./attackRunner.js";
+import type { TokenTracker } from "./tokenTracker.js";
 
 export interface AgentAttackContext {
   targetConfig?: UnifiedTargetConfig;
@@ -32,6 +33,7 @@ export interface AgentAttackContext {
    * id, capture whatever the target returns).
    */
   initialSessionId?: string;
+  tokenTracker?: TokenTracker;
 }
 
 /**
@@ -136,6 +138,7 @@ export class AgentAttackDriver implements AttackDriver<string, string> {
       traceContext: this.attack.traceContext,
       previousTechnique: this.previousTechnique,
       upstreamSessions: this.attack.upstreamSessions,
+      tokenTracker: this.context?.tokenTracker,
     });
     this.previousTechnique = result.technique;
     log.dim(
@@ -218,7 +221,8 @@ export class AgentAttackDriver implements AttackDriver<string, string> {
               ),
               this.history.size > 2 ? this.history.messages : undefined,
               { patternName: this.attack.patternName, judgeHint: this.attack.judgeHint },
-              this.attack.upstreamSessions
+              this.attack.upstreamSessions,
+              this.context?.tokenTracker
             );
 
     return {

core/src/execute/aggregate.ts

@@ -39,6 +39,7 @@ const SEVERITY_WEIGHTS: Record<string, number> = {
   low: 1,
 };
 
+/** Return the numeric weight for a severity level (critical=4 … low=1). */
 function severityWeight(severity: string): number {
   return SEVERITY_WEIGHTS[severity.toLowerCase()] ?? 2;
 }

core/src/execute/evaluatorLoop.ts

@@ -16,6 +16,7 @@ import { errorJudge } from "../lib/judgeTypes.js";
 import { verdictIcon } from "../lib/verdictIcon.js";
 import { TurnPlan } from "./turnPlan.js";
 import { isStopError, getStopReason } from "../lib/llmRetry.js";
+import type { TokenTracker } from "./tokenTracker.js";
 import { log } from "../lib/logger.js";
 import type {
   RunConfig,
@@ -43,6 +44,8 @@ export interface EvaluatorLoopContext {
   notify: (event: ProgressEvent) => void;
   /** Cancellation signal — when aborted, the loop finishes the in-flight attack then stops. */
   signal?: AbortSignal;
+  /** Token usage accumulator. Per-evaluator children are auto-created in the loop. */
+  tokenTracker?: TokenTracker;
 }
 
 /**
@@ -63,6 +66,7 @@ export async function runEvaluatorAttacks(
     traceContext,
     notify,
     signal,
+    tokenTracker,
   } = ctx;
   const sessionMap = new Map<string, SessionContext>();
   const evaluatorResults: EvaluatorResult[] = [];
@@ -97,6 +101,7 @@ export async function runEvaluatorAttacks(
     }
 
     const { turnMode, effectiveTurns } = TurnPlan.from(config);
+    const evalTracker = tokenTracker?.child();
 
     let attacks: AttackSpec[];
     try {
@@ -113,6 +118,7 @@ export async function runEvaluatorAttacks(
           upstreamSessions,
           attackObjective: config.attackObjective,
           businessUseCase: config.businessUseCase,
+          tokenTracker: evalTracker,
         },
       });
     } catch (err) {
@@ -172,15 +178,19 @@ export async function runEvaluatorAttacks(
       try {
         result =
           attack.kind === "mcp"
-            ? await runMcpAttack(attack, mcpTarget!, attackModel, judgeLlmConfig)
+            ? await runMcpAttack(attack, mcpTarget!, attackModel, judgeLlmConfig, evalTracker)
             : await runAgentAttack(
                 attack,
                 attackModel,
                 judgeModel,
                 attack.id,
                 evaluator.patterns,
                 ctx.agentTarget ?? createAgentTarget(config.target as AgentTargetConfig),
-                { targetConfig: config.target, telemetry: config.telemetry }
+                {
+                  targetConfig: config.target,
+                  telemetry: config.telemetry,
+                  tokenTracker: evalTracker,
+                }
               );
       } catch (err) {
         const makeFailedResult = (reason: string): AttackResult =>
@@ -215,7 +225,9 @@ export async function runEvaluatorAttacks(
         notify({ type: "run_stopped", reason: stopReason });
         attackResults.push(makeFailedResult(stopReason));
         notify({ type: "attack_done", attackId: attack.id, verdict: "ERROR" });
-        evaluatorResults.push(toEvaluatorResult(evaluatorMeta, attackResults));
+        const partialResult = toEvaluatorResult(evaluatorMeta, attackResults);
+        if (evalTracker) partialResult.tokenUsage = evalTracker.totals;
+        evaluatorResults.push(partialResult);
         return { evaluatorResults, stopReason };
       }
 
@@ -230,13 +242,16 @@ export async function runEvaluatorAttacks(
     const { passed, failed, errors } = summarizeVerdicts(attackResults);
     notify({ type: "evaluator_done", evaluatorId: evaluator.id, passed, failed, errors });
 
-    evaluatorResults.push(toEvaluatorResult(evaluatorMeta, attackResults));
+    const evResult = toEvaluatorResult(evaluatorMeta, attackResults);
+    if (evalTracker) evResult.tokenUsage = evalTracker.totals;
+    evaluatorResults.push(evResult);
     sessionMap.set(evaluator.id, captureSessionContext(evaluator, attackResults));
   }
 
   return { evaluatorResults };
 }
 
+/** Build a {@link SessionContext} from an evaluator's attack results for `dependsOn` dependents. */
 function captureSessionContext(
   evaluator: EvaluatorSpec,
   attackResults: AttackResult[]

core/src/execute/mcpAttackDriver.ts

@@ -11,6 +11,7 @@ import type { LlmConfig } from "../config/types.js";
 import type { McpTarget, McpToolCallResult } from "../targets/mcpTarget.js";
 import type { McpAttackSpec, McpTurnRecord, AttackResult } from "./types.js";
 import { runAttack, type AttackDriver } from "./attackRunner.js";
+import type { TokenTracker } from "./tokenTracker.js";
 
 /**
  * Drives one MCP attack: call a tool (seed args on turn 1, else adaptively
@@ -35,7 +36,8 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
     private readonly target: McpTarget,
     private readonly toolName: string,
     private readonly attackModel: LanguageModel,
-    private readonly judgeLlm: LlmConfig
+    private readonly judgeLlm: LlmConfig,
+    private readonly tokenTracker?: TokenTracker
   ) {
     this.judgeHint = attack.judgeHint;
     this.totalTurns = attack.turns;
@@ -50,7 +52,8 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
       `${this.attack.patternName} — ${this.attack.evaluatorName}`,
       this.toolName,
       this.attack.toolArguments ?? {},
-      this.attackModel
+      this.attackModel,
+      this.tokenTracker
     );
     if (next.judgeHint) this.judgeHint = next.judgeHint;
     return next.args;
@@ -146,6 +149,7 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
       toolError,
       judgeHint: this.judgeHint,
       priorTurns: this.mcpHistory.length > 1 ? this.mcpHistory.slice(0, -1) : undefined,
+      tokenTracker: this.tokenTracker,
     });
     return sanitizeJudgeResult(result, {
       attackSummary: this.attack.patternName,
@@ -164,7 +168,8 @@ export async function runMcpAttack(
   attack: McpAttackSpec,
   target: McpTarget,
   attackModel: LanguageModel,
-  judgeLlm: LlmConfig
+  judgeLlm: LlmConfig,
+  tokenTracker?: TokenTracker
 ): Promise<AttackResult> {
   if (!attack.toolName) {
     return {
@@ -179,5 +184,7 @@ export async function runMcpAttack(
       judge: mcpErrorJudge("no toolName in attack spec"),
     };
   }
-  return runAttack(new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm));
+  return runAttack(
+    new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm, tokenTracker)
+  );
 }

core/src/execute/runAll.ts

@@ -16,6 +16,7 @@ import { createModel } from "../providers/factory.js";
 import type { LlmConfig } from "../config/types.js";
 import { getAdapter } from "../telemetry/adapter.js";
 import { runSetupTraceCuration } from "../telemetry/curation.js";
+import { TokenTracker } from "./tokenTracker.js";
 import { log } from "../lib/logger.js";
 
 export interface RunAllOptions {
@@ -62,6 +63,7 @@ export async function runAll(
   // if listTools() throws after connect). Everything else — model + evaluator
   // resolution included — runs inside the try so any failure reaches onRunError.
   let mcpTarget: Awaited<ReturnType<typeof createMcpTarget>> | null = null;
+  const tokenTracker = new TokenTracker();
 
   try {
     const attackModel = resolveModel(config.attackerLlm);
@@ -130,12 +132,17 @@ export async function runAll(
       agentTarget: options?.agentTarget,
       notify,
       signal: options?.signal,
+      tokenTracker,
     });
     evaluatorResults.push(...loop.evaluatorResults);
     const stopReason = loop.stopReason;
 
-    // Build report (partial or complete) with stop reason if applicable.
+    // Build report (partial or complete) with stop reason and token usage.
     const report = buildReport(config, evaluatorResults);
+    const usage = tokenTracker.totals;
+    if (usage.totalTokens > 0) {
+      report.summary.tokenUsage = usage;
+    }
     if (stopReason) {
       (report as UnifiedRunReport & { stopReason?: string }).stopReason = stopReason;
     }
@@ -269,10 +276,12 @@ function applyConfigDependsOn(
   });
 }
 
+/** Create a Vercel AI SDK LanguageModel from an opfor LlmConfig. */
 function resolveModel(cfg: LlmConfig): LanguageModel {
   return createModel(cfg);
 }
 
+/** Fetch and curate telemetry traces (if configured) to ground attacks in real usage. */
 async function curateTracesIfConfigured(
   config: RunConfig,
   model: LanguageModel,
@@ -298,6 +307,7 @@ async function curateTracesIfConfigured(
   }
 }
 
+/** Assemble a {@link UnifiedRunReport} from Node-side run config and evaluator results. */
 function buildReport(config: RunConfig, evaluators: EvaluatorResult[]): UnifiedRunReport {
   const { attackModel, judgeModel } = modelLabel(config.attackerLlm, config.judgeLlm);
   return buildUnifiedReport(