Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
310 changes: 218 additions & 92 deletions core/src/autonomous/report/html.ts

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion core/src/autonomous/report/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@ export interface AutonomousReport {
operatorModel: string;
/** Optional — older reports predate these fields. */
scoutModel?: string;
/** Falls back to the commander model when unset. */
/** Unset means the run never used the independent verifier (no --verify flag). When
* verification IS on and no dedicated model was configured, this falls back to the
* commander model, but it is only ever set at all if verification actually ran. */
verifierModel?: string;
/** Whether the run was truncated by a budget/turn ceiling. */
truncated: boolean;
Expand Down
4 changes: 2 additions & 2 deletions core/src/execute/runAll.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,11 +317,11 @@ async function curateTracesIfConfigured(
/**
* Label for the "Evaluation Suite" report field. Only a `mode: "suite"` selection
* runs a whole named suite verbatim — explicit evaluator lists and preloaded specs
* are, by definition, a hand-picked subset, so they're labeled "Custom Suite"
* are, by definition, a hand-picked subset, so they're labeled "Custom"
* rather than borrowing a suite name that would overstate what actually ran.
*/
function suiteLabel(selection: RunConfig["selection"]): string {
return selection.mode === "suite" ? selection.suite : "Custom Suite";
return selection.mode === "suite" ? selection.suite : "Custom";
}

/** Assemble a {@link UnifiedRunReport} from Node-side run config and evaluator results. */
Expand Down
2 changes: 1 addition & 1 deletion core/src/execute/runAllBrowser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ function buildBrowserReport(
// The browser path takes preloaded evaluator specs with no suite concept —
// the extension builds its own report/suite label independently of this
// return value (see popup.js's buildFinalReport).
suiteId: "Custom Suite",
suiteId: "Custom",
effort: config.effort,
attackModel,
judgeModel,
Expand Down
2 changes: 1 addition & 1 deletion core/src/execute/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ export interface UnifiedRunReport {
targetName: string;
targetKind: "agent" | "mcp";
/**
* The evaluation suite that was run, or "Custom Suite" when the selection
* The evaluation suite that was run, or "Custom" when the selection
* doesn't correspond to one whole named suite (an explicit evaluator list,
* preloaded specs, or a hand-picked subset of a suite's evaluators).
*/
Expand Down
267 changes: 160 additions & 107 deletions core/src/report/render.ts

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions docs/hunt.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,11 @@ respond before it's killed.

### Models

| Option | Default |
| ----------------------- | -------- |
| `--model <id>` | `sonnet` |
| `--operator-model <id>` | `sonnet` |
| `--scout-model <id>` | `haiku` |
| Option | Default |
| ------------------------ | -------- |
| `--commander-model <id>` | `sonnet` |
| `--operator-model <id>` | `sonnet` |
| `--scout-model <id>` | `haiku` |

### Limits

Expand Down Expand Up @@ -203,7 +203,7 @@ The credential actually in use is printed at startup (`Authenticating via: …`)

### Pinning model snapshots

`--model`, `--operator-model`, and `--scout-model` take the aliases `haiku` / `sonnet` / `opus`. To pin those aliases to specific snapshots — for a gateway that only exposes certain ids, or to freeze behaviour across runs — set:
`--commander-model`, `--operator-model`, and `--scout-model` take the aliases `haiku` / `sonnet` / `opus`. To pin those aliases to specific snapshots — for a gateway that only exposes certain ids, or to freeze behaviour across runs — set:

```bash
ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-haiku-4-5-20251001
Expand Down
8 changes: 4 additions & 4 deletions runners/cli/src/commands/hunt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ interface HuntCliOptions {
targetModel?: string;
header?: string[];
name?: string;
model: string;
commanderModel: string;
operatorModel: string;
scoutModel: string;
maxOperators: string;
Expand Down Expand Up @@ -189,7 +189,7 @@ export function registerHuntCommand(program: Command): void {
(v: string, acc: string[]) => [...acc, v],
[]
)
.option("--model <id>", "Commander model (alias or id)", "sonnet")
.option("--commander-model <id>", "Commander model (alias or id)", "sonnet")
.option("--operator-model <id>", "Operator subagent model", "sonnet")
.option("--scout-model <id>", "Scout subagent model", "haiku")
.option("--max-operators <n>", "Max parallel operator subagents", "6")
Expand Down Expand Up @@ -274,7 +274,7 @@ export function registerHuntCommand(program: Command): void {
targetName: opts.name,
objective: opts.objective,
apiKeyEnv: opts.targetKeyEnv,
commanderModel: opts.model,
commanderModel: opts.commanderModel,
operatorModel: opts.operatorModel,
scoutModel: opts.scoutModel,
maxOperators: opts.maxOperators,
Expand Down Expand Up @@ -453,7 +453,7 @@ export function registerHuntCommand(program: Command): void {
const huntOptions: HuntOptions = {
target,
objective,
commanderModel: opts.model,
commanderModel: opts.commanderModel,
operatorModel: opts.operatorModel,
scoutModel: opts.scoutModel,
maxOperators: intOr(opts.maxOperators, 6),
Expand Down
6 changes: 3 additions & 3 deletions runners/extension/popup.js
Original file line number Diff line number Diff line change
Expand Up @@ -1155,12 +1155,12 @@ function judgedCount(summary) {
* Label for the report's "Evaluation Suite" field. A suite name is only
* accurate when every one of its evaluators was actually selected — picking
* a subset (or the catch-all "Custom Evaluators" bucket) is a hand-picked
* list, not that suite, so it's labeled "Custom Suite" instead of borrowing
* list, not that suite, so it's labeled "Custom" instead of borrowing
* a name that would overstate what actually ran.
*/
function resolveSuiteLabel() {
const suite = state.catalog?.suites?.find((s) => s.id === state.suiteId);
if (!suite) return "Custom Suite";
if (!suite) return "Custom";
// Based on what actually completed, not what was selected — a run stopped
// partway through a full-suite selection is not that suite either.
const completedIds = new Set(
Expand All @@ -1169,7 +1169,7 @@ function resolveSuiteLabel() {
const fullSuite =
suite.evaluatorIds.length === completedIds.size &&
suite.evaluatorIds.every((id) => completedIds.has(id));
return fullSuite ? suite.id : "Custom Suite";
return fullSuite ? suite.id : "Custom";
}

function buildReport() {
Expand Down
2 changes: 1 addition & 1 deletion runners/sdk/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ export interface RunResults {
id: string;
timestamp: string;
targetName: string;
/** The evaluation suite that was run, or "Custom Suite" for a hand-picked selection. */
/** The evaluation suite that was run, or "Custom" for a hand-picked selection. */
suiteId: string;
targetKind: "agent" | "mcp";
effort: Effort;
Expand Down
Loading