From f94adec12841b93f5ffeeab969d0f78d5133dda1 Mon Sep 17 00:00:00 2001 From: johanlarar Date: Sat, 11 Jul 2026 00:32:10 +0200 Subject: [PATCH] docs: extend run-record schema v1 with agent/acceptance/normalization fields --- docs/benchmark/run-record.md | 51 +++++++++++++ docs/benchmark/run-record.schema.json | 105 ++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/docs/benchmark/run-record.md b/docs/benchmark/run-record.md index a37e06f..f9f9973 100644 --- a/docs/benchmark/run-record.md +++ b/docs/benchmark/run-record.md @@ -29,7 +29,28 @@ against [`run-record.schema.json`](./run-record.schema.json) | `metrics` | object | yes | Quantitative outcomes; see below. | | `metrics.score` | object | yes | Final `ctx score` metrics for the run's diff. Exactly the seven keys `complexity_delta`, `fan_out_delta`, `new_duplication`, `check_violations`, `symbols_added`, `symbols_removed`, `files_changed`, all numbers. | | `metrics.gate_evaluations` | integer >= 0 | yes | Number of gate evaluations performed during the run. | +| `metrics.gate_blocks` | integer >= 0 | no | Number of gate evaluations that blocked the agent. | +| `metrics.gate_block_recovered` | boolean | no | Whether the agent recovered (eventually passed the gate) after at least one block. | | `metrics.wall_clock_seconds` | number >= 0 | yes | Total wall-clock duration of the run in seconds. | +| `agent` | object | no | Agent-side accounting reported by the agent harness (e.g. `claude -p`); see below. | +| `agent.session_id` | string | yes (within `agent`) | Agent session identifier, for exact joins against gate logs and transcripts. | +| `agent.total_cost_usd` | number >= 0 | yes (within `agent`) | Total model cost of the run in USD. | +| `agent.total_tokens` | integer >= 0 | yes (within `agent`) | Total tokens (input + output) consumed by the run. | +| `agent.num_turns` | integer >= 0 | no | Number of agent turns; not guaranteed to be reported by `claude -p`. | +| `acceptance` | object | no | Result of the task's acceptance command -- the ctx-independent functional endpoint of the run; see below. | +| `acceptance.command` | string | yes (within `acceptance`) | Acceptance command that was executed. | +| `acceptance.exit_code` | integer | yes (within `acceptance`) | Exit code of the acceptance command. | +| `acceptance.passed` | boolean | yes (within `acceptance`) | Whether the acceptance command succeeded. | +| `acceptance.duration_seconds` | number >= 0 | yes (within `acceptance`) | Wall-clock duration of the acceptance command in seconds. | +| `normalization` | object | no | Diff-size data: the denominator for per-line deltas; see below. | +| `normalization.lines_added` | integer >= 0 | yes (within `normalization`) | Lines added by the run's final diff. | +| `normalization.lines_removed` | integer >= 0 | yes (within `normalization`) | Lines removed by the run's final diff. | +| `normalization.lines_changed` | integer >= 0 | yes (within `normalization`) | Total lines changed (added + removed) by the run's final diff. | +| `study_id` | string | no | Identifier of the study the run belongs to, so multiple studies can share one record store. | +| `task_seed` | integer | no | Seed used by the task generator to produce this run's task instance. | +| `generator_version` | string | no | Version of the task generator that produced the task instance. | +| `scorer_ctx_version` | string | no | Version of the ctx build used by the offline scorer; `ctx_version` remains the harness build. | +| `retry_attempt` | integer >= 0 | no | Zero-based retry attempt number when a run was retried after an infrastructure error. | | `notes` | string | no | Free-form operator notes. | The top-level object rejects unknown keys (`additionalProperties: false`). @@ -47,6 +68,12 @@ The top-level object rejects unknown keys (`additionalProperties: false`). (`https://docs.agentis.tools/schemas/run-record-v2.json`, ...). Old records are never rewritten; analysis code dispatches on `schema_version`. +The optional `agent`, `acceptance`, and `normalization` objects and the +`study_id`, `task_seed`, `generator_version`, `scorer_ctx_version`, +`retry_attempt`, `metrics.gate_blocks`, and `metrics.gate_block_recovered` +fields arrived additively (per the policy above) for the ctx-bench pilot +runner; `schema_version` remains `1`. + ## Example record ```json @@ -79,8 +106,32 @@ The top-level object rejects unknown keys (`additionalProperties: false`). "files_changed": 3 }, "gate_evaluations": 12, + "gate_blocks": 1, + "gate_block_recovered": true, "wall_clock_seconds": 2323.4 }, + "agent": { + "session_id": "8b2f4a1c-0d3e-4f6a-b7c8-9e0d1f2a3b4c", + "total_cost_usd": 1.87, + "total_tokens": 412034, + "num_turns": 41 + }, + "acceptance": { + "command": "cargo test --workspace", + "exit_code": 0, + "passed": true, + "duration_seconds": 148.2 + }, + "normalization": { + "lines_added": 96, + "lines_removed": 42, + "lines_changed": 138 + }, + "study_id": "pilot-2026-07", + "task_seed": 1337, + "generator_version": "0.1.0", + "scorer_ctx_version": "0.9.2", + "retry_attempt": 0, "notes": "One gate evaluation flagged a transient duplication warning." } ``` diff --git a/docs/benchmark/run-record.schema.json b/docs/benchmark/run-record.schema.json index a2e1e89..23a6241 100644 --- a/docs/benchmark/run-record.schema.json +++ b/docs/benchmark/run-record.schema.json @@ -126,6 +126,15 @@ "minimum": 0, "description": "Number of gate evaluations performed during the run." }, + "gate_blocks": { + "type": "integer", + "minimum": 0, + "description": "Optional: number of gate evaluations that blocked the agent during the run." + }, + "gate_block_recovered": { + "type": "boolean", + "description": "Optional: whether the agent recovered (eventually passed the gate) after at least one gate block." + }, "wall_clock_seconds": { "type": "number", "minimum": 0, @@ -133,6 +142,102 @@ } } }, + "agent": { + "type": "object", + "description": "Optional agent-side accounting reported by the agent harness (e.g. `claude -p` result payload).", + "additionalProperties": false, + "required": ["session_id", "total_cost_usd", "total_tokens"], + "properties": { + "session_id": { + "type": "string", + "description": "Agent session identifier, used for exact joins against gate logs and transcripts." + }, + "total_cost_usd": { + "type": "number", + "minimum": 0, + "description": "Total model cost of the run in USD." + }, + "total_tokens": { + "type": "integer", + "minimum": 0, + "description": "Total tokens (input + output) consumed by the run." + }, + "num_turns": { + "type": "integer", + "minimum": 0, + "description": "Number of agent turns. Optional within `agent`: not guaranteed to be reported by `claude -p`." + } + } + }, + "acceptance": { + "type": "object", + "description": "Optional result of the task's acceptance command -- the ctx-independent functional endpoint of the run (did the change actually work).", + "additionalProperties": false, + "required": ["command", "exit_code", "passed", "duration_seconds"], + "properties": { + "command": { + "type": "string", + "description": "Acceptance command that was executed (e.g. the task's test command)." + }, + "exit_code": { + "type": "integer", + "description": "Exit code of the acceptance command." + }, + "passed": { + "type": "boolean", + "description": "Whether the acceptance command succeeded." + }, + "duration_seconds": { + "type": "number", + "minimum": 0, + "description": "Wall-clock duration of the acceptance command in seconds." + } + } + }, + "normalization": { + "type": "object", + "description": "Optional diff-size normalization data: the denominator for per-line deltas when comparing runs of different sizes.", + "additionalProperties": false, + "required": ["lines_added", "lines_removed", "lines_changed"], + "properties": { + "lines_added": { + "type": "integer", + "minimum": 0, + "description": "Lines added by the run's final diff." + }, + "lines_removed": { + "type": "integer", + "minimum": 0, + "description": "Lines removed by the run's final diff." + }, + "lines_changed": { + "type": "integer", + "minimum": 0, + "description": "Total lines changed (added + removed) by the run's final diff." + } + } + }, + "study_id": { + "type": "string", + "description": "Optional identifier of the study this run belongs to, so multiple studies can share one record store." + }, + "task_seed": { + "type": "integer", + "description": "Optional seed used by the task generator to produce this run's task instance." + }, + "generator_version": { + "type": "string", + "description": "Optional version of the task generator that produced the task instance." + }, + "scorer_ctx_version": { + "type": "string", + "description": "Optional version of the ctx build used by the offline scorer. The existing `ctx_version` remains the harness build installed during the run." + }, + "retry_attempt": { + "type": "integer", + "minimum": 0, + "description": "Optional zero-based retry attempt number when a run was retried after an infrastructure error." + }, "notes": { "type": "string", "description": "Optional free-form operator notes about the run."