diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2ec4540..56ea5fc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -45,7 +45,8 @@ updates: ignore: # Playwright ships the browser build that gate measurements run against, so a # major bump changes churn results. Bump it deliberately in its own PR - # alongside experiments/gate-v1/versions.json. Security updates still land. + # alongside scripts/testbed/matrix.json (the single version matrix since + # #26 deleted experiments/gate-v1/versions.json). Security updates still land. - dependency-name: playwright update-types: - version-update:semver-major diff --git a/docs/architecture.md b/docs/architecture.md index c72ad69..999169c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,7 +111,7 @@ record → compile → cache-write → replay) is the issue that closes both. | `src/runner/` | Replay a compiled program in Playwright; repair actions only on failure; emit measured metrics | `src/runner/index.ts` (library only — driven by `experiments/gate-v1/run-matrix.ts`) | `cache-row.schema.json`, `assertion.schema.json` shapes (via `CompiledProgram`) | none directly — emits through `src/metrics/` | [gate/runner.md](./gate/runner.md) | | `src/metrics/` | Cost arithmetic, NDJSON emitter, PRD §9 aggregates that report `no_data` on an empty denominator | `src/metrics/index.ts` (library only) | `metrics.schema.json` (`readMetricNdjson`) | `metrics.schema.json` | §9 sections in [prd/PRD-trajectory-cache-v0.2.md](./prd/PRD-trajectory-cache-v0.2.md) | | `src/shared/` | **Not a pipeline stage.** In-page JS source strings two capture sites must run identically — today the `visible_landmarks` enumeration | `src/shared/index.ts` (library only) | none | none — feeds the `trajectory.schema.json` `visible_landmarks` field written by the recorder | see below | -| `experiments/gate-v1/` | Throwaway harness: walk the version list, emit rows, render the report. **Not a product API** | `npm run gate:matrix`, `npm run gate:report` | `metrics.schema.json`, local `versions.json` | `metrics.schema.json` | [experiments/gate-v1/README.md](../experiments/gate-v1/README.md) | +| `experiments/gate-v1/` | Throwaway harness: walk the version list, emit rows, render the report. **Not a product API** | `npm run gate:matrix`, `npm run gate:report` | `metrics.schema.json`, `scripts/testbed/matrix.json` (via `src/testbed/matrix.ts`) | `metrics.schema.json` | [experiments/gate-v1/README.md](../experiments/gate-v1/README.md) | `src/cache/` has no spec doc under `gate/`; its contract is [privacy/boundary-spec.md](./privacy/boundary-spec.md) and the merge-blocking canary @@ -163,7 +163,7 @@ assertion against `assertion.schema.json`); the wrapper around them is not. | `experiments/gate-v1/out/report/report.{json,csv,html}`, `amortized.svg` | `npm run gate:report` | gitignored | | `scripts/testbed/.runtime//provisioning/` | `prepareProvisioningOverlay` on every testbed invocation, including `--dry-run` | gitignored (`scripts/testbed/.gitignore`) | | `experiments/gate-v1/fixtures/compiled-program.json` | hand-written | **committed** — stands in for a real compiled program, see break 2 | -| `experiments/gate-v1/versions.json` | hand-written | **committed** — one-element placeholder, see below | +| `experiments/gate-v1/out/matrix-run.json` | `npm run gate:matrix -- --dry-run` | gitignored — selection, versions walked, versions skipped with reason | Generated output that is gitignored **must stay that way**. The recorder's own artifacts are committed on purpose: they are fixtures, and `assertNoLiteralSecrets` runs on every write @@ -180,7 +180,7 @@ The section a new agent needs most. Blunt, and current as of `605c384` + ADR-000 | 1 | **Repair proposes nothing.** `StubRepairModelClient` always returns `corrected_action: null`, `tokens_in: 0`, `tokens_out: 0` | `src/runner/repair.ts:17-27` | Every repair attempt lands on `REPAIR_EXHAUSTED`. Self-heal rate is structurally 0; `cost_repair` tokens are structurally 0 | [#27](https://github.com/DevToolie/Paragent/issues/27) | | 2 | **`cost_fresh` is always zeros.** `ReplayRunner` defaults it to `zeroCost()` and no caller ever passes it | `src/runner/replay.ts:79`; no `costFresh` argument anywhere in `src/` or `experiments/` | The §9 kill line "repair cost ≥ 70% of fresh" has no denominator. `repairCostVsFresh` correctly returns `status: no_data` rather than a ratio | [#39](https://github.com/DevToolie/Paragent/issues/39) | | 3 | **The matrix runner refuses to run live.** `--dry-run` is mandatory; anything else exits 2 | `experiments/gate-v1/run-matrix.ts:49-54` | No browser is ever driven by the harness. Dry-run outcomes are hard-coded `PASS` with zero tokens — explicitly *not* a gate measurement | [#62](https://github.com/DevToolie/Paragent/issues/62) | -| 4 | **`versions.json` is a placeholder, not the matrix.** One entry, `id: "pending-b1@placeholder"` | `experiments/gate-v1/versions.json` | The gate harness walks one fake version while `scripts/testbed/matrix.json` holds the real eight ADR-0003 pins. The two files are unconnected | [#26](https://github.com/DevToolie/Paragent/issues/26) | +| 4 | ~~`versions.json` is a placeholder, not the matrix.~~ **Fixed by [#26](https://github.com/DevToolie/Paragent/issues/26)** — deleted; `run-matrix.ts` reads `scripts/testbed/matrix.json` through `src/testbed/matrix.ts` | `experiments/gate-v1/run-matrix.ts` | The harness now walks the real eight ADR-0003 pins, one run row each, and records skipped versions in `out/matrix-run.json`. **Still not a measurement** — dry-run outcomes remain hard-coded `PASS` with zero tokens (stub 3) | — | | 5 | **The cache has a write path only.** No read path, no persistence — the only `CacheStore` in the tree is `{ write(_row) { /* sink */ } }`; no `writeFile`/`appendFile` anywhere in `src/cache/` | `src/cache/pipeline.ts:125`; `src/cache/write.ts` | Nothing can be replayed *from* cache. There is no cache hit, so there is no cache hit-rate | [#63](https://github.com/DevToolie/Paragent/issues/63) | | 6 | **Confidence never moves.** `confidence`, `success_count`, `failure_count` are written as `0` and never updated by any code path | `src/compiler/compile.ts:85-87`; `src/cache/pipeline.ts:91` | PRD §5.3's self-invalidating, self-healing cache does not exist. The fields are shape, not behaviour | [#64](https://github.com/DevToolie/Paragent/issues/64) | | 7 | **Bundle → cache and bundle → runner are unwired *in the runtime*** (the two dashed edges above) | no import of `src/cache/` outside `src/cache/` and `tests/`; the `CompiledTrajectoryBundle` → `CompiledProgram` adapter lives in `tests/integration/pipeline.test.ts`, deliberately not in `src/` | Narrower than it was: since [#52](https://github.com/DevToolie/Paragent/issues/52) the seam **is** exercised end to end by the integration test, which caught a real compiler bug on its first run. What is still missing is a *product* path — nothing outside a test walks bundle → cache → replay | [#62](https://github.com/DevToolie/Paragent/issues/62), [#63](https://github.com/DevToolie/Paragent/issues/63) | diff --git a/docs/gate/runner.md b/docs/gate/runner.md index aa36c2a..1508d0e 100644 --- a/docs/gate/runner.md +++ b/docs/gate/runner.md @@ -70,3 +70,11 @@ npm run gate:report - Whether `compiled_trajectory` bundle `$id` becomes a first-class contract (B3 packaging convention today). - Fresh-reasoning cost capture for `cost_fresh` — measured separately; defaults to zeros when unwired. - Live `page` injection API for matrix vs caller-owned browser lifecycle — not locked yet. +- ~~Which version list the gate matrix walks.~~ **Settled (#26)** — `scripts/testbed/matrix.json` + (ADR-0003 pins), read through `src/testbed/matrix.ts`. The placeholder + `experiments/gate-v1/versions.json` is deleted, so `npm run gate:matrix -- --dry-run` now + emits one run row per pinned version instead of one row for `pending-b1@placeholder`. Still a + dry run: outcomes are hard-coded `PASS`, every token count is 0, and the rows say so. +- Whether walking eight versions changes anything the report can *conclude*. It does not — more + rows over the same hand-written 2-step program is a better-shaped denominator, not a + measurement. That waits on live execution ([#62](https://github.com/DevToolie/Paragent/issues/62)). diff --git a/experiments/gate-v1/README.md b/experiments/gate-v1/README.md index 6ee907e..6de58b4 100644 --- a/experiments/gate-v1/README.md +++ b/experiments/gate-v1/README.md @@ -4,7 +4,7 @@ doc_type: runbook status: draft owner: B4 created: 2026-07-24 -updated: 2026-07-25 +updated: 2026-07-27 confidence: MED supersedes: null sources_verified: true @@ -18,7 +18,6 @@ sources_verified: true | Path | Role | | --- | --- | -| `versions.json` | Pluggable version matrix (prefer B1 `scripts/testbed/matrix.json` when wired) | | `fixtures/compiled-program.json` | 2-step local demo program | | `fixtures/local-demo.html` | Static page for future live runs | | `trajectories/` | B2 recorded trajectories (fixture gate task) | @@ -26,16 +25,52 @@ sources_verified: true | `report/generate-amortized.ts` | SVG/CSV/HTML/JSON from NDJSON (`writeReport`) | | `out/` | Generated metrics + reports (gitignored) | +## The version list + +[`scripts/testbed/matrix.json`](../../scripts/testbed/matrix.json) — the ADR-0003 pins — is the +**only** version list, read through `src/testbed/matrix.ts`. This directory used to carry its +own `versions.json` holding a single `pending-b1@placeholder` entry, so every report generated +here was a report about nothing. It was deleted in +[#26](https://github.com/DevToolie/Paragent/issues/26) rather than kept in sync: two lists for +one thing drift, and its only non-duplicated fields (`site_key` / `task_key` overrides) were +actively misleading — they relabelled the local-demo program per version as if a Grafana run had +happened. + +Consequently **only `testbed_version` varies per run.** `site_key` and `task_key` stay whatever +the compiled program actually is. + +A version marked `"status": "unavailable"` in `matrix.json` is **skipped and recorded**, never +silently dropped — see `out/matrix-run.json`. + ## Commands ```bash -npm run gate:matrix -- --dry-run +npm run gate:matrix -- --dry-run # every available pin +npm run gate:matrix -- --dry-run --versions 11.0.0,12.0.0 # a subset +npm run gate:matrix -- --dry-run --versions all # explicit default npm run gate:report npm run recorder -- --fixture ``` -Dry-run emits zero-token step/run rows under `experiments/gate-v1/out/metrics.ndjson`. -Live runs exit **2** until B1 pins and Playwright live wiring land. +Unknown ids are rejected with the valid list, never defaulted: + +```text +gate:matrix: unknown version id(s): 99.0.0 +valid ids (scripts/testbed/matrix.json): 9.5.21, 10.0.13, 10.4.19, 11.0.0, 11.5.2, 12.0.0, 12.2.1, 13.0.3 +``` + +Dry-run emits zero-token step/run rows under `experiments/gate-v1/out/metrics.ndjson`, one run +row per version walked, each labelled `dry-run — tokens remain 0; not a gate measurement`. +Live runs exit **2** until Playwright live wiring lands ([#62](https://github.com/DevToolie/Paragent/issues/62)). + +Exit codes: **2** for a usage error or a live-run attempt; **1** if the selection walked zero +versions — an empty NDJSON is a missing denominator, not a successful run. + +### `out/matrix-run.json` — the skip ledger + +Written every run beside the NDJSON. Records the selection, `versions_in_matrix`, +`versions_walked`, and `versions_skipped[] {id, reason}`. Without it a later reader cannot tell +"8 pins, 3 unavailable" from "5 pins", and the denominator shrinks silently. Report paths (after `gate:report`): @@ -56,6 +91,12 @@ Empty NDJSON → scaffold with `status: no_data` and null values (never invented ## Open questions / what I could not verify -- Wire `versions.json` to ADR-0003 / `scripts/testbed/matrix.json` pins. +- ~~Wire `versions.json` to ADR-0003 / `scripts/testbed/matrix.json` pins.~~ **Done (#26)** — + `versions.json` is deleted and the harness reads the pins directly. What this does **not** + change: the run is still a dry run over a hand-written 2-step local-demo program, so walking + eight versions instead of one makes the row count honest, not the numbers meaningful. +- The program under test is still `fixtures/compiled-program.json`, whose own + `testbed_version` reads `pending-b1@placeholder` because it was compiled against no testbed at + all. That is accurate and left alone; the runner overrides it per version. - Fresh-reasoning baseline cost measurement path — not wired; `cost_fresh` stays zeros until measured. -- Live matrix against Grafana OSS — needs Docker + `--base-url` recording. +- Live matrix against Grafana OSS — needs Docker + `--base-url` recording ([#62](https://github.com/DevToolie/Paragent/issues/62)). diff --git a/experiments/gate-v1/run-matrix.ts b/experiments/gate-v1/run-matrix.ts index 60a9853..14ce903 100644 --- a/experiments/gate-v1/run-matrix.ts +++ b/experiments/gate-v1/run-matrix.ts @@ -1,79 +1,192 @@ #!/usr/bin/env node /** * Gate-v1 version matrix driver. - * Live Playwright runs are not enabled yet — --dry-run is required (exit 2 otherwise). + * + * The version list is `scripts/testbed/matrix.json` — the ADR-0003 pins — read + * through `src/testbed/matrix.ts`. There is no second list. `versions.json` + * used to hold a one-element placeholder (`pending-b1@placeholder`), so every + * report generated from this harness was a report about nothing; it was deleted + * rather than kept in sync (issue #26). + * + * Live Playwright runs are not enabled yet — --dry-run is required (exit 2 + * otherwise). That gating is deliberately unchanged here. */ -import { mkdir } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { MetricsEmitter } from "../../src/metrics/emitter.js"; import { ReplayRunner } from "../../src/runner/replay.js"; import type { CompiledProgram } from "../../src/runner/types.js"; import type { StepOutcome } from "../../src/metrics/types.js"; -import versionsDoc from "./versions.json" with { type: "json" }; +import { + isUnavailable, + listVersions, + loadMatrix, + type MatrixVersion, +} from "../../src/testbed/matrix.js"; import programFixture from "./fixtures/compiled-program.json" with { type: "json" }; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const OUT_DIR = path.join(__dirname, "out"); -function parseArgs(argv: string[]): { +const DRY_RUN_NOTE = "dry-run — tokens remain 0; not a gate measurement"; + +interface Args { dryRun: boolean; help: boolean; -} { - let dryRun = false; - let help = false; - for (const a of argv) { - if (a === "--dry-run") dryRun = true; - if (a === "--help" || a === "-h") help = true; + /** Raw --versions value; `undefined` means "not passed" (defaults to all). */ + versions?: string; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { dryRun: false, help: false }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i] ?? ""; + if (a === "--dry-run") args.dryRun = true; + else if (a === "--help" || a === "-h") args.help = true; + else if (a === "--versions") { + const value = argv[++i]; + if (value === undefined) { + throw new Error("--versions requires a value (comma-separated ids, or `all`)"); + } + args.versions = value; + } else if (a.startsWith("--versions=")) { + args.versions = a.slice("--versions=".length); + } else { + throw new Error(`unknown argument: ${a}`); + } } - return { dryRun, help }; + return args; } function usage(): void { - console.log(`Usage: npm run gate:matrix -- --dry-run + console.log(`Usage: npm run gate:matrix -- --dry-run [--versions |all] - --dry-run Required for now. Emits zero-token metrics; no browser. - --help Show this help. + --dry-run Required for now. Emits zero-token metrics; no browser. + --versions Comma-separated matrix ids, or "all" (default). + Unknown ids are rejected — never silently skipped. + --help Show this help. -Live matrix runs exit with code 2 until B1 pins and live wiring land. +Versions come from scripts/testbed/matrix.json (ADR-0003 pins). Versions marked +"status": "unavailable" there are skipped and recorded, never dropped silently. + +Live matrix runs exit with code 2 until live wiring lands (issue #62). +Exits 1 if the selection walked zero versions — an empty NDJSON is a missing +denominator, not a successful run. `); } +/** + * Resolve `--versions` against the matrix. Unknown ids are an error naming the + * valid ids: falling through to a default is how a placeholder run gets + * mistaken for a measurement. + */ +function selectVersions( + selection: string | undefined, + all: MatrixVersion[], +): { selected: MatrixVersion[]; selectionLabel: string | string[] } { + if (selection === undefined || selection.trim() === "all") { + return { selected: all, selectionLabel: "all" }; + } + + const requested = selection + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + if (requested.length === 0) { + throw new Error("--versions was empty (use comma-separated ids, or `all`)"); + } + + const byId = new Map(all.map((v) => [v.id, v])); + const unknown = requested.filter((id) => !byId.has(id)); + if (unknown.length > 0) { + throw new Error( + `unknown version id(s): ${unknown.join(", ")}\n` + + `valid ids (scripts/testbed/matrix.json): ${all.map((v) => v.id).join(", ")}`, + ); + } + + // Dedupe while keeping the caller's order. + const seen = new Set(); + const selected: MatrixVersion[] = []; + for (const id of requested) { + if (seen.has(id)) continue; + seen.add(id); + selected.push(byId.get(id)!); + } + return { selected, selectionLabel: selected.map((v) => v.id) }; +} + async function main(): Promise { - const args = parseArgs(process.argv.slice(2)); + let args: Args; + try { + args = parseArgs(process.argv.slice(2)); + } catch (err) { + console.error(`gate:matrix: ${err instanceof Error ? err.message : err}`); + usage(); + process.exit(2); + return; + } + if (args.help) { usage(); process.exit(0); + return; } if (!args.dryRun) { console.error( "gate:matrix: live runs are not enabled yet. Pass --dry-run (exit 2).", ); process.exit(2); + return; } + const matrix = loadMatrix(); + const all = listVersions(matrix); + + let selected: MatrixVersion[]; + let selectionLabel: string | string[]; + try { + ({ selected, selectionLabel } = selectVersions(args.versions, all)); + } catch (err) { + console.error(`gate:matrix: ${err instanceof Error ? err.message : err}`); + process.exit(2); + return; + } + + const walked = selected.filter((v) => !isUnavailable(v)); + const skipped = selected.filter(isUnavailable).map((v) => ({ + id: v.id, + reason: v.reason ?? "marked unavailable in scripts/testbed/matrix.json", + })); + await mkdir(OUT_DIR, { recursive: true }); const ndjsonPath = path.join(OUT_DIR, "metrics.ndjson"); + const summaryPath = path.join(OUT_DIR, "matrix-run.json"); const emitter = new MetricsEmitter(ndjsonPath); const program = programFixture as CompiledProgram; - const versions = versionsDoc.versions ?? []; console.log( - `gate:matrix dry-run — ${versions.length} version(s), program=${program.program_id}`, + `gate:matrix dry-run — ${matrix.target} matrix, ` + + `${walked.length} version(s) to walk, program=${program.program_id}`, ); + for (const s of skipped) { + console.log(` skipped ${s.id}: ${s.reason}`); + } + + const runs: Array> = []; - for (const ver of versions) { - const testbed = ver.testbed_version ?? ver.id; + for (const ver of walked) { + // Only testbed_version varies. site_key/task_key stay whatever the compiled + // program actually is — relabelling a local-demo program as a Grafana one + // per version would make the row claim a run that never happened. const runProgram: CompiledProgram = { ...program, - site_key: ver.site_key ?? program.site_key, - task_key: ver.task_key ?? program.task_key, - testbed_version: testbed, + testbed_version: ver.id, }; - // Placeholder matrix: first-pass PASS outcomes, zero tokens. No invented rates. const dryOutcomes: StepOutcome[] = runProgram.steps.map(() => "PASS"); const runner = new ReplayRunner({ dryRun: true, @@ -87,21 +200,58 @@ async function main(): Promise { resource_label: "widget", }); - console.log( - JSON.stringify({ - version: testbed, - run_id: result.run_id, - task_success: result.task_success, - repair_count: result.repair_count, - steps_replay_valid: result.steps_replay_valid, - steps_total: result.steps_total, - note: "dry-run — tokens remain 0; not a gate measurement", - }), - ); + const row = { + version: ver.id, + run_id: result.run_id, + task_success: result.task_success, + repair_count: result.repair_count, + steps_replay_valid: result.steps_replay_valid, + steps_total: result.steps_total, + note: DRY_RUN_NOTE, + }; + runs.push(row); + console.log(JSON.stringify(row)); } await emitter.flush(); + + // The skip ledger. Without it a later report cannot tell "8 versions, 3 of + // them unavailable" from "5 versions" — the denominator silently shrinks. + await writeFile( + summaryPath, + `${JSON.stringify( + { + schema_version: "1.0.0", + kind: "gate_matrix_run", + generated_at: new Date().toISOString(), + dry_run: true, + not_a_measurement: DRY_RUN_NOTE, + matrix_source: "scripts/testbed/matrix.json", + matrix_target: matrix.target, + selection: selectionLabel, + versions_in_matrix: all.length, + versions_walked: walked.map((v) => v.id), + versions_skipped: skipped, + program_id: program.program_id, + runs, + }, + null, + 2, + )}\n`, + "utf8", + ); + console.log(`wrote ${ndjsonPath}`); + console.log(`wrote ${summaryPath}`); + + if (walked.length === 0) { + console.error( + "gate:matrix: walked 0 versions — nothing was measured. " + + `${skipped.length} of ${selected.length} selected version(s) are unavailable.`, + ); + process.exit(1); + return; + } } main().catch((err) => { diff --git a/experiments/gate-v1/versions.json b/experiments/gate-v1/versions.json deleted file mode 100644 index ddad725..0000000 --- a/experiments/gate-v1/versions.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "schema_version": "1.0.0", - "matrix_id": "gate-v1", - "description": "Pluggable version matrix for gate-v1. B1 fills real pins; until then placeholders only.", - "versions": [ - { - "id": "pending-b1@placeholder", - "site_key": "local-demo", - "task_key": "local-demo-click", - "testbed_version": "pending-b1@placeholder", - "status": "placeholder", - "notes": "Awaiting B1 test-bed pin. Do not invent a console version here." - } - ] -} diff --git a/src/testbed/matrix.ts b/src/testbed/matrix.ts index 1bbfd08..eee59d8 100644 --- a/src/testbed/matrix.ts +++ b/src/testbed/matrix.ts @@ -10,6 +10,18 @@ export interface MatrixVersion { github_release_url: string; whats_new_url?: string; access_date: string; + /** + * `"unavailable"` marks a pin that could not be made to work (issue #23). + * The row stays in the matrix on purpose — a shrinking matrix is a finding, + * and the gate writeup has to disclose it — but the gate harness skips it. + * + * No pin carries it today: #23 booted and seeded all eight. It exists for the + * day one stops resolving, so the denominator shrinks loudly rather than + * silently. + */ + status?: string; + /** Why this version is `unavailable`. Required in practice when status is set. */ + reason?: string; } export interface Matrix { @@ -96,3 +108,16 @@ export function testdataTypeFor(versionId: string): string { export function listVersions(matrix = loadMatrix()): MatrixVersion[] { return [...matrix.versions]; } + +/** True when a pin is recorded as not working and must not be walked. */ +export function isUnavailable(version: MatrixVersion): boolean { + return version.status === "unavailable"; +} + +/** + * Versions the gate harness may actually walk. Callers must report what was + * skipped rather than quietly shrinking their denominator. + */ +export function availableVersions(matrix = loadMatrix()): MatrixVersion[] { + return listVersions(matrix).filter((v) => !isUnavailable(v)); +} diff --git a/tests/unit/testbed.test.ts b/tests/unit/testbed.test.ts index 7b84028..7235e12 100644 --- a/tests/unit/testbed.test.ts +++ b/tests/unit/testbed.test.ts @@ -3,9 +3,14 @@ import { parseArgs } from "../../src/testbed/args.js"; import { PACKAGE } from "../../src/testbed/constants.js"; import { buildComposeEnv } from "../../src/testbed/docker.js"; import { + availableVersions, getVersion, + isUnavailable, + listVersions, loadMatrix, testdataTypeFor, + type Matrix, + type MatrixVersion, } from "../../src/testbed/matrix.js"; import { composeProjectSlug } from "../../src/testbed/paths.js"; import { @@ -69,6 +74,48 @@ describe("testbed matrix", () => { }); }); +describe("matrix availability (#26)", () => { + // `npm run gate:matrix` walks this list. A version that cannot be made to work + // stays in the matrix — a shrinking matrix is a finding the gate writeup has + // to disclose — but must not be walked, and must not vanish from the count. + const withStatus = (id: string, status?: string, reason?: string): MatrixVersion => ({ + id, + image_tag: id, + released: "2025-01", + churn_role: "test", + docker_hub_tag_url: "https://example.invalid", + github_release_url: "https://example.invalid", + access_date: "2026-07-27", + ...(status === undefined ? {} : { status }), + ...(reason === undefined ? {} : { reason }), + }); + + const fake = (versions: MatrixVersion[]) => + ({ ...loadMatrix(), versions }) as Matrix; + + it("treats only `unavailable` as unwalkable", () => { + expect(isUnavailable(withStatus("1.0.0"))).toBe(false); + expect(isUnavailable(withStatus("1.0.0", "unavailable", "tag 404s"))).toBe(true); + // Any other status value is not a skip signal — do not guess. + expect(isUnavailable(withStatus("1.0.0", "verified"))).toBe(false); + }); + + it("filters unavailable versions out of the walkable list, keeping them in the matrix", () => { + const m = fake([ + withStatus("1.0.0"), + withStatus("2.0.0", "unavailable", "tag 404s"), + withStatus("3.0.0"), + ]); + expect(availableVersions(m).map((v) => v.id)).toEqual(["1.0.0", "3.0.0"]); + // The skipped row is still there to be counted and reported. + expect(listVersions(m)).toHaveLength(3); + }); + + it("walks every pinned version today — none is marked unavailable", () => { + expect(availableVersions()).toHaveLength(listVersions().length); + }); +}); + describe("compose project name", () => { // `docker compose -p paragent-tb-11.0.0` is rejected outright: project names // allow no dots. Every matrix id has them, so the slug must strip them.