From f69806adecd2fd4df49d80097cc3cfe9a0723797 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:55:01 -0700 Subject: [PATCH 1/4] Key the snippet baseline by occurrence count, not by array index (#270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The baseline keyed on (file, locator, defect, detail), and locator embeds the evidence-array index. 114 baselined findings sat at index >= 1, so deleting a duplicate evidence item renumbered the survivors and turned unchanged findings into new ones — `just qc` failing on a change that strictly improved the corpus. The remedy a curator reaches for is --write-baseline, which re-freezes anything genuinely new in the same PR. That is how a ratchet rots. detail went too, for the same reason at a different granularity: it carries the full snippet for ELLIPTICAL/UNSUPPORTIVE and the DOI for MISSING, so retyping a still-elliptical snippet or correcting the DOI on a still-snippet-less reference also flipped the key. audit_causal_graphs.py learned this first and keys on only the leading fragment of its detail; the docstring claimed "same shape" and this was the one place it wasn't. Dropping both collapses findings onto shared keys, which is why #267 deferred the obvious fix: set membership would then let a THIRD missing snippet match a baselined pair and pass silently, trading a false positive for a false negative inside the integrity mechanism. So the baseline is read as a COUNT per key — "two of these were accepted" — and only occurrences in excess are new. Fewer than baselined is an improvement and passes. Canaried on the record the issue names. Deleting biofilm_formation's evidence[0] so evidence[1] renumbers: 2736 findings, 0 new, exit 0 — the case that used to fail. Adding a third snippet-less item: 2738 findings, 1 new, exit 1. The baseline file format is unchanged, so no regeneration: load_baseline aggregates the existing rows by key. 348 tests pass (6 new), `just qc` exits 0. Co-Authored-By: Claude Opus 5 --- scripts/audit_evidence_snippets.py | 60 ++++++++++++++++++++++++--- tests/test_audit_evidence_snippets.py | 56 +++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/scripts/audit_evidence_snippets.py b/scripts/audit_evidence_snippets.py index 3ffaaacab..1ad51083d 100644 --- a/scripts/audit_evidence_snippets.py +++ b/scripts/audit_evidence_snippets.py @@ -272,15 +272,63 @@ def audit(check_reports: bool = True) -> list[dict[str, str]]: return findings -def _key(row: dict[str, str]) -> tuple[str, str, str, str]: - return (row["file"], row["locator"], row["defect"], row["detail"]) +_INDEX_RE = re.compile(r"\[\d+\]$") -def load_baseline(path: Path) -> set[tuple[str, str, str, str]]: +def _key(row: dict[str, str]) -> tuple[str, str, str]: + """Baseline identity: file, locator WITHOUT its array index, and defect. + + The index has to go. `evidence[1]` renumbers to `evidence[0]` when item 0 is + deleted, so an unchanged finding became a new one and `just qc` failed on a + change that strictly improved the corpus — 114 baselined findings sat at + index >= 1. The remedy a curator reaches for is `--write-baseline`, which + re-freezes anything genuinely new in the same PR. That is how a ratchet rots + (#270). + + `detail` goes too, for the same reason at a different granularity: it + carries the full snippet for ELLIPTICAL/UNSUPPORTIVE and the DOI for + MISSING, so retyping a still-elliptical snippet or correcting the DOI on a + still-snippet-less reference flipped the key. audit_causal_graphs.py learned + this first and keys on only the leading fragment of its detail. + + Dropping both collapses several findings onto one key, which is why the + baseline carries a COUNT rather than a set of keys — see compare(). + """ + return (row["file"], _INDEX_RE.sub("[]", row["locator"]), row["defect"]) + + +def load_baseline(path: Path) -> dict[tuple[str, str, str], int]: + """Baselined occurrences per key. + + A count, not a set. Keying without the index means `evidence[0]` and + `evidence[1]` share a key, so set membership would let a THIRD missing + snippet at `evidence[2]` match a baselined key and pass silently — trading a + false positive for a false negative inside the integrity mechanism. The + count says "two of these were accepted"; a third is new. + """ + counts: dict[tuple[str, str, str], int] = {} if not path.exists(): - return set() + return counts with path.open() as fh: - return {_key(row) for row in csv.DictReader(fh, delimiter="\t")} + for row in csv.DictReader(fh, delimiter="\t"): + counts[_key(row)] = counts.get(_key(row), 0) + 1 + return counts + + +def compare(findings: list[dict[str, str]], + baseline: dict[tuple[str, str, str], int]) -> list[dict[str, str]]: + """Findings in excess of what the baseline accepted, per key. + + Fewer occurrences than baselined is an improvement and passes; more is new. + """ + seen: dict[tuple[str, str, str], int] = {} + new: list[dict[str, str]] = [] + for row in findings: + key = _key(row) + seen[key] = seen.get(key, 0) + 1 + if seen[key] > baseline.get(key, 0): + new.append(row) + return new def write_tsv(path: Path, rows: list[dict[str, str]]) -> None: @@ -316,7 +364,7 @@ def main(argv: list[str] | None = None) -> int: return 0 baseline = load_baseline(Path(args.baseline)) - new = [r for r in findings if _key(r) not in baseline] + new = compare(findings, baseline) if args.fail_on == "any": blocking = findings elif args.fail_on == "error": diff --git a/tests/test_audit_evidence_snippets.py b/tests/test_audit_evidence_snippets.py index 21ec9c6c8..492db811e 100644 --- a/tests/test_audit_evidence_snippets.py +++ b/tests/test_audit_evidence_snippets.py @@ -233,3 +233,59 @@ def test_the_audit_report_is_not_tracked(tmp_path): assert out == "", ( "reports/evidence_snippet_audit.tsv is tracked again; either untrack it " "or wire it into audit-derived-reports on a `git show HEAD:` basis") + + +# --- baseline identity (#270) ------------------------------------------- + +def _row(file, locator, defect, detail=""): + return {"file": file, "locator": locator, "defect": defect, + "severity": "WARN", "detail": detail} + + +def test_the_key_ignores_the_evidence_array_index(): + """evidence[1] renumbers to evidence[0] when item 0 is deleted — an + improvement that used to fail qc as a new finding (#270).""" + from audit_evidence_snippets import _key + assert _key(_row("f.yaml", "evidence[1]", "MISSING_SNIPPET")) == \ + _key(_row("f.yaml", "evidence[0]", "MISSING_SNIPPET")) + assert _key(_row("f.yaml", "g1:a->b[2]", "MISSING_SNIPPET")) == \ + _key(_row("f.yaml", "g1:a->b[0]", "MISSING_SNIPPET")) + + +def test_the_key_ignores_volatile_detail(): + """detail carries the full snippet and the DOI, so retyping a still-bad + snippet flipped the key. audit_causal_graphs learned this first.""" + from audit_evidence_snippets import _key + assert _key(_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET", "a ... b")) == \ + _key(_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET", "c ... d")) + + +def test_the_key_still_separates_files_locators_and_defects(): + from audit_evidence_snippets import _key + base = _row("f.yaml", "evidence[0]", "MISSING_SNIPPET") + assert _key(base) != _key(_row("g.yaml", "evidence[0]", "MISSING_SNIPPET")) + assert _key(base) != _key(_row("f.yaml", "g1:a->b[0]", "MISSING_SNIPPET")) + assert _key(base) != _key(_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET")) + + +def test_fewer_occurrences_than_baselined_is_an_improvement(): + from audit_evidence_snippets import compare + rows = [_row("f.yaml", "evidence[0]", "MISSING_SNIPPET")] + baseline = {("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2} + assert compare(rows, baseline) == [] + + +def test_one_more_occurrence_than_baselined_is_new(): + """The false negative a set-membership key would have introduced: a THIRD + missing snippet matching a baselined pair and passing silently (#270).""" + from audit_evidence_snippets import compare + rows = [_row("f.yaml", f"evidence[{i}]", "MISSING_SNIPPET") for i in range(3)] + baseline = {("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2} + new = compare(rows, baseline) + assert len(new) == 1 + + +def test_an_unbaselined_key_is_new(): + from audit_evidence_snippets import compare + rows = [_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET")] + assert len(compare(rows, {})) == 1 From 183533af49f04677df928f7da9ab88df66c6bfc7 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:03:53 -0700 Subject: [PATCH 2/4] Ratchet REUSED_SNIPPET's magnitude as a value, not as identity (#291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping detail from the key cost nothing on the index problem for per-item defects, and disarmed the ratchet on the one aggregate this audit emits. REUSED_SNIPPET's locator is `{graph_id}:*` with no index, and its magnitude lives entirely in detail — so a graph going from 3 shared snippets to 9 is one finding either way, same key, occurrence count unchanged, and it passed. All 13 baselined rows were exposed; gc_content is already at 5. The naive repair fails the other way: putting the count back in the key makes 3 -> 2, an improvement, an unbaselined finding that exits 1 — the exact rot #270 was about. So the magnitude is ratcheted as a VALUE. load_baseline records the worst accepted per key; compare() flags a finding whose magnitude exceeds it even when the count does not. Scoped to REUSED_SNIPPET deliberately. UNSUPPORTIVE_SNIPPET's detail also leads with an integer, but that one is a character count where larger is better, so ratcheting it would flag a snippet growing from 6 chars to 10 as a regression. MAGNITUDE_DEFECTS says which, and why. Canaried against the real baseline: 3 -> 9 flagged, 3 -> 2 and 3 -> 3 pass. 352 tests pass (4 new), `just qc` exits 0. Co-Authored-By: Claude Opus 5 --- scripts/audit_evidence_snippets.py | 42 ++++++++++++++++++++++- tests/test_audit_evidence_snippets.py | 49 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/scripts/audit_evidence_snippets.py b/scripts/audit_evidence_snippets.py index 1ad51083d..2ae0367e0 100644 --- a/scripts/audit_evidence_snippets.py +++ b/scripts/audit_evidence_snippets.py @@ -297,6 +297,36 @@ def _key(row: dict[str, str]) -> tuple[str, str, str]: return (row["file"], _INDEX_RE.sub("[]", row["locator"]), row["defect"]) +# Defects whose detail leads with a magnitude that can get WORSE without the +# finding count changing. REUSED_SNIPPET is the only aggregate this audit emits: +# one finding per (graph, snippet) whatever the number of items sharing it, with +# the number in the detail. Everything else is per-item, so more of it means +# more findings and the occurrence count already catches it. +# +# UNSUPPORTIVE_SNIPPET is deliberately NOT here even though its detail also leads +# with an integer: that integer is a character count, where larger is BETTER, so +# ratcheting it would flag a snippet growing from 6 chars to 10 as a regression. +MAGNITUDE_DEFECTS = {"REUSED_SNIPPET"} + +# Worst magnitude accepted per key, populated by load_baseline(). +BASELINE_MAGNITUDE: dict[tuple[str, str, str], int] = {} + + +def _magnitude(row: dict[str, str]) -> int: + """The leading integer of `detail`, for defects where more is worse. + + Ratcheted as a VALUE, not folded into the key. Putting it in the key would + reintroduce the rot #270 fixed from the other side: 3 -> 2 is an + improvement, and a key carrying the count would make it an unbaselined + finding and fail. audit_causal_graphs.py keys on the leading token for the + same information and accepts that cost; a value comparison does not have to. + """ + if row.get("defect") not in MAGNITUDE_DEFECTS: + return 0 + match = re.match(r"\s*(\d+)", row.get("detail", "")) + return int(match.group(1)) if match else 0 + + def load_baseline(path: Path) -> dict[tuple[str, str, str], int]: """Baselined occurrences per key. @@ -307,11 +337,16 @@ def load_baseline(path: Path) -> dict[tuple[str, str, str], int]: count says "two of these were accepted"; a third is new. """ counts: dict[tuple[str, str, str], int] = {} + BASELINE_MAGNITUDE.clear() if not path.exists(): return counts with path.open() as fh: for row in csv.DictReader(fh, delimiter="\t"): - counts[_key(row)] = counts.get(_key(row), 0) + 1 + key = _key(row) + counts[key] = counts.get(key, 0) + 1 + magnitude = _magnitude(row) + if magnitude > BASELINE_MAGNITUDE.get(key, 0): + BASELINE_MAGNITUDE[key] = magnitude return counts @@ -328,6 +363,11 @@ def compare(findings: list[dict[str, str]], seen[key] = seen.get(key, 0) + 1 if seen[key] > baseline.get(key, 0): new.append(row) + elif _magnitude(row) > BASELINE_MAGNITUDE.get(key, 0): + # Same key, same occurrence count, but worse — a graph going from 3 + # shared snippets to 9 is one REUSED_SNIPPET finding either way + # (#291). + new.append(row) return new diff --git a/tests/test_audit_evidence_snippets.py b/tests/test_audit_evidence_snippets.py index 492db811e..31b7c5014 100644 --- a/tests/test_audit_evidence_snippets.py +++ b/tests/test_audit_evidence_snippets.py @@ -289,3 +289,52 @@ def test_an_unbaselined_key_is_new(): from audit_evidence_snippets import compare rows = [_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET")] assert len(compare(rows, {})) == 1 + + +# --- magnitude ratchet for aggregate defects (#291) ---------------------- + +def _reused(n, graph="g1:*", file="f.yaml"): + return _row(file, graph, "REUSED_SNIPPET", + f"{n} evidence items share one snippet: 'virulence factors'") + + +def test_a_worse_reused_count_is_new_despite_the_same_key(): + """REUSED_SNIPPET has no index and carries its magnitude in detail, so + dropping detail from the key let 3 -> 9 pass as one unchanged finding.""" + import audit_evidence_snippets as aes + from audit_evidence_snippets import _key, compare + baseline = {_key(_reused(3)): 1} + aes.BASELINE_MAGNITUDE.clear() + aes.BASELINE_MAGNITUDE[_key(_reused(3))] = 3 + assert len(compare([_reused(9)], baseline)) == 1 + + +def test_a_better_reused_count_still_passes(): + """The rot #270 fixed, from the other side: a count in the KEY would make + 3 -> 2 an unbaselined finding and fail on an improvement.""" + import audit_evidence_snippets as aes + from audit_evidence_snippets import _key, compare + baseline = {_key(_reused(3)): 1} + aes.BASELINE_MAGNITUDE.clear() + aes.BASELINE_MAGNITUDE[_key(_reused(3))] = 3 + assert compare([_reused(2)], baseline) == [] + assert compare([_reused(3)], baseline) == [] + + +def test_a_character_count_is_not_ratcheted(): + """UNSUPPORTIVE_SNIPPET's leading integer is a length, where larger is + BETTER — ratcheting it would flag 6 chars growing to 10 as a regression.""" + from audit_evidence_snippets import _magnitude + assert _magnitude(_row("f.yaml", "evidence[0]", "UNSUPPORTIVE_SNIPPET", + "6 chars, supports nothing specific: 'toxins'")) == 0 + assert _magnitude(_reused(7)) == 7 + + +def test_the_real_baseline_records_reused_magnitudes(): + """Guards the wiring: a baseline read without magnitudes silently disarms.""" + import audit_evidence_snippets as aes + from audit_evidence_snippets import DEFAULT_BASELINE, load_baseline + load_baseline(DEFAULT_BASELINE) + magnitudes = [v for v in aes.BASELINE_MAGNITUDE.values() if v] + assert magnitudes, "no REUSED_SNIPPET magnitudes captured from the baseline" + assert max(magnitudes) >= 3 From 21f8a763d01f4e2759d9e2522dbcbba0516b127f Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:12:53 -0700 Subject: [PATCH 3/4] Make the baseline a value, and per-snippet rather than per-graph (#291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review items, one of them a hole the previous commit opened. The baseline-currency test — the only one comparing the committed baseline to the live corpus — still read `_key(r) not in baseline`. Against the new dict that tests key PRESENCE, so it silently stopped checking both the occurrence count and the magnitude: a third snippet-less reference, or a graph growing from 5 shared snippets to 50, would fail `just qc` while the test passed. Now uses compare(), so it asserts what qc enforces. Magnitudes were keyed per _key(), and REUSED_SNIPPET's locator is `{graph_id}:*` — so every reused snippet in one graph shared a key and a per-key max let the smaller of an uneven pair grow up to the larger unnoticed. trophic_type_classification_axes already carries two such rows, equal today. The snippet itself is now the discriminator. compare() took the baseline as an argument while reading magnitudes from a module global, which the tests exposed by having to clear and poke it by hand. load_baseline returns a Baseline NamedTuple carrying both maps, so compare() is a pure function of its arguments and the tests construct a baseline instead. The module docstring still claimed "same shape as audit_causal_graphs.py". It is not, and the divergence is the point: that shape is set membership on a 4-tuple carrying the index and a detail fragment, which is exactly what rotted here. Now says what this does and why. compare() also documents that for a count excess it names the LAST row at a key rather than the new one — the occurrences are interchangeable by construction, so no better answer exists. 353 tests pass (1 new), `just qc` exits 0, and the real-baseline canary still flags 3 -> 9 while passing 3 -> 2. Co-Authored-By: Claude Opus 5 --- scripts/audit_evidence_snippets.py | 75 ++++++++++++++++++++------- tests/test_audit_evidence_snippets.py | 60 +++++++++++++-------- 2 files changed, 95 insertions(+), 40 deletions(-) diff --git a/scripts/audit_evidence_snippets.py b/scripts/audit_evidence_snippets.py index 2ae0367e0..ba50e38da 100644 --- a/scripts/audit_evidence_snippets.py +++ b/scripts/audit_evidence_snippets.py @@ -32,9 +32,15 @@ this audit exists to catch, so it asks for a check against the source rather than asserting a defect. -Baseline ratchet, same shape as audit_causal_graphs.py: pre-existing findings -are frozen in ``conf/evidence_snippet_baseline.tsv`` and never fail; anything -new exits 1. +Baseline ratchet: pre-existing findings are frozen in +``conf/evidence_snippet_baseline.tsv`` and never fail; anything new exits 1. + +NOT the same shape as audit_causal_graphs.py, which is set membership on a +4-tuple carrying the locator index and a detail fragment. That shape rotted here +(#270): the index renumbers when an evidence item is deleted, so an improvement +read as a new finding. This keys without the index and compares OCCURRENCE +COUNTS, plus a magnitude ratchet for REUSED_SNIPPET, whose locator has no index +and whose severity lives in its detail (#291). Usage: just audit-snippets @@ -48,6 +54,7 @@ import re import sys from pathlib import Path +from typing import NamedTuple import yaml @@ -308,8 +315,19 @@ def _key(row: dict[str, str]) -> tuple[str, str, str]: # ratcheting it would flag a snippet growing from 6 chars to 10 as a regression. MAGNITUDE_DEFECTS = {"REUSED_SNIPPET"} -# Worst magnitude accepted per key, populated by load_baseline(). -BASELINE_MAGNITUDE: dict[tuple[str, str, str], int] = {} +def _magnitude_key(row: dict[str, str]) -> tuple[str, str, str, str] | None: + """Identity for a magnitude, or None where magnitude is not ratcheted. + + REUSED_SNIPPET's locator is `{graph_id}:*`, so every reused snippet in one + graph shares a _key(). Keying the magnitude on the graph alone would let the + smaller of an uneven pair grow up to the larger unnoticed — + `trophic_type_classification_axes:*` already carries two rows. The snippet + itself is the discriminator (#291). + """ + if row.get("defect") not in MAGNITUDE_DEFECTS: + return None + match = re.search(r"share one snippet: (.+)$", row.get("detail", "")) + return (*_key(row), match.group(1) if match else "") def _magnitude(row: dict[str, str]) -> int: @@ -327,7 +345,19 @@ def _magnitude(row: dict[str, str]) -> int: return int(match.group(1)) if match else 0 -def load_baseline(path: Path) -> dict[tuple[str, str, str], int]: +class Baseline(NamedTuple): + """What the frozen backlog accepted: how many, and how bad. + + Both maps travel together rather than one being a module global, so + compare() is a pure function of its arguments and the tests exercise the + magnitude branch by constructing a Baseline instead of poking a global. + """ + + counts: dict[tuple[str, str, str], int] + magnitudes: dict[tuple[str, str, str, str], int] + + +def load_baseline(path: Path) -> Baseline: """Baselined occurrences per key. A count, not a set. Keying without the index means `evidence[0]` and @@ -337,33 +367,42 @@ def load_baseline(path: Path) -> dict[tuple[str, str, str], int]: count says "two of these were accepted"; a third is new. """ counts: dict[tuple[str, str, str], int] = {} - BASELINE_MAGNITUDE.clear() + magnitudes: dict[tuple[str, str, str, str], int] = {} if not path.exists(): - return counts + return Baseline(counts, magnitudes) with path.open() as fh: for row in csv.DictReader(fh, delimiter="\t"): key = _key(row) counts[key] = counts.get(key, 0) + 1 - magnitude = _magnitude(row) - if magnitude > BASELINE_MAGNITUDE.get(key, 0): - BASELINE_MAGNITUDE[key] = magnitude - return counts + mkey = _magnitude_key(row) + if mkey is not None: + magnitudes[mkey] = max(magnitudes.get(mkey, 0), _magnitude(row)) + return Baseline(counts, magnitudes) def compare(findings: list[dict[str, str]], - baseline: dict[tuple[str, str, str], int]) -> list[dict[str, str]]: - """Findings in excess of what the baseline accepted, per key. - - Fewer occurrences than baselined is an improvement and passes; more is new. + baseline: Baseline) -> list[dict[str, str]]: + """Findings in excess of what the baseline accepted. + + Two ways to exceed it: more occurrences at a key than were frozen, or an + aggregate finding whose magnitude is worse than the frozen one. Fewer + occurrences, or a smaller magnitude, is an improvement and passes. + + Note which ROW is reported for a count excess: findings are walked in audit + order and the ones PAST the baselined count are marked, so the row named is + the last at that key rather than the one that is new. Adding a snippet-less + reference at evidence[0] points a curator at evidence[2]. The exit code is + right and a count-based key cannot do better — the occurrences are + interchangeable by construction. """ seen: dict[tuple[str, str, str], int] = {} new: list[dict[str, str]] = [] for row in findings: key = _key(row) seen[key] = seen.get(key, 0) + 1 - if seen[key] > baseline.get(key, 0): + if seen[key] > baseline.counts.get(key, 0): new.append(row) - elif _magnitude(row) > BASELINE_MAGNITUDE.get(key, 0): + elif _magnitude(row) > baseline.magnitudes.get(_magnitude_key(row), 0): # Same key, same occurrence count, but worse — a graph going from 3 # shared snippets to 9 is one REUSED_SNIPPET finding either way # (#291). diff --git a/tests/test_audit_evidence_snippets.py b/tests/test_audit_evidence_snippets.py index 31b7c5014..76b2a9cfb 100644 --- a/tests/test_audit_evidence_snippets.py +++ b/tests/test_audit_evidence_snippets.py @@ -203,10 +203,16 @@ def test_a_short_snippet_does_not_match_by_coincidence(tmp_path, monkeypatch): def test_the_committed_baseline_matches_the_corpus(): """The ratchet is only a ratchet if the frozen set is current.""" - from audit_evidence_snippets import DEFAULT_BASELINE, _key, load_baseline + from audit_evidence_snippets import DEFAULT_BASELINE, compare, load_baseline baseline = load_baseline(DEFAULT_BASELINE) - assert baseline, "baseline is empty — run `just audit-snippets --write-baseline`" - new = [r for r in audit() if _key(r) not in baseline] + assert baseline.counts, "baseline is empty — run `just audit-snippets --write-baseline`" + # compare(), not key membership: `_key(r) not in baseline.counts` would + # check presence only, ignoring both the occurrence count and the + # REUSED_SNIPPET magnitude, so a third snippet-less reference or a graph + # growing from 5 shared snippets to 50 would fail `just qc` while this test + # passed. This is the only test comparing the committed baseline to the live + # corpus, so it has to assert what qc enforces (#291). + new = compare(audit(), baseline) assert new == [], f"{len(new)} findings are not baselined, e.g. {new[:2]}" @@ -269,26 +275,26 @@ def test_the_key_still_separates_files_locators_and_defects(): def test_fewer_occurrences_than_baselined_is_an_improvement(): - from audit_evidence_snippets import compare + from audit_evidence_snippets import Baseline, compare rows = [_row("f.yaml", "evidence[0]", "MISSING_SNIPPET")] - baseline = {("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2} + baseline = Baseline({("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2}, {}) assert compare(rows, baseline) == [] def test_one_more_occurrence_than_baselined_is_new(): """The false negative a set-membership key would have introduced: a THIRD missing snippet matching a baselined pair and passing silently (#270).""" - from audit_evidence_snippets import compare + from audit_evidence_snippets import Baseline, compare rows = [_row("f.yaml", f"evidence[{i}]", "MISSING_SNIPPET") for i in range(3)] - baseline = {("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2} + baseline = Baseline({("f.yaml", "evidence[]", "MISSING_SNIPPET"): 2}, {}) new = compare(rows, baseline) assert len(new) == 1 def test_an_unbaselined_key_is_new(): - from audit_evidence_snippets import compare + from audit_evidence_snippets import Baseline, compare rows = [_row("f.yaml", "evidence[0]", "ELLIPTICAL_SNIPPET")] - assert len(compare(rows, {})) == 1 + assert len(compare(rows, Baseline({}, {}))) == 1 # --- magnitude ratchet for aggregate defects (#291) ---------------------- @@ -301,22 +307,16 @@ def _reused(n, graph="g1:*", file="f.yaml"): def test_a_worse_reused_count_is_new_despite_the_same_key(): """REUSED_SNIPPET has no index and carries its magnitude in detail, so dropping detail from the key let 3 -> 9 pass as one unchanged finding.""" - import audit_evidence_snippets as aes - from audit_evidence_snippets import _key, compare - baseline = {_key(_reused(3)): 1} - aes.BASELINE_MAGNITUDE.clear() - aes.BASELINE_MAGNITUDE[_key(_reused(3))] = 3 + from audit_evidence_snippets import Baseline, _key, _magnitude_key, compare + baseline = Baseline({_key(_reused(3)): 1}, {_magnitude_key(_reused(3)): 3}) assert len(compare([_reused(9)], baseline)) == 1 def test_a_better_reused_count_still_passes(): """The rot #270 fixed, from the other side: a count in the KEY would make 3 -> 2 an unbaselined finding and fail on an improvement.""" - import audit_evidence_snippets as aes - from audit_evidence_snippets import _key, compare - baseline = {_key(_reused(3)): 1} - aes.BASELINE_MAGNITUDE.clear() - aes.BASELINE_MAGNITUDE[_key(_reused(3))] = 3 + from audit_evidence_snippets import Baseline, _key, _magnitude_key, compare + baseline = Baseline({_key(_reused(3)): 1}, {_magnitude_key(_reused(3)): 3}) assert compare([_reused(2)], baseline) == [] assert compare([_reused(3)], baseline) == [] @@ -332,9 +332,25 @@ def test_a_character_count_is_not_ratcheted(): def test_the_real_baseline_records_reused_magnitudes(): """Guards the wiring: a baseline read without magnitudes silently disarms.""" - import audit_evidence_snippets as aes from audit_evidence_snippets import DEFAULT_BASELINE, load_baseline - load_baseline(DEFAULT_BASELINE) - magnitudes = [v for v in aes.BASELINE_MAGNITUDE.values() if v] + magnitudes = [v for v in load_baseline(DEFAULT_BASELINE).magnitudes.values() if v] assert magnitudes, "no REUSED_SNIPPET magnitudes captured from the baseline" assert max(magnitudes) >= 3 + + +def test_two_reused_snippets_in_one_graph_get_separate_magnitudes(): + """`{graph_id}:*` means every reused snippet in a graph shares a _key(), so + a per-key max would let the smaller of an uneven pair grow to the larger + unnoticed — trophic_type_classification_axes already carries two (#291).""" + from audit_evidence_snippets import Baseline, _key, _magnitude_key, compare + a = _row("f.yaml", "g1:*", "REUSED_SNIPPET", + "3 evidence items share one snippet: 'carbon source'") + b = _row("f.yaml", "g1:*", "REUSED_SNIPPET", + "8 evidence items share one snippet: 'energy source'") + assert _key(a) == _key(b) + assert _magnitude_key(a) != _magnitude_key(b) + baseline = Baseline({_key(a): 2}, + {_magnitude_key(a): 3, _magnitude_key(b): 8}) + worse_a = dict(a, detail="7 evidence items share one snippet: 'carbon source'") + assert len(compare([worse_a, b], baseline)) == 1, \ + "the smaller snippet grew to below the larger's magnitude and passed" From 0a7ab21d2408408af6dae1462f7897c8be760d7c Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:21:38 -0700 Subject: [PATCH 4/4] Document the magnitude key's cost (#292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editing a REUSED_SNIPPET's shared quote produces an unseen magnitude key, so its baselined value reads as 0 and any count beats it — a rewording reported as new when nothing got worse. The obvious fallback to the graph's per-key max would restore the sheltering #291 removed, so this fails closed and is tracked rather than patched. Co-Authored-By: Claude Opus 5 --- scripts/audit_evidence_snippets.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/audit_evidence_snippets.py b/scripts/audit_evidence_snippets.py index ba50e38da..2a10c7eac 100644 --- a/scripts/audit_evidence_snippets.py +++ b/scripts/audit_evidence_snippets.py @@ -323,6 +323,12 @@ def _magnitude_key(row: dict[str, str]) -> tuple[str, str, str, str] | None: smaller of an uneven pair grow up to the larger unnoticed — `trophic_type_classification_axes:*` already carries two rows. The snippet itself is the discriminator (#291). + + Cost of that choice, tracked in #292: editing the shared snippet's TEXT + produces an unseen key, so its baselined magnitude reads as 0 and any count + beats it — a reworded quote is reported as new when nothing got worse. + Falling back to the graph's per-key max would fix that and restore the + sheltering this key exists to remove, so it fails closed instead. """ if row.get("defect") not in MAGNITUDE_DEFECTS: return None