diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22e225c..5d9c721 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,3 +68,15 @@ jobs: pip install -e "./runtime[dev]" - name: interscript-ml runtime tests (tiny-graph zips, golden e2e) run: python -m pytest runtime/tests -v + + metrics-provenance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: "3.11" } + - run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: WO10 — metadata metrics must match RESULTS.md sources + run: PYTHONPATH=src python -m imf metrics diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml new file mode 100644 index 0000000..1b2260d --- /dev/null +++ b/models/metrics-sources.yaml @@ -0,0 +1,42 @@ +# WO10: where each model's metadata metrics come from. Every metrics +# block is GENERATED from these RESULTS.md tables and CI fails on drift. +# Refs are pinned; bump them deliberately when RESULTS.md is re-published. +khm-latn-1.0: + repo: secryst/secryst + ref: 23261bbf0d03e562d4d0fad107250349c4777368 + path: docs/RESULTS.md + anchor: khmer-transliteration-2026-08-14 + protocol: "greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42; ByT5-small early stop @ep15" + tables: + - {row: "ByT5-small, early stop @ep15", column: CER, as: cer} + - {row: "ByT5-small, early stop @ep15", column: EM, as: em} +urd-g2p-1.0: + repo: interscript/rababa-urdu + ref: 5225b17df356afb4ad23b4721a3b5af03e9f71ab + path: docs/RESULTS.md + anchor: g2p-urdu-text-ipa + display_anchor: g2p-urdu-text--ipa + protocol: "greedy decode; 12,699 held-out words from the 635K humair025 urdu-g2p dictionary; ByT5-small" + tables: + - {row: "CER (char-level)", as: cer} + - {row: "Exact match", as: em} +urd-diac-1.0: + repo: interscript/rababa-urdu + ref: 5225b17df356afb4ad23b4721a3b5af03e9f71ab + path: docs/RESULTS.md + anchor: diacritization-urdu-text-text-haraqat + display_anchor: diacritization-urdu-text--text--haraqat + protocol: "greedy decode; 11,940 held-out; labels derived IPA->haraqat (deterministic conversion, 597K pairs); ByT5-small, 2 epochs" + tables: + - {row: "CER", as: cer} +heb-diac-1.0: + repo: interscript/rababa + ref: 82d508bd496572ddbc94a8fd7bd1aacdf7875a03 + path: docs/RESULTS.md + anchor: hebrew-diacritization + protocol: "beam=1 greedy decode (the v1 runtime path); Nakdimon test split, 5,095 examples; ByT5-base s43" + tables: + - {row: "beam=1 (s43/v4)", as: der_greedy} + - row: "s43 (production)" + as: der_beam4 + protocol: "beam=4 standard decode (reference quality; beam search is not in v1 runtimes); Nakdimon test split, 5,095 examples; ByT5-base s43" diff --git a/src/imf/cli.py b/src/imf/cli.py index 85ae220..55520ae 100644 --- a/src/imf/cli.py +++ b/src/imf/cli.py @@ -69,6 +69,32 @@ def _cmd_pack(args: argparse.Namespace) -> int: return 0 +def _cmd_metrics(args: argparse.Namespace) -> int: + import sys + + import yaml + + from imf.metrics import check_against_metadata + + mapping = Path(args.mapping) + entries = yaml.safe_load(mapping.read_text(encoding="utf-8")) + entries = entries.get("models", entries) + problems: list[str] = [] + for model_id, spec in entries.items(): + metadata = Path(spec.get("metadata")) if spec.get("metadata") else ( + Path("models") / model_id.rsplit("-", 1)[0] / f"{model_id}.metadata.yaml" + ) + if not metadata.is_file(): + problems.append(f"{model_id}: metadata source not found at {metadata}") + continue + problems += check_against_metadata(model_id, metadata, mapping) + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + label = "all models trace to RESULTS.md" if not problems else "MISMATCH" + print(f"metrics provenance: {label} ({len(entries)} models)") + return 0 if not problems else 1 + + def _default_readme(metadata: ModelMetadata) -> str: return ( f"# {metadata.id}\n\n" @@ -198,6 +224,12 @@ def build_parser() -> argparse.ArgumentParser: p_golden.add_argument("--max-len", type=int, default=256) p_golden.set_defaults(func=_cmd_golden) + p_metrics = sub.add_parser( + "metrics", help="WO10: check every metadata metrics block against its RESULTS.md source" + ) + p_metrics.add_argument("--mapping", type=Path, default=Path("models/metrics-sources.yaml")) + p_metrics.set_defaults(func=_cmd_metrics) + return parser diff --git a/src/imf/metrics.py b/src/imf/metrics.py new file mode 100644 index 0000000..e60ee70 --- /dev/null +++ b/src/imf/metrics.py @@ -0,0 +1,213 @@ +"""WO10: RESULTS.md -> metadata metrics generator. + +Every IMF zip's metrics block is generated from a RESULTS.md table — +never hand-written — and CI refuses to release a model whose metadata +disagrees with the documented protocol numbers. + +Sources are pinned in models/metrics-sources.yaml (repo, ref, path, +anchor). Extraction is by table position (row label + optional column +header), not regexes over prose: the mapping says where a number lives, +the parser reads exactly that cell. +""" + +from __future__ import annotations + +import re +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +class MetricsError(ValueError): + """The RESULTS.md source cannot yield the mapped metrics.""" + + +@dataclass(frozen=True) +class TableSpec: + row: str + column: str | None = None # None: first value cell in the row + as_name: str = "" + protocol: str | None = None # override the source-level protocol + + +@dataclass(frozen=True) +class SourceSpec: + repo: str + ref: str + path: str + anchor: str + protocol: str + tables: tuple[TableSpec, ...] + display_anchor: str = "" + + +def _slugify(heading: str) -> str: + text = heading.strip().lower() + text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE) + return re.sub(r"\s+", "-", text).strip("-") + + +def _cell_to_value(cell: str) -> float: + text = cell.replace("**", "").replace("%", "").replace(",", "").strip() + match = re.search(r"-?\d+(?:\.\d+)?", text) + if not match: + raise MetricsError(f"no numeric value in cell {cell!r}") + return float(match.group()) + + +def parse_tables(markdown: str, anchor: str) -> list[dict[str, Any]]: + """All tables in the section whose heading slugifies to `anchor`.""" + lines = markdown.splitlines() + start = None + for index, line in enumerate(lines): + if line.startswith("## "): + if start is not None: + end = index + break + if _slugify(line[3:]) == anchor: + start = index + else: + end = len(lines) if start is not None else None + if start is None: + raise MetricsError(f"section anchor {anchor!r} not found") + + tables: list[dict[str, Any]] = [] + index = start + while index < end: + line = lines[index] + if line.startswith("|") and index + 1 < end and set(lines[index + 1]) <= set("|-: "): + header = [cell.strip() for cell in line.strip("|").split("|")] + index += 2 + rows: list[dict[str, Any]] = [] + while index < end and lines[index].startswith("|"): + cells = [cell.strip() for cell in lines[index].strip("|").split("|")] + rows.append({"label": cells[0], "cells": cells, "header": header}) + index += 1 + tables.append({"header": header, "rows": rows}) + else: + index += 1 + if not tables: + raise MetricsError(f"no tables under anchor {anchor!r}") + return tables + + +def extract( + markdown: str, anchor: str, specs: tuple[TableSpec, ...] +) -> list[dict[str, Any]]: + tables = parse_tables(markdown, anchor) + out: list[dict[str, Any]] = [] + for spec in specs: + found = None + for table in tables: + for row in table["rows"]: + if spec.row.lower() in row["label"].lower(): + found = row + break + if found: + break + if found is None: + raise MetricsError(f"row {spec.row!r} not found under {anchor!r}") + if spec.column is None: + values = [ + _cell_to_value(cell) + for cell in found["cells"][1:] + if "%" in cell or re.search(r"\d", cell.replace("**", "")) + ] + if not values: + raise MetricsError(f"no value cells in row {spec.row!r}") + value = values[0] + else: + try: + column_index = found["header"].index(spec.column) + except ValueError as e: + raise MetricsError( + f"column {spec.column!r} not in table header {found['header']}" + ) from e + value = _cell_to_value(found["cells"][column_index]) + out.append({"name": spec.as_name or spec.row, "value": value}) + return out + + +def load_source(source: SourceSpec, cache_dir: Path | None = None) -> str: + url = ( + f"https://raw.githubusercontent.com/{source.repo}/{source.ref}/{source.path}" + ) + if cache_dir is not None: + cached = cache_dir / _cache_name(source) + if cached.is_file(): + return cached.read_text(encoding="utf-8") + with urllib.request.urlopen(url) as response: + text = response.read().decode("utf-8") + if cache_dir is not None: + cache_dir.mkdir(parents=True, exist_ok=True) + (cache_dir / _cache_name(source)).write_text(text, encoding="utf-8") + return text + + +def _cache_name(source: SourceSpec) -> str: + return f"{source.repo.replace('/', '_')}@{source.ref}_{source.path.replace('/', '_')}" + + + + +def generate_metrics( + model_id: str, + mapping_path: Path | str, + cache_dir: Path | None = None, +) -> list[dict[str, Any]]: + raw = yaml.safe_load(Path(mapping_path).read_text(encoding="utf-8")) + entry = raw.get("models", raw).get(model_id) + if entry is None: + raise MetricsError(f"no metrics source mapped for {model_id!r}") + source = SourceSpec( + repo=entry["repo"], + ref=entry["ref"], + path=entry["path"], + anchor=entry["anchor"], + protocol=entry["protocol"], + display_anchor=entry.get("display_anchor", ""), + tables=tuple( + TableSpec( + row=t["row"], + column=t.get("column"), + as_name=t.get("as", ""), + protocol=t.get("protocol"), + ) + for t in entry["tables"] + ), + ) + markdown = load_source(source, cache_dir) + extracted = extract(markdown, source.anchor, source.tables) + source_ref = f"{source.path}#{source.display_anchor or source.anchor}" + return [ + { + "name": m["name"], + "value": m["value"], + "protocol": spec.protocol or source.protocol, + "source": source_ref, + } + for m, spec in zip(extracted, source.tables, strict=True) + ] + + +def check_against_metadata( + model_id: str, + metadata_path: Path | str, + mapping_path: Path | str, + cache_dir: Path | None = None, +) -> list[str]: + """Diff generated metrics vs a metadata source file. Returns problems.""" + generated = generate_metrics(model_id, mapping_path, cache_dir) + meta = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8")) + recorded = meta.get("metrics", []) + problems: list[str] = [] + if [(m["name"], m["value"]) for m in generated] != [(m["name"], m["value"]) for m in recorded]: + problems.append( + f"{model_id}: metrics mismatch — generated " + f"{[(m['name'], m['value']) for m in generated]} vs metadata " + f"{[(m['name'], m['value']) for m in recorded]}" + ) + return problems diff --git a/tests/test_imf_metrics.py b/tests/test_imf_metrics.py new file mode 100644 index 0000000..be0cc77 --- /dev/null +++ b/tests/test_imf_metrics.py @@ -0,0 +1,74 @@ +"""Tests for the WO10 metrics generator (inline markdown; no network — +the network provenance check runs in CI as `imf metrics`).""" + +from __future__ import annotations + +import pytest + +from imf.metrics import MetricsError, TableSpec, _slugify, extract, parse_tables + +SAMPLE = """# Results + +## G2P (Urdu text → IPA) + +### Best result + +| Metric | Value | Test set | +|---|---|---| +| PER (word-level) | 72.0%* | 12,699 held-out | +| **CER (char-level)** | **14.77%** | 12,699 held-out | +| Exact match | 33.6% | 12,699 held-out | + +## Khmer transliteration (2026-08-14) + +| System | EM | CER | n | +|---|---|---|---| +| ByT5-small, early stop @ep15 | **59.66%** | **27.42%** | 895 | + +## Key findings + +prose without tables +""" + + +def test_slugify_matches_section_headings() -> None: + assert _slugify("G2P (Urdu text → IPA)") == "g2p-urdu-text-ipa" + assert _slugify("Khmer transliteration (2026-08-14)") == "khmer-transliteration-2026-08-14" + + +def test_parse_tables_scopes_to_anchor_section() -> None: + tables = parse_tables(SAMPLE, "g2p-urdu-text-ipa") + assert len(tables) == 1 + labels = [row["label"] for row in tables[0]["rows"]] + assert "PER (word-level)" in labels + + +def test_extract_row_mode_takes_first_value_cell() -> None: + metrics = extract( + SAMPLE, + "g2p-urdu-text-ipa", + ( + TableSpec(row="CER (char-level)", as_name="cer"), + TableSpec(row="Exact match", as_name="em"), + ), + ) + assert metrics == [{"name": "cer", "value": 14.77}, {"name": "em", "value": 33.6}] + + +def test_extract_column_mode() -> None: + metrics = extract( + SAMPLE, + "khmer-transliteration-2026-08-14", + (TableSpec(row="ByT5-small, early stop", column="CER", as_name="cer"),), + ) + assert metrics == [{"name": "cer", "value": 27.42}] + + +def test_missing_anchor_raises() -> None: + with pytest.raises(MetricsError, match="anchor"): + parse_tables(SAMPLE, "no-such-section") + + +def test_missing_row_raises() -> None: + with pytest.raises(MetricsError, match="row"): + extract(SAMPLE, "g2p-urdu-text-ipa", (TableSpec(row="Nonexistent row"),))