feat(imf): WO10 — RESULTS.md to metadata metrics generator + CI provenance gate

.github/workflows/test.yml

@@ -68,3 +68,15 @@ jobs:
           pip install -e "./runtime[dev]"
       - name: interscript-ml runtime tests (tiny-graph zips, golden e2e)
         run: python -m pytest runtime/tests -v
+
+  metrics-provenance:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: WO10 — metadata metrics must match RESULTS.md sources
+        run: PYTHONPATH=src python -m imf metrics

models/metrics-sources.yaml

@@ -0,0 +1,42 @@
+# WO10: where each model's metadata metrics come from. Every metrics
+# block is GENERATED from these RESULTS.md tables and CI fails on drift.
+# Refs are pinned; bump them deliberately when RESULTS.md is re-published.
+khm-latn-1.0:
+  repo: secryst/secryst
+  ref: 23261bbf0d03e562d4d0fad107250349c4777368
+  path: docs/RESULTS.md
+  anchor: khmer-transliteration-2026-08-14
+  protocol: "greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42; ByT5-small early stop @ep15"
+  tables:
+    - {row: "ByT5-small, early stop @ep15", column: CER, as: cer}
+    - {row: "ByT5-small, early stop @ep15", column: EM, as: em}
+urd-g2p-1.0:
+  repo: interscript/rababa-urdu
+  ref: 5225b17df356afb4ad23b4721a3b5af03e9f71ab
+  path: docs/RESULTS.md
+  anchor: g2p-urdu-text-ipa
+  display_anchor: g2p-urdu-text--ipa
+  protocol: "greedy decode; 12,699 held-out words from the 635K humair025 urdu-g2p dictionary; ByT5-small"
+  tables:
+    - {row: "CER (char-level)", as: cer}
+    - {row: "Exact match", as: em}
+urd-diac-1.0:
+  repo: interscript/rababa-urdu
+  ref: 5225b17df356afb4ad23b4721a3b5af03e9f71ab
+  path: docs/RESULTS.md
+  anchor: diacritization-urdu-text-text-haraqat
+  display_anchor: diacritization-urdu-text--text--haraqat
+  protocol: "greedy decode; 11,940 held-out; labels derived IPA->haraqat (deterministic conversion, 597K pairs); ByT5-small, 2 epochs"
+  tables:
+    - {row: "CER", as: cer}
+heb-diac-1.0:
+  repo: interscript/rababa
+  ref: 82d508bd496572ddbc94a8fd7bd1aacdf7875a03
+  path: docs/RESULTS.md
+  anchor: hebrew-diacritization
+  protocol: "beam=1 greedy decode (the v1 runtime path); Nakdimon test split, 5,095 examples; ByT5-base s43"
+  tables:
+    - {row: "beam=1 (s43/v4)", as: der_greedy}
+    - row: "s43 (production)"
+      as: der_beam4
+      protocol: "beam=4 standard decode (reference quality; beam search is not in v1 runtimes); Nakdimon test split, 5,095 examples; ByT5-base s43"

src/imf/cli.py

@@ -69,6 +69,32 @@ def _cmd_pack(args: argparse.Namespace) -> int:
     return 0
 
 
+def _cmd_metrics(args: argparse.Namespace) -> int:
+    import sys
+
+    import yaml
+
+    from imf.metrics import check_against_metadata
+
+    mapping = Path(args.mapping)
+    entries = yaml.safe_load(mapping.read_text(encoding="utf-8"))
+    entries = entries.get("models", entries)
+    problems: list[str] = []
+    for model_id, spec in entries.items():
+        metadata = Path(spec.get("metadata")) if spec.get("metadata") else (
+            Path("models") / model_id.rsplit("-", 1)[0] / f"{model_id}.metadata.yaml"
+        )
+        if not metadata.is_file():
+            problems.append(f"{model_id}: metadata source not found at {metadata}")
+            continue
+        problems += check_against_metadata(model_id, metadata, mapping)
+    for problem in problems:
+        print(f"error: {problem}", file=sys.stderr)
+    label = "all models trace to RESULTS.md" if not problems else "MISMATCH"
+    print(f"metrics provenance: {label} ({len(entries)} models)")
+    return 0 if not problems else 1
+
+
 def _default_readme(metadata: ModelMetadata) -> str:
     return (
         f"# {metadata.id}\n\n"
@@ -198,6 +224,12 @@ def build_parser() -> argparse.ArgumentParser:
     p_golden.add_argument("--max-len", type=int, default=256)
     p_golden.set_defaults(func=_cmd_golden)
 
+    p_metrics = sub.add_parser(
+        "metrics", help="WO10: check every metadata metrics block against its RESULTS.md source"
+    )
+    p_metrics.add_argument("--mapping", type=Path, default=Path("models/metrics-sources.yaml"))
+    p_metrics.set_defaults(func=_cmd_metrics)
+
     return parser
 
 

src/imf/metrics.py

@@ -0,0 +1,213 @@
+"""WO10: RESULTS.md -> metadata metrics generator.
+
+Every IMF zip's metrics block is generated from a RESULTS.md table —
+never hand-written — and CI refuses to release a model whose metadata
+disagrees with the documented protocol numbers.
+
+Sources are pinned in models/metrics-sources.yaml (repo, ref, path,
+anchor). Extraction is by table position (row label + optional column
+header), not regexes over prose: the mapping says where a number lives,
+the parser reads exactly that cell.
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+class MetricsError(ValueError):
+    """The RESULTS.md source cannot yield the mapped metrics."""
+
+
+@dataclass(frozen=True)
+class TableSpec:
+    row: str
+    column: str | None = None  # None: first value cell in the row
+    as_name: str = ""
+    protocol: str | None = None  # override the source-level protocol
+
+
+@dataclass(frozen=True)
+class SourceSpec:
+    repo: str
+    ref: str
+    path: str
+    anchor: str
+    protocol: str
+    tables: tuple[TableSpec, ...]
+    display_anchor: str = ""
+
+
+def _slugify(heading: str) -> str:
+    text = heading.strip().lower()
+    text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE)
+    return re.sub(r"\s+", "-", text).strip("-")
+
+
+def _cell_to_value(cell: str) -> float:
+    text = cell.replace("**", "").replace("%", "").replace(",", "").strip()
+    match = re.search(r"-?\d+(?:\.\d+)?", text)
+    if not match:
+        raise MetricsError(f"no numeric value in cell {cell!r}")
+    return float(match.group())
+
+
+def parse_tables(markdown: str, anchor: str) -> list[dict[str, Any]]:
+    """All tables in the section whose heading slugifies to `anchor`."""
+    lines = markdown.splitlines()
+    start = None
+    for index, line in enumerate(lines):
+        if line.startswith("## "):
+            if start is not None:
+                end = index
+                break
+            if _slugify(line[3:]) == anchor:
+                start = index
+    else:
+        end = len(lines) if start is not None else None
+    if start is None:
+        raise MetricsError(f"section anchor {anchor!r} not found")
+
+    tables: list[dict[str, Any]] = []
+    index = start
+    while index < end:
+        line = lines[index]
+        if line.startswith("|") and index + 1 < end and set(lines[index + 1]) <= set("|-: "):
+            header = [cell.strip() for cell in line.strip("|").split("|")]
+            index += 2
+            rows: list[dict[str, Any]] = []
+            while index < end and lines[index].startswith("|"):
+                cells = [cell.strip() for cell in lines[index].strip("|").split("|")]
+                rows.append({"label": cells[0], "cells": cells, "header": header})
+                index += 1
+            tables.append({"header": header, "rows": rows})
+        else:
+            index += 1
+    if not tables:
+        raise MetricsError(f"no tables under anchor {anchor!r}")
+    return tables
+
+
+def extract(
+    markdown: str, anchor: str, specs: tuple[TableSpec, ...]
+) -> list[dict[str, Any]]:
+    tables = parse_tables(markdown, anchor)
+    out: list[dict[str, Any]] = []
+    for spec in specs:
+        found = None
+        for table in tables:
+            for row in table["rows"]:
+                if spec.row.lower() in row["label"].lower():
+                    found = row
+                    break
+            if found:
+                break
+        if found is None:
+            raise MetricsError(f"row {spec.row!r} not found under {anchor!r}")
+        if spec.column is None:
+            values = [
+                _cell_to_value(cell)
+                for cell in found["cells"][1:]
+                if "%" in cell or re.search(r"\d", cell.replace("**", ""))
+            ]
+            if not values:
+                raise MetricsError(f"no value cells in row {spec.row!r}")
+            value = values[0]
+        else:
+            try:
+                column_index = found["header"].index(spec.column)
+            except ValueError as e:
+                raise MetricsError(
+                    f"column {spec.column!r} not in table header {found['header']}"
+                ) from e
+            value = _cell_to_value(found["cells"][column_index])
+        out.append({"name": spec.as_name or spec.row, "value": value})
+    return out
+
+
+def load_source(source: SourceSpec, cache_dir: Path | None = None) -> str:
+    url = (
+        f"https://raw.githubusercontent.com/{source.repo}/{source.ref}/{source.path}"
+    )
+    if cache_dir is not None:
+        cached = cache_dir / _cache_name(source)
+        if cached.is_file():
+            return cached.read_text(encoding="utf-8")
+    with urllib.request.urlopen(url) as response:
+        text = response.read().decode("utf-8")
+    if cache_dir is not None:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        (cache_dir / _cache_name(source)).write_text(text, encoding="utf-8")
+    return text
+
+
+def _cache_name(source: SourceSpec) -> str:
+    return f"{source.repo.replace('/', '_')}@{source.ref}_{source.path.replace('/', '_')}"
+
+
+
+
+def generate_metrics(
+    model_id: str,
+    mapping_path: Path | str,
+    cache_dir: Path | None = None,
+) -> list[dict[str, Any]]:
+    raw = yaml.safe_load(Path(mapping_path).read_text(encoding="utf-8"))
+    entry = raw.get("models", raw).get(model_id)
+    if entry is None:
+        raise MetricsError(f"no metrics source mapped for {model_id!r}")
+    source = SourceSpec(
+        repo=entry["repo"],
+        ref=entry["ref"],
+        path=entry["path"],
+        anchor=entry["anchor"],
+        protocol=entry["protocol"],
+        display_anchor=entry.get("display_anchor", ""),
+        tables=tuple(
+            TableSpec(
+                row=t["row"],
+                column=t.get("column"),
+                as_name=t.get("as", ""),
+                protocol=t.get("protocol"),
+            )
+            for t in entry["tables"]
+        ),
+    )
+    markdown = load_source(source, cache_dir)
+    extracted = extract(markdown, source.anchor, source.tables)
+    source_ref = f"{source.path}#{source.display_anchor or source.anchor}"
+    return [
+        {
+            "name": m["name"],
+            "value": m["value"],
+            "protocol": spec.protocol or source.protocol,
+            "source": source_ref,
+        }
+        for m, spec in zip(extracted, source.tables, strict=True)
+    ]
+
+
+def check_against_metadata(
+    model_id: str,
+    metadata_path: Path | str,
+    mapping_path: Path | str,
+    cache_dir: Path | None = None,
+) -> list[str]:
+    """Diff generated metrics vs a metadata source file. Returns problems."""
+    generated = generate_metrics(model_id, mapping_path, cache_dir)
+    meta = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8"))
+    recorded = meta.get("metrics", [])
+    problems: list[str] = []
+    if [(m["name"], m["value"]) for m in generated] != [(m["name"], m["value"]) for m in recorded]:
+        problems.append(
+            f"{model_id}: metrics mismatch — generated "
+            f"{[(m['name'], m['value']) for m in generated]} vs metadata "
+            f"{[(m['name'], m['value']) for m in recorded]}"
+        )
+    return problems

tests/test_imf_metrics.py

@@ -0,0 +1,74 @@
+"""Tests for the WO10 metrics generator (inline markdown; no network —
+the network provenance check runs in CI as `imf metrics`)."""
+
+from __future__ import annotations
+
+import pytest
+
+from imf.metrics import MetricsError, TableSpec, _slugify, extract, parse_tables
+
+SAMPLE = """# Results
+
+## G2P (Urdu text → IPA)
+
+### Best result
+
+| Metric | Value | Test set |
+|---|---|---|
+| PER (word-level) | 72.0%* | 12,699 held-out |
+| **CER (char-level)** | **14.77%** | 12,699 held-out |
+| Exact match | 33.6% | 12,699 held-out |
+
+## Khmer transliteration (2026-08-14)
+
+| System | EM | CER | n |
+|---|---|---|---|
+| ByT5-small, early stop @ep15 | **59.66%** | **27.42%** | 895 |
+
+## Key findings
+
+prose without tables
+"""
+
+
+def test_slugify_matches_section_headings() -> None:
+    assert _slugify("G2P (Urdu text → IPA)") == "g2p-urdu-text-ipa"
+    assert _slugify("Khmer transliteration (2026-08-14)") == "khmer-transliteration-2026-08-14"
+
+
+def test_parse_tables_scopes_to_anchor_section() -> None:
+    tables = parse_tables(SAMPLE, "g2p-urdu-text-ipa")
+    assert len(tables) == 1
+    labels = [row["label"] for row in tables[0]["rows"]]
+    assert "PER (word-level)" in labels
+
+
+def test_extract_row_mode_takes_first_value_cell() -> None:
+    metrics = extract(
+        SAMPLE,
+        "g2p-urdu-text-ipa",
+        (
+            TableSpec(row="CER (char-level)", as_name="cer"),
+            TableSpec(row="Exact match", as_name="em"),
+        ),
+    )
+    assert metrics == [{"name": "cer", "value": 14.77}, {"name": "em", "value": 33.6}]
+
+
+def test_extract_column_mode() -> None:
+    metrics = extract(
+        SAMPLE,
+        "khmer-transliteration-2026-08-14",
+        (TableSpec(row="ByT5-small, early stop", column="CER", as_name="cer"),),
+    )
+    assert metrics == [{"name": "cer", "value": 27.42}]
+
+
+def test_missing_anchor_raises() -> None:
+    with pytest.raises(MetricsError, match="anchor"):
+        parse_tables(SAMPLE, "no-such-section")
+
+
+def test_missing_row_raises() -> None:
+    with pytest.raises(MetricsError, match="row"):
+        extract(SAMPLE, "g2p-urdu-text-ipa", (TableSpec(row="Nonexistent row"),))