From 21e4fa1534c4584408ec37fc6d8f999ea8c39fe4 Mon Sep 17 00:00:00 2001 From: Yasmin Moslem <48152713+ymoslem@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:17:21 +0100 Subject: [PATCH] Fix TeleMath answer parsing and numeric tolerance The answer parser under-scored every model in the pool by 3 to 8 points. The error is not uniform across models, so it moved per-cluster error rates relative to each other and changed the Stage 1 routing, not only accuracy levels. The scoring change is confined to TeleMath. parse_telemath_answer and numeric_match are reached only by the telemath, telemath_nothink and telemath_gemma4 tasks; AIME and TeleQnA match exactly and their verdicts are untouched. The version stamp and both guards below apply to every task but change no score. Three parsing faults: - A fraction in a final answer was split into separate digit tokens by the last-number fallback, so 7/6 scored as 6.0 rather than 1.1667. \frac, \dfrac, \tfrac and bare a/b are now read as one value. - \boxed{} had to hold a bare number, so a trailing unit or comma thousands separators fell through to a much noisier full-text scan. A leading value followed by a \text{...} unit is now accepted, while other trailing content is still refused as unevaluated maths, so \boxed{2e^{-2}} is not truncated to 2. - Only the final \boxed{} was tried, so a model that boxed a clean decimal and then restated it symbolically had the unparseable second box shadow the first. Boxes are now scanned last to first. numeric_match used abs_tol=1e-9, but five TeleMath gold answers are smaller than that, down to 1e-10, so any answer within 1e-9 including zero scored correct against them. Set to 0 and rely on the relative tolerance. Add SCORER_VERSION, stamped into stats, generations and outcomes, so an artifact records which grader produced it. prep_qe.qe_row refuses a generation stamped with an older version and warns when it copies a stored verdict. Labels taken from the old field are what produced the first quality estimators, which learned to predict parser failures rather than wrong answers. Add check_pre_rendered, which refuses un-templated prompts on a task whose chat template is baked into the dataset. Feeding raw data to such a task truncated 52% of outputs and gave an accuracy of 0.005. --- data/prep_qe.py | 53 +++++++++-- src/cre_router/evaluate.py | 185 +++++++++++++++++++++++++++++++++---- tests/test_evaluate.py | 11 ++- tests/test_prep_qe.py | 10 +- tests/test_telemath.py | 110 +++++++++++++++++++++- 5 files changed, 339 insertions(+), 30 deletions(-) diff --git a/data/prep_qe.py b/data/prep_qe.py index fc0d2df..cf422e1 100644 --- a/data/prep_qe.py +++ b/data/prep_qe.py @@ -21,18 +21,59 @@ import argparse import json from pathlib import Path +import warnings -def qe_row(gen: dict) -> dict: - """One generation row -> one QE example (columns match ymoslem/*-router).""" - correct = bool(gen["correct"]) +from cre_router.evaluate import SCORER_VERSION + +_WARNED: list[int] = [] + + +def qe_row(gen: dict, task: str | None = None) -> dict: + """One generation row -> one QE example (columns match ymoslem/*-router). + + ``task`` names an entry in ``cre_router.evaluate.TASKS``. When given, the + label and the extracted answer are recomputed from ``full_output`` with the + current grader instead of being copied from the generation log. Pass it + whenever the log predates a grader change, so a stale verdict cannot become + a training label. Without it the stored fields are used unchanged. + """ full_output = gen["full_output"] + answer = gen.get("answer") + if task is not None: + from cre_router.evaluate import TASKS + + spec = TASKS[task] + answer = spec.parse(full_output) + match = spec.match or (lambda p, g: p is not None and p == g) + correct = bool(match(answer, gen.get("ground_truth_answer"))) + else: + # No task given, so the stored verdict is copied through. This is the + # hole that produced the first TeleMath QE classifiers: they were built + # from logs graded before the 2026-08-13 fixes, so every training label + # was the old grader's. Copying is still allowed, because AIME and + # TeleQnA logs are unaffected and re-parsing them needs no task, but it + # is never silent: a log that names a scorer older than the current one + # is refused outright. + stored_version = gen.get("scorer_version") + if stored_version is not None and stored_version < SCORER_VERSION: + raise ValueError( + f"generation was graded by scorer_version {stored_version}, current " + f"is {SCORER_VERSION}. Pass task=... so the label is recomputed from " + f"full_output; copying it would train on a stale verdict." + ) + if not _WARNED: + _WARNED.append(1) + warnings.warn( + "qe_row(task=None): copying the stored `correct` field. Pass task= " + "to regrade from full_output.", RuntimeWarning, stacklevel=2) + correct = bool(gen["correct"]) return { "question": gen.get("question", gen.get("prompt", "")), "prompt": gen.get("prompt", ""), "ground_truth_answer": gen.get("ground_truth_answer"), "full_output": full_output, - "answer": gen.get("answer"), + "answer": answer, "accuracy": float(correct), "num_words": len(full_output.split()), "num_tokens": gen["num_tokens"], @@ -45,9 +86,9 @@ def qe_row(gen: dict) -> dict: } -def to_qe_rows(generations: list[dict]) -> list[dict]: +def to_qe_rows(generations: list[dict], task: str | None = None) -> list[dict]: """Convert generation rows to QE examples, pooling multiple files/models.""" - return [qe_row(g) for g in generations] + return [qe_row(g, task=task) for g in generations] def _read_jsonl(path: Path) -> list[dict]: diff --git a/src/cre_router/evaluate.py b/src/cre_router/evaluate.py index 7058d0c..2a6c69d 100644 --- a/src/cre_router/evaluate.py +++ b/src/cre_router/evaluate.py @@ -31,6 +31,17 @@ from cre_router.textutils import split_thinking +# Identifies the grading rules that produced a result file. Bump this whenever a +# parser or a match rule changes what counts as correct, so saved artifacts +# declare their own provenance and a reader never has to infer it from a +# timestamp. Stamped into stats files and generation rows. +# +# 1 original rules +# 2 2026-08-13: TeleMath parser fix (fractions, units and comma separators +# inside \boxed{}, double-boxed answers, exponent leakage) and +# numeric_match abs_tol 1e-9 -> 0 +SCORER_VERSION = 2 + # --------------------------------------------------------------------------- # Answer parsing # --------------------------------------------------------------------------- @@ -79,35 +90,101 @@ def parse_teleqna_answer(text: str) -> int | None: # optional exponent. Unlike ``\d*\.?\d+`` this has no two adjacent # variable-length digit runs, so it cannot backtrack catastrophically. _TELEMATH_NUMBER = r"[-+]?(?:\d+(?:\.\d+)?|\.\d+)(?:[eE][-+]?\d+)?" +# A LaTeX fraction (``\frac{7}{6}`` or ``\dfrac``/``\tfrac``), a bare ``a/b``, +# or a plain number, in that priority. Each numerator/denominator/number is +# the bounded ``_TELEMATH_NUMBER`` above, so this has the same no-backtracking +# guarantee. +_TELEMATH_VALUE = ( + rf"(?:(?P[-+])?\\[dt]?frac\{{\s*(?P{_TELEMATH_NUMBER})\s*\}}\{{\s*(?P{_TELEMATH_NUMBER})\s*\}}" + rf"|(?P{_TELEMATH_NUMBER})\s*/\s*(?P{_TELEMATH_NUMBER})" + rf"|(?P{_TELEMATH_NUMBER}))" +) +_TELEMATH_BOXED = re.compile(r"\\boxed\{((?:[^{}]|\{[^{}]*\})*)\}") +# A value is only trusted with trailing content after it (e.g. a unit) when +# that content is a harmless label rather than more math -- otherwise +# ``\boxed{2e^{-2}}`` (meaning 2 times e to the minus 2) would truncate to 2. +_TELEMATH_SAFE_TRAILER = re.compile(r"^\s*(\\text\{|$)") +# A superscript exponent, braced or bare. Bounded repetition keeps this linear. +_TELEMATH_SUPERSCRIPT = re.compile(r"\^\s*\{[^{}]{0,20}\}|\^\s*-?\d+(?:\.\d+)?") + + +def _telemath_value(match: re.Match) -> float | None: + if match.group("fn") is not None: + num, den = float(match.group("fn")), float(match.group("fd")) + if match.group("fsign") == "-": + num = -num + elif match.group("rn") is not None: + num, den = float(match.group("rn")), float(match.group("rd")) + else: + return float(match.group("num")) + return num / den if den != 0 else None + + +def _telemath_value_at_start(s: str) -> float | None: + """A fraction or number from the start of ``s``, ignoring a trailing + unit label, or None if nothing trustworthy is at the start.""" + s = s.lstrip() + match = re.match(_TELEMATH_VALUE, s) + if match and _TELEMATH_SAFE_TRAILER.match(s[match.end():]): + try: + return _telemath_value(match) + except (ValueError, ZeroDivisionError): + return None + return None def parse_telemath_answer(text: str) -> float | None: """Extract a TeleMath numerical answer (a float) from a completion. - TeleMath answers are numerical quantities, often long decimals or in - scientific notation (e.g. 233.333333333333, 7.2e-05, -62.0854). The final - value is taken from a ``\\boxed{}`` when present, then an explicit - ``Answer:``, then the last number in the text. LaTeX scientific notation - (``7.2 \\times 10^{-5}``) is normalised to ``7.2e-5`` before matching. + TeleMath answers are numerical quantities, often long decimals, LaTeX + fractions, or scientific notation (e.g. 233.333333333333, 7.2e-05, + \\frac{7}{6}, -62.0854). The final value is taken from a ``\\boxed{}`` + when present, then an explicit ``Answer:``, then the last value anywhere + in the text. LaTeX scientific notation (``7.2 \\times 10^{-5}``) and + comma thousands separators (``1,382,400``) are normalised before + matching. A fraction inside ``\\boxed{}``/``Answer:`` is evaluated + (``\\frac{7}{6}`` -> 1.1667); a bare unit label after the value is + ignored (``\\boxed{0.2 \\text{ packets/s}}`` -> 0.2), but anything else + trailing it is treated as more math and the match is rejected rather than + silently truncated (``\\boxed{2e^{-2}}`` is not truncated to 2). """ content = split_thinking(text)[-_TELEMATH_TAIL_CHARS:] + content = re.sub(r"\d{1,3}(?:,\d{3})+", lambda m: m.group(0).replace(",", ""), content) content = re.sub( r"([-+]?(?:\d+(?:\.\d+)?|\.\d+))\s*\\times\s*10\^\{?(-?\d+)\}?", r"\1e\2", content, ) - for pattern in ( - rf"\\boxed\{{\s*({_TELEMATH_NUMBER})\s*\}}", - rf"\*{{0,2}}Answer\*{{0,2}}\s*[:：]\s*({_TELEMATH_NUMBER})", - rf"({_TELEMATH_NUMBER})", - ): - matches = re.findall(pattern, content) - if matches: - try: - return float(matches[-1]) - except ValueError: - continue - return None + + # A model sometimes boxes the same answer twice, a clean decimal first and + # a symbolic restatement last (``\boxed{1.732}`` ... ``\boxed{\sqrt{3}}``). + # Scan boxed occurrences from last to first so an unparseable final box + # falls back to an earlier clean one, rather than to the noisier tiers + # below. + for candidate in reversed(_TELEMATH_BOXED.findall(content)): + value = _telemath_value_at_start(candidate) + if value is not None: + return value + + labelled = re.findall(rf"\*{{0,2}}Answer\*{{0,2}}\s*[:：]\s*(.{{0,80}})", content) + if labelled: + value = _telemath_value_at_start(labelled[-1]) + if value is not None: + return value + + # Last resort: the last value anywhere. Blank out superscript exponents + # first, or ``2e^{-2}`` and ``10^{-0.3}`` contribute their exponent as a + # standalone candidate and the scan ends on it. + content = _TELEMATH_SUPERSCRIPT.sub(" ", content) + last = None + for match in re.finditer(_TELEMATH_VALUE, content): + try: + value = _telemath_value(match) + except (ValueError, ZeroDivisionError): + continue + if value is not None: + last = value + return last def answers_match(predicted: int | None, gold: Any) -> bool: @@ -124,12 +201,21 @@ def numeric_match(predicted: float | None, gold: Any, rel_tol: float = 1e-2) -> Uses ``math.isclose`` with a 1% relative tolerance, which accepts the same quantity reported at different rounding (233.33 vs 233.333333) and rejects - genuinely different values; the small absolute floor covers near-zero golds. + genuinely different values. TeleMath publishes no tolerance of its own, so + 1% is our stated choice; it sits inside the range over which model ranking + and cascade break-even are both invariant (see + ``ref/results/telemath/tolerance_decision.md``). + + The comparison is purely relative. An absolute floor cannot be used here + because some gold answers are themselves smaller than any plausible floor + (down to 1e-10), so a floor would accept zero as correct for them. A gold + of exactly zero still matches a predicted zero, since ``math.isclose`` + compares equal values as close at any tolerance. """ if predicted is None: return False try: - return math.isclose(float(predicted), float(gold), rel_tol=rel_tol, abs_tol=1e-9) + return math.isclose(float(predicted), float(gold), rel_tol=rel_tol, abs_tol=0.0) except (TypeError, ValueError): return False @@ -399,13 +485,20 @@ def merge_model_into_stats( stats_path: str | Path, model_name: str, entry: dict, sizes: dict[str, int] ) -> None: """Add or update one model's entry in a stats JSON, preserving other - models. Cluster sizes are (re)written from the evaluated dataset.""" + models. Cluster sizes are (re)written from the evaluated dataset. + + The file records ``scorer_version``, so a reader can tell which grader + produced its per-cluster error rates instead of guessing from the file's + timestamp. A file without the key predates the stamp and should be treated + as ungraded by the current rules. + """ path = Path(stats_path) stats = json.loads(path.read_text()) if path.exists() else {} stats.setdefault("cluster_sizes", {}) stats.setdefault("models", {}) stats["cluster_sizes"] = {str(k): int(v) for k, v in sorted(sizes.items())} stats["models"][model_name] = entry + stats["scorer_version"] = SCORER_VERSION path.write_text(json.dumps(stats, indent=2) + "\n") @@ -527,6 +620,51 @@ def question_id(item: dict) -> str: return hashlib.md5(item["prompt"].encode("utf-8")).hexdigest()[:12] +# Chat-template control tokens, across the families we serve. A pre_rendered +# prompt has been through the tokenizer's template and carries at least one. +_TEMPLATE_MARKER = re.compile( + r"<\|[^|>]{1,40}\|?>?" # Gemma 4 <|turn>, GPT-style <|im_start|> + r"||" # SentencePiece BOS + r"|" # earlier Gemma + r"|\[INST\]" # Llama 2 / Mistral + r"|<｜[^｜]{1,40}｜>" # DeepSeek full-width bars +) + + +def check_pre_rendered(dataset: list[dict], task: Task, sample: int = 32) -> None: + """Fail fast when a pre_rendered task is handed un-templated prompts. + + A pre_rendered task serves ``prompt`` verbatim, with the chat template + already baked in by ``data/prep_gemma4_thinking.py``. Passing the raw + dataset instead is not an error the server reports: it answers happily, but + the model never receives the tokens that open and close its thinking + section, so it never emits the matching stop token and generates until it + hits ``max_tokens``. A run that hit this burned three and a half GPU-hours + and returned 52% truncated output at 0.5% accuracy, which is only + recognisable after the fact. + + Only ``telemath_gemma4`` is pre_rendered today, so in practice this guards + the Gemma thinking runs, but it keys off the task flag rather than the model + name so any future pre-rendered task is covered too. + """ + if not task.pre_rendered: + return + prompts = [row.get("prompt", "") for row in dataset[:sample]] + if not prompts: + return + bad = sum(1 for p in prompts if not _TEMPLATE_MARKER.search(p)) + if bad: + raise ValueError( + f"task {task.name!r} is pre_rendered, but {bad} of {len(prompts)} sampled " + f"prompts carry no chat-template markers. The raw dataset was almost " + f"certainly passed instead of the pre-rendered one; serving it would " + f"generate to max_tokens without terminating. Use the output of " + f"data/prep_gemma4_thinking.py, e.g. " + f"telemath_train_gemma__think.jsonl.\n" + f" first offending prompt: {next(p for p in prompts if not _TEMPLATE_MARKER.search(p))[:120]!r}" + ) + + def evaluate_model( dataset: list[dict], model: str, @@ -561,6 +699,7 @@ def evaluate_model( generations themselves it is irrecoverable once the run ends. """ run_benchmark = benchmark or run_vllm_benchmark + check_pre_rendered(dataset, task) workdir = Path(workdir) workdir.mkdir(parents=True, exist_ok=True) @@ -598,6 +737,11 @@ def evaluate_model( "run": run, "correct": bool(ok), "output_len": olen, + # An outcomes file keeps no generated text, so its + # verdicts can never be rechecked on their own. The + # stamp is the only way a reader can tell whether + # they were graded by the current rules. + "scorer_version": SCORER_VERSION, } ) if generations is not None: @@ -620,6 +764,7 @@ def evaluate_model( "full_output": text, "num_tokens": olen, "correct": bool(ok), + "scorer_version": SCORER_VERSION, } ) tpot_ms = result.get("mean_tpot_ms") diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index e07bc54..20efd3f 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -6,6 +6,7 @@ import pytest from cre_router.evaluate import ( + SCORER_VERSION, TASKS, RunMeasurement, aggregate_runs, @@ -196,8 +197,13 @@ def fake_benchmark(dataset_path, model, task, *, host, port, max_concurrency, se recs = [json.loads(line) for line in open(out)] assert len(recs) == 3 * 2 # questions x runs for r in recs: - assert set(r) == {"qid", "cluster", "run", "correct", "output_len"} + assert set(r) == { + "qid", "cluster", "run", "correct", "output_len", "scorer_version", + } assert r["output_len"] == 5 + # An outcomes row keeps no text, so the stamp is the only record of + # which grading rules produced its verdict. + assert r["scorer_version"] == SCORER_VERSION by_qid: dict = {} for r in recs: by_qid.setdefault(r["qid"], []).append(r["correct"]) @@ -232,9 +238,10 @@ def fake_benchmark(dataset_path, model, task, *, host, port, max_concurrency, se for r in recs: assert set(r) == { "qid", "cluster", "run", "question", "prompt", "ground_truth_answer", - "answer", "full_output", "num_tokens", "correct", + "answer", "full_output", "num_tokens", "correct", "scorer_version", } assert r["num_tokens"] == 7 + assert r["scorer_version"] == SCORER_VERSION by_qid = {r["qid"]: r for r in recs} # no explicit question field -> falls back to the (raw) prompt assert by_qid["a"]["question"] == "q0" and by_qid["a"]["prompt"] == "q0" diff --git a/tests/test_prep_qe.py b/tests/test_prep_qe.py index 18188e1..3a02532 100644 --- a/tests/test_prep_qe.py +++ b/tests/test_prep_qe.py @@ -23,10 +23,18 @@ def _gen(qid, cluster, correct, out="the answer is 4", ntok=12): class TestQeRow: def test_correct_maps_to_accept(self): - row = prep_qe.qe_row(_gen("a", 0, True)) + # No task, so the stored verdict is copied and the caller is warned. + with pytest.warns(RuntimeWarning, match="copying the stored"): + row = prep_qe.qe_row(_gen("a", 0, True)) assert row["decision_label"] == 1 and row["decision_str"] == "accept" assert row["score"] == 1.0 and row["accuracy"] == 1.0 + def test_stale_scorer_version_is_refused(self): + """A log naming an older scorer must not become a training label.""" + gen = _gen("a", 0, True) | {"scorer_version": 1} + with pytest.raises(ValueError, match="scorer_version"): + prep_qe.qe_row(gen) + def test_wrong_maps_to_route(self): row = prep_qe.qe_row(_gen("b", 1, False)) assert row["decision_label"] == 0 and row["decision_str"] == "route" diff --git a/tests/test_telemath.py b/tests/test_telemath.py index eae0d5c..f04ea81 100644 --- a/tests/test_telemath.py +++ b/tests/test_telemath.py @@ -8,6 +8,7 @@ from cre_router.evaluate import ( TASKS, + check_pre_rendered, numeric_match, parse_telemath_answer, score_generations, @@ -46,6 +47,61 @@ def test_no_number_returns_none(self): def test_leading_dot_decimal(self): assert parse_telemath_answer(r"\boxed{.5}") == pytest.approx(0.5) + @pytest.mark.parametrize( + "text, expected", + [ + (r"Final answer: $7/6 \approx 1.1667$\n\n$$\frac{7}{6}$$", 7 / 6), + (r"$$\boxed{\frac{7}{6}}$$", 7 / 6), + (r"$$\boxed{-\frac{3}{4}}$$", -0.75), + (r"$$\boxed{-\dfrac{3}{4}}$$", -0.75), + ("Answer: 7/6", 7 / 6), + ], + ) + def test_fraction_answer(self, text, expected): + # A fraction is one value, not two separate numbers -- the naive + # "last number in the text" fallback used to read \frac{7}{6} as its + # denominator alone (6.0) rather than 7/6. + assert parse_telemath_answer(text) == pytest.approx(expected) + + def test_boxed_with_unit_label(self): + # A unit after the value inside \boxed{} must not make the whole + # match fail and fall through to a noisier, unrelated number. + assert parse_telemath_answer(r"$$\boxed{0.2 \text{ packets/s}}$$") == pytest.approx(0.2) + + def test_boxed_comma_thousands(self): + assert parse_telemath_answer(r"$$\boxed{1,382,400,000,000}$$") == pytest.approx(1382400000000.0) + + def test_exponent_is_not_mistaken_for_the_answer(self): + # With no \boxed{} to anchor on, the last-value scan must not end on + # the "-2" inside e^{-2}; the stated decimal is the answer. + text = "We get $2e^{-2}$, i.e. 0.2707 in decimal." + assert parse_telemath_answer(text) == pytest.approx(0.2707) + + def test_later_unparseable_box_falls_back_to_earlier_clean_box(self): + # A model sometimes boxes the same answer twice: a clean decimal, + # then a symbolic restatement at the very end. The symbolic box + # should not shadow the clean one that already answered the question. + text = ( + r"$$\boxed{1.732} \text{V}$$" + "\n\nSince this is exact, we can also write it as:\n\n" + r"$$\boxed{\sqrt{3}}$$" + ) + assert parse_telemath_answer(text) == pytest.approx(1.732) + + def test_boxed_trailing_math_is_not_truncated(self): + # \boxed{2e^{-2}} means 2 * e^-2 (approx 0.2707), not the literal + # digit 2 with "e^{-2}" as an ignorable label -- unlike a unit, this + # trailing content changes the value, so the leading digit must not + # be accepted on its own. Matches a real generation: the model + # restates the decimal after the box, which the parser should prefer + # over misreading a fragment of the box's own exponent. + text = ( + r"$$P[X=2] \approx 0.27067056$$" + "\n\nThe exact numerical answer is $2e^{-2}$.\n\n" + r"$$\boxed{2e^{-2}}$$ (or approximately 0.2707)" + ) + assert parse_telemath_answer(text) == pytest.approx(0.2707, rel=1e-3) + def test_answer_at_end_of_long_reasoning(self): # A long completion whose answer sits at the very end still parses; the # tail window keeps parsing bounded without clipping a real answer. @@ -90,9 +146,31 @@ def test_negative(self): def test_near_zero_gold(self): assert numeric_match(0.0, 0.0) - assert numeric_match(1e-10, 0.0) + assert not numeric_match(1e-10, 0.0) assert not numeric_match(0.5, 0.0) + @pytest.mark.parametrize( + "pred, gold, expected", + [ + (5e-10, 5e-10, True), + (4.99e-10, 5e-10, True), # within 1%, a genuine answer + (0.0, 5e-10, False), # answering zero is not solving for 5e-10 + (1.5e-55, 5e-10, False), + (0.0, 2e-10, False), + ], + ) + def test_tiny_gold_is_not_given_away(self, pred, gold, expected): + # Several TeleMath golds are smaller than any plausible absolute floor + # (down to 1e-10). An abs_tol of 1e-9 made every one of them passable by + # answering 0, inflating accuracy by up to 1pp and most on the strong + # tier. The comparison is relative only. + assert numeric_match(pred, gold) is expected + + def test_long_decimal_rounded_by_the_model(self): + # The common shape: gold stored at full precision, model reports a + # correctly rounded value. Relative error 4.8e-07, far inside 1%. + assert numeric_match(8.92339, 8.9233943) + def test_gold_as_string(self): assert numeric_match(6.0, "6.0") @@ -113,3 +191,33 @@ def test_score_generations_uses_task_matcher(self): ) assert correct == [True, True, False] assert error == pytest.approx(1 / 3) + + +class TestPreRenderedGuard: + """A pre_rendered task served raw prompts fails silently at runtime: the + model never sees its thinking delimiters, so it generates to max_tokens. + The guard turns that into an immediate error.""" + + def _raw(self, n=3): + return [{"prompt": f"Determine the throughput of a network with {i} nodes.", + "answer": 1.0, "cluster": "0", "id": str(i)} for i in range(n)] + + def _templated(self, n=3): + return [{"prompt": f"<|turn>system\n<|think|>\n\n<|turn>user\nQ{i}", + "answer": 1.0, "cluster": "0", "id": str(i)} for i in range(n)] + + def test_raw_prompts_rejected_for_pre_rendered_task(self): + with pytest.raises(ValueError, match="pre_rendered"): + check_pre_rendered(self._raw(), TASKS["telemath_gemma4"]) + + def test_templated_prompts_accepted(self): + check_pre_rendered(self._templated(), TASKS["telemath_gemma4"]) + + @pytest.mark.parametrize("task", ["telemath", "telemath_nothink"]) + def test_non_pre_rendered_tasks_are_unaffected(self, task): + # These serve raw text by design; the guard must not fire. + check_pre_rendered(self._raw(), TASKS[task]) + + def test_error_names_the_fix(self): + with pytest.raises(ValueError, match="prep_gemma4_thinking"): + check_pre_rendered(self._raw(), TASKS["telemath_gemma4"])