From 54ca63bfda5342b08de6cd0f9f52f47b0935d902 Mon Sep 17 00:00:00 2001 From: Wolfie Date: Sat, 14 Feb 2026 00:04:54 -0800 Subject: [PATCH] fix: make capability failures explicit and add fidelity delta report --- src/excelbench/harness/adapters/base.py | 8 +- src/excelbench/harness/runner.py | 19 +++++ src/excelbench/results/renderer.py | 98 +++++++++++++++++++++++++ tests/test_adapter_base.py | 12 +++ tests/test_named_ranges.py | 10 ++- tests/test_renderer_utils.py | 54 ++++++++++++++ tests/test_runner_utils.py | 20 +++++ tests/test_tables.py | 10 ++- 8 files changed, 219 insertions(+), 12 deletions(-) diff --git a/src/excelbench/harness/adapters/base.py b/src/excelbench/harness/adapters/base.py index 095040b..93202e0 100644 --- a/src/excelbench/harness/adapters/base.py +++ b/src/excelbench/harness/adapters/base.py @@ -316,7 +316,7 @@ def read_named_ranges(self, workbook: Any, sheet: str) -> list[JSONDict]: - refers_to: reference formula (e.g. Sheet1!$A$1) """ - return [] + raise NotImplementedError(f"{self.name} does not implement named range reads") def add_named_range(self, workbook: Any, sheet: str, named_range: JSONDict) -> None: """Add a named range. @@ -324,7 +324,7 @@ def add_named_range(self, workbook: Any, sheet: str, named_range: JSONDict) -> N named_range should include keys: name, scope, refers_to. """ - return None + raise NotImplementedError(f"{self.name} does not implement named range writes") def read_tables(self, workbook: Any, sheet: str) -> list[JSONDict]: """Read table (ListObject) definitions from a sheet. @@ -339,7 +339,7 @@ def read_tables(self, workbook: Any, sheet: str) -> list[JSONDict]: - autofilter: bool (optional) """ - return [] + raise NotImplementedError(f"{self.name} does not implement table reads") def add_table(self, workbook: Any, sheet: str, table: JSONDict) -> None: """Add a table (ListObject) to a sheet. @@ -347,7 +347,7 @@ def add_table(self, workbook: Any, sheet: str, table: JSONDict) -> None: table dict should include keys: name, ref, style, columns, header_row, totals_row. """ - return None + raise NotImplementedError(f"{self.name} does not implement table writes") # ========================================================================= # Write Operations diff --git a/src/excelbench/harness/runner.py b/src/excelbench/harness/runner.py index 4fb3b5d..0d98e80 100644 --- a/src/excelbench/harness/runner.py +++ b/src/excelbench/harness/runner.py @@ -37,6 +37,23 @@ JSONDict = dict[str, Any] +def _failure_note_from_actual(actual: JSONDict) -> str: + if "error" in actual: + error_text = str(actual.get("error", "")).lower() + unsupported_markers = ( + "notimplemented", + "not implemented", + "unsupported", + "not supported", + "read-only", + "write-only", + ) + if any(marker in error_text for marker in unsupported_markers): + return "Not implemented" + return "Incorrect result" + return "Incorrect result" + + def _build_exception_diagnostic( adapter: ExcelAdapter, *, @@ -417,6 +434,7 @@ def test_read_case( passed=passed, expected=expected, actual=actual, + notes=None if passed else _failure_note_from_actual(actual), diagnostics=( [] if passed @@ -492,6 +510,7 @@ def test_read_case( passed=passed, expected=expected, actual=actual, + notes=None if passed else _failure_note_from_actual(actual), diagnostics=( [] if passed diff --git a/src/excelbench/results/renderer.py b/src/excelbench/results/renderer.py index 02d48a0..3491b88 100644 --- a/src/excelbench/results/renderer.py +++ b/src/excelbench/results/renderer.py @@ -87,6 +87,7 @@ def render_results(results: BenchmarkResults, output_dir: Path) -> None: render_markdown(results, output_dir / "README.md") render_csv(results, output_dir / "matrix.csv") _append_history(results, output_dir) + _render_fidelity_deltas(output_dir) def render_json(results: BenchmarkResults, path: Path) -> None: @@ -719,6 +720,103 @@ def _append_history(results: BenchmarkResults, output_dir: Path) -> None: f.write(json.dumps(entry) + "\n") +def _render_fidelity_deltas(output_dir: Path) -> None: + """Render a markdown report comparing the two most recent fidelity runs.""" + history_path = output_dir / "history.jsonl" + out_path = output_dir / "FIDELITY_DELTAS.md" + if not history_path.exists(): + out_path.write_text("# Fidelity Deltas\n\nNo history available.\n") + return + + entries: list[dict[str, Any]] = [] + for line in history_path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + entries.append(parsed) + + if len(entries) < 2: + out_path.write_text("# Fidelity Deltas\n\nNeed at least two runs in history.jsonl.\n") + return + + previous = entries[-2] + current = entries[-1] + deltas = _compute_fidelity_deltas(previous, current) + + lines: list[str] = ["# Fidelity Deltas", ""] + lines.append(f"- Previous run: `{previous.get('run_date', 'unknown')}`") + lines.append(f"- Current run: `{current.get('run_date', 'unknown')}`") + lines.append("") + + if not deltas: + lines.append("No score changes detected.") + lines.append("") + out_path.write_text("\n".join(lines)) + return + + regressions = [d for d in deltas if d["delta"] < 0] + improvements = [d for d in deltas if d["delta"] > 0] + + lines.append("## Summary") + lines.append("") + lines.append(f"- Regressions: **{len(regressions)}**") + lines.append(f"- Improvements: **{len(improvements)}**") + lines.append(f"- Net score change: **{sum(d['delta'] for d in deltas):+d}**") + lines.append("") + + lines.append("## Changed Scores") + lines.append("") + lines.append("| Library | Feature | Mode | Previous | Current | Δ |") + lines.append("|---------|---------|------|----------|---------|---|") + for item in sorted(deltas, key=lambda d: (d["delta"], d["library"], d["feature"], d["mode"])): + lines.append( + f"| {item['library']} | {item['feature']} | {item['mode']} | " + f"{item['previous']} | {item['current']} | {item['delta']:+d} |" + ) + lines.append("") + + out_path.write_text("\n".join(lines)) + + +def _compute_fidelity_deltas( + previous: dict[str, Any], current: dict[str, Any] +) -> list[dict[str, Any]]: + """Compute score deltas between two history entries.""" + deltas: list[dict[str, Any]] = [] + prev_scores: dict[str, Any] = previous.get("scores", {}) + curr_scores: dict[str, Any] = current.get("scores", {}) + + for library in sorted(set(prev_scores) | set(curr_scores)): + prev_lib = prev_scores.get(library, {}) + curr_lib = curr_scores.get(library, {}) + for feature in sorted(set(prev_lib) | set(curr_lib)): + prev_feature = prev_lib.get(feature, {}) + curr_feature = curr_lib.get(feature, {}) + for mode in ("read", "write"): + prev_value = prev_feature.get(mode) + curr_value = curr_feature.get(mode) + if prev_value is None or curr_value is None: + continue + if prev_value == curr_value: + continue + deltas.append( + { + "library": library, + "feature": feature, + "mode": mode, + "previous": int(prev_value), + "current": int(curr_value), + "delta": int(curr_value) - int(prev_value), + } + ) + return deltas + + def _diagnostic_to_json(diagnostic: Diagnostic) -> dict[str, Any]: return { "category": diagnostic.category.value, diff --git a/tests/test_adapter_base.py b/tests/test_adapter_base.py index 391260a..3602d83 100644 --- a/tests/test_adapter_base.py +++ b/tests/test_adapter_base.py @@ -332,3 +332,15 @@ def test_build_mismatch_diagnostic() -> None: ) assert diag.category == DiagnosticCategory.DATA_MISMATCH assert "expected" in diag.adapter_message + + +def test_tier3_defaults_raise_not_implemented() -> None: + adapter = ConcreteReadOnly() + with pytest.raises(NotImplementedError, match="named range reads"): + adapter.read_named_ranges(None, "S") + with pytest.raises(NotImplementedError, match="named range writes"): + adapter.add_named_range(None, "S", {}) + with pytest.raises(NotImplementedError, match="table reads"): + adapter.read_tables(None, "S") + with pytest.raises(NotImplementedError, match="table writes"): + adapter.add_table(None, "S", {}) diff --git a/tests/test_named_ranges.py b/tests/test_named_ranges.py index ee04302..306884a 100644 --- a/tests/test_named_ranges.py +++ b/tests/test_named_ranges.py @@ -16,13 +16,15 @@ class TestNamedRangesBase: """Base adapter API surface for named ranges.""" - def test_read_named_ranges_default_returns_empty(self) -> None: + def test_read_named_ranges_default_raises(self) -> None: adapter = StubExcelAdapter() - assert adapter.read_named_ranges(object(), "S1") == [] + with pytest.raises(NotImplementedError, match="named range reads"): + adapter.read_named_ranges(object(), "S1") - def test_add_named_range_default_is_noop(self) -> None: + def test_add_named_range_default_raises(self) -> None: adapter = StubExcelAdapter() - adapter.add_named_range(object(), "S1", {"name": "X", "refers_to": "S1!$A$1"}) + with pytest.raises(NotImplementedError, match="named range writes"): + adapter.add_named_range(object(), "S1", {"name": "X", "refers_to": "S1!$A$1"}) class TestOpenpyxlNamedRanges: diff --git a/tests/test_renderer_utils.py b/tests/test_renderer_utils.py index 1ac7a4b..56197c9 100644 --- a/tests/test_renderer_utils.py +++ b/tests/test_renderer_utils.py @@ -16,8 +16,10 @@ TestResult, ) from excelbench.results.renderer import ( + _compute_fidelity_deltas, _get_git_commit, _group_test_cases, + _render_fidelity_deltas, _render_per_test_table, render_markdown, score_emoji, @@ -260,3 +262,55 @@ def test_render_markdown_write_only_lib_stats(tmp_path: Path) -> None: content = out.read_text() assert "xlsxwriter" in content assert "Write" in content + + +# ───────────────────────────────────────────────── +# fidelity deltas +# ───────────────────────────────────────────────── + + +def test_compute_fidelity_deltas_detects_changes() -> None: + previous = { + "scores": {"openpyxl": {"cell_values": {"read": 3, "write": 3}}}, + } + current = { + "scores": {"openpyxl": {"cell_values": {"read": 2, "write": 3}}}, + } + deltas = _compute_fidelity_deltas(previous, current) + assert deltas == [ + { + "library": "openpyxl", + "feature": "cell_values", + "mode": "read", + "previous": 3, + "current": 2, + "delta": -1, + } + ] + + +def test_render_fidelity_deltas_needs_two_runs(tmp_path: Path) -> None: + out_dir = tmp_path / "results" + out_dir.mkdir(parents=True) + (out_dir / "history.jsonl").write_text('{"scores": {}}\n') + _render_fidelity_deltas(out_dir) + content = (out_dir / "FIDELITY_DELTAS.md").read_text() + assert "Need at least two runs" in content + + +def test_render_fidelity_deltas_writes_regression_table(tmp_path: Path) -> None: + out_dir = tmp_path / "results" + out_dir.mkdir(parents=True) + (out_dir / "history.jsonl").write_text( + "\n".join( + [ + '{"run_date":"2026-01-01T00:00:00Z","scores":{"openpyxl":{"cell_values":{"read":3,"write":3}}}}', + '{"run_date":"2026-01-02T00:00:00Z","scores":{"openpyxl":{"cell_values":{"read":2,"write":3}}}}', + ] + ) + + "\n" + ) + _render_fidelity_deltas(out_dir) + content = (out_dir / "FIDELITY_DELTAS.md").read_text() + assert "Regressions: **1**" in content + assert "| openpyxl | cell_values | read | 3 | 2 | -1 |" in content diff --git a/tests/test_runner_utils.py b/tests/test_runner_utils.py index 93e44fe..918bf75 100644 --- a/tests/test_runner_utils.py +++ b/tests/test_runner_utils.py @@ -18,6 +18,7 @@ _deep_compare, _extract_column, _extract_formula_sheet_names, + _failure_note_from_actual, _find_by_key, _find_range, _find_rule, @@ -619,3 +620,22 @@ def test_border_from_expected_edge_color_no_style() -> None: assert border.top is not None assert border.top.style == BorderStyle.THIN assert border.top.color == "#FF0000" + + +# ───────────────────────────────────────────────── +# failure note mapping +# ───────────────────────────────────────────────── + + +def test_failure_note_from_actual_not_implemented() -> None: + assert _failure_note_from_actual({"error": "NotImplementedError: foo"}) == "Not implemented" + + +def test_failure_note_from_actual_unsupported() -> None: + actual = _failure_note_from_actual({"error": "feature unsupported by adapter"}) + assert actual == "Not implemented" + + +def test_failure_note_from_actual_incorrect() -> None: + assert _failure_note_from_actual({"value": 1}) == "Incorrect result" + assert _failure_note_from_actual({"error": "ValueError: mismatch"}) == "Incorrect result" diff --git a/tests/test_tables.py b/tests/test_tables.py index 03a7a74..55b824e 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -20,13 +20,15 @@ class _StubAdapter(StubExcelAdapter): class TestTablesBase: """Base adapter API surface for tables.""" - def test_read_tables_default_returns_empty(self) -> None: + def test_read_tables_default_raises(self) -> None: adapter = _StubAdapter() - assert adapter.read_tables(object(), "S1") == [] + with pytest.raises(NotImplementedError, match="table reads"): + adapter.read_tables(object(), "S1") - def test_add_table_default_is_noop(self) -> None: + def test_add_table_default_raises(self) -> None: adapter = _StubAdapter() - adapter.add_table(object(), "S1", {"table": {"name": "T", "ref": "A1:B2"}}) + with pytest.raises(NotImplementedError, match="table writes"): + adapter.add_table(object(), "S1", {"table": {"name": "T", "ref": "A1:B2"}}) class TestOpenpyxlTables: