From 4900e6e620bcaed0522a07e49feea1b14cc02bc5 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Mon, 29 Jun 2026 17:22:45 -0400 Subject: [PATCH] Refuse to publish releases with skipped out-of-sample reforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release built with --skip-out-of-sample-reforms publishes blank OBBBA validation, indistinguishable from a real result — and the build invocation that passes that flag lives outside this repo, so it can't be fixed in code. Enforce it at the publish boundary instead: refuse to publish when reform_validation.json reports out_of_sample_simulated false, unless --allow-incomplete-reform-validation is given. UK releases (no reform_validation.json) are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/populace/data/publish_cli.py | 41 +++++++++++++++++++ .../populace-data/tests/test_publish_guard.py | 34 +++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 packages/populace-data/tests/test_publish_guard.py diff --git a/packages/populace-data/src/populace/data/publish_cli.py b/packages/populace-data/src/populace/data/publish_cli.py index cdd0a45f..f18bf1f9 100644 --- a/packages/populace-data/src/populace/data/publish_cli.py +++ b/packages/populace-data/src/populace/data/publish_cli.py @@ -4,12 +4,31 @@ import argparse import json +import sys from pathlib import Path from populace.data.release import publish_release from populace.data.slack import notify_release +def _reform_validation_skipped(release_dir: Path) -> bool: + """True if the release carries a reform_validation.json that was built with + out-of-sample reforms skipped (``out_of_sample_simulated`` false). + + Such a release publishes blank out-of-sample (OBBBA) rows on the dashboard, + indistinguishable from a real result — so publishing one is refused unless + explicitly allowed. Absent/unreadable file or a true flag → not skipped. + """ + path = release_dir / "reform_validation.json" + if not path.exists(): + return False + try: + payload = json.loads(path.read_text()) + except (OSError, ValueError): + return False + return payload.get("out_of_sample_simulated") is False + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("release_dir", help="Local releases/ directory.") @@ -54,8 +73,30 @@ def main(argv: list[str] | None = None) -> int: "--updated-at", help="Optional latest.json timestamp override for reproducible tests.", ) + parser.add_argument( + "--allow-incomplete-reform-validation", + action="store_true", + help=( + "Publish even if reform_validation.json was built with out-of-sample " + "reforms skipped (out_of_sample_simulated false). Off by default so a " + "release never silently ships blank OBBBA validation." + ), + ) args = parser.parse_args(argv) + if not args.allow_incomplete_reform_validation and _reform_validation_skipped( + Path(args.release_dir) + ): + print( + "refusing to publish: reform_validation.json has " + "out_of_sample_simulated=false (built with " + "--skip-out-of-sample-reforms), so the dashboard would show blank " + "out-of-sample reforms. Rebuild without skipping, or pass " + "--allow-incomplete-reform-validation to publish anyway.", + file=sys.stderr, + ) + return 1 + pointer = publish_release( Path(args.release_dir), args.repo_id, diff --git a/packages/populace-data/tests/test_publish_guard.py b/packages/populace-data/tests/test_publish_guard.py new file mode 100644 index 00000000..8240d9c4 --- /dev/null +++ b/packages/populace-data/tests/test_publish_guard.py @@ -0,0 +1,34 @@ +import json +from pathlib import Path + +from populace.data.publish_cli import _reform_validation_skipped, main + + +def _write_rv(release_dir: Path, *, out_of_sample_simulated: bool | None) -> None: + payload: dict = {"schema_version": 1, "reforms": []} + if out_of_sample_simulated is not None: + payload["out_of_sample_simulated"] = out_of_sample_simulated + (release_dir / "reform_validation.json").write_text(json.dumps(payload)) + + +def test_skipped_detects_false_flag(tmp_path): + _write_rv(tmp_path, out_of_sample_simulated=False) + assert _reform_validation_skipped(tmp_path) is True + + +def test_not_skipped_when_simulated(tmp_path): + _write_rv(tmp_path, out_of_sample_simulated=True) + assert _reform_validation_skipped(tmp_path) is False + + +def test_not_skipped_when_no_reform_validation(tmp_path): + # e.g. a UK release, which carries no reform_validation.json + assert _reform_validation_skipped(tmp_path) is False + + +def test_publish_refused_when_out_of_sample_skipped(tmp_path, capsys): + _write_rv(tmp_path, out_of_sample_simulated=False) + # The guard returns before publish_release is ever called (no HF upload). + rc = main([str(tmp_path), "--repo-id", "policyengine/populace-us"]) + assert rc == 1 + assert "refusing to publish" in capsys.readouterr().err