From 1c5b074bced571f1bf1dcd5b64ab8d146c56d927 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 1 Jul 2026 16:13:37 -0400 Subject: [PATCH 1/2] Add SOI-actuals baseline backtest to reform validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second external-validation family alongside the JCT reform scores: compare simulated baseline totals to published IRS SOI actuals for tax items that are NOT calibration targets (the calibrated national SOI concepts are income lines, EITC, and itemized components). Nine lines from the TY2023 Individual Complete Report (Pub 1304, rev. March 2026), Tables 3.3/1.4: net income tax, AMT, CDCC, nonrefundable education credits, saver's credit, nonrefundable CTC+ODC, refundable ACTC, NIIT, and self-employment tax — each mapped to the PE-US variable with the same scope (documented where scope differs, e.g. income_tax also carries retirement-penalty and unreported-payroll amounts, ~0.5%). Levels need no counterfactual, so all nine share the one baseline simulation. Rows publish in reform_validation.json with category 'IRS SOI actual', window TY2023, and the SOI table as source; an unsimulated backtest marks out_of_sample_simulated=false like skipped OBBBA rows, so the publish guard covers it. Co-Authored-By: Claude Fable 5 --- .../build/us/soi_baseline_levels.json | 128 ++++++++++++++++++ .../build/us_runtime/reform_validation.py | 98 +++++++++++++- .../tests/test_reform_validation.py | 94 +++++++++++++ tools/build_us_fiscal_refresh_release.py | 2 + 4 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 packages/populace-build/src/populace/build/us/soi_baseline_levels.json diff --git a/packages/populace-build/src/populace/build/us/soi_baseline_levels.json b/packages/populace-build/src/populace/build/us/soi_baseline_levels.json new file mode 100644 index 00000000..02f801ac --- /dev/null +++ b/packages/populace-build/src/populace/build/us/soi_baseline_levels.json @@ -0,0 +1,128 @@ +{ + "_comment": "Out-of-sample baseline-level backtest: simulated baseline totals compared to published IRS SOI actuals for tax items that are NOT calibration targets (verified against the release target set — the calibrated national SOI concepts are income lines, EITC, and itemized components). Unlike the reform rows, these need no counterfactual: the populace value is the plain baseline weighted total of the variable at the dataset period, so all lines share one simulation. Benchmarks are tax-year actuals from the SOI Individual Complete Report (Publication 1304); the dataset period is 2024, so a one-year vintage gap (TY2023 actuals) plus population/income growth applies — populace should run slightly above the actuals.", + "schema_version": 1, + "source": { + "publication": "IRS Statistics of Income, Individual Complete Report (Publication 1304), Tax Year 2023 (rev. March 2026)", + "table_3_3": "https://www.irs.gov/pub/irs-soi/23in33ar.xls", + "table_1_4": "https://www.irs.gov/pub/irs-soi/23in14ar.xls" + }, + "levels": [ + { + "id": "soi_income_tax_net", + "name": "Total income tax (net of refundable credits)", + "variable": "income_tax", + "period": 2024, + "benchmark": { + "value": 2042047899000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (total income tax minus refundable credits)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "SOI total income tax minus refundable credits — the same net concept as populace's income_tax. Small known scope difference: the populace measure also includes qualified-retirement-plan penalty taxes and unreported payroll tax (~0.5% of the total), which SOI books outside income tax." + }, + { + "id": "soi_amt", + "name": "Alternative minimum tax", + "variable": "alternative_minimum_tax", + "period": 2024, + "benchmark": { + "value": 2752164000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 1.4 (alternative minimum tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls" + }, + "description": "Post-TCJA AMT is small (about 150,000 returns in TY2023), so this line is sensitive to the top tail of the income distribution." + }, + { + "id": "soi_cdcc", + "name": "Child and dependent care credit", + "variable": "cdcc", + "period": 2024, + "benchmark": { + "value": 3468910000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (credit for child and dependent care)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_education_credits", + "name": "Education credits (nonrefundable)", + "variable": "education_tax_credits", + "period": 2024, + "benchmark": { + "value": 7554668000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable education credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "populace's education_tax_credits is the nonrefundable American Opportunity credit plus the Lifetime Learning credit — the same scope as SOI's nonrefundable education credit column (the refundable AOTC portion is excluded on both sides)." + }, + { + "id": "soi_savers_credit", + "name": "Retirement savings contribution (saver's) credit", + "variable": "savers_credit", + "period": 2024, + "benchmark": { + "value": 2037572000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (retirement savings contribution credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_ctc_nonrefundable", + "name": "Child tax credit + ODC (nonrefundable)", + "variable": "non_refundable_ctc", + "period": 2024, + "benchmark": { + "value": 84418366000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable child and other dependent tax credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_ctc_refundable", + "name": "Additional child tax credit (refundable)", + "variable": "refundable_ctc", + "period": 2024, + "benchmark": { + "value": 34533251000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (refundable child tax credit or additional child tax credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_niit", + "name": "Net investment income tax", + "variable": "net_investment_income_tax", + "period": 2024, + "benchmark": { + "value": 39330017000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (net investment income tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "Concentrated in top-decile capital income, so this line tests the investment-income tail." + }, + { + "id": "soi_se_tax", + "name": "Self-employment tax", + "variable": "self_employment_tax", + "period": 2024, + "benchmark": { + "value": 81087060000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (self-employment tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + } + ] +} diff --git a/packages/populace-build/src/populace/build/us_runtime/reform_validation.py b/packages/populace-build/src/populace/build/us_runtime/reform_validation.py index 19b518ed..c62de7b3 100644 --- a/packages/populace-build/src/populace/build/us_runtime/reform_validation.py +++ b/packages/populace-build/src/populace/build/us_runtime/reform_validation.py @@ -38,10 +38,12 @@ __all__ = [ "REFORM_VALIDATION_SCHEMA_VERSION", + "BaselineLevelSpec", "ReformValidationSpec", "in_sample_reform_specs", "out_of_sample_reform_specs", "tax_expenditure_reform_specs", + "soi_baseline_level_specs", "load_default_reform_specs", "reform_validation_payload", "write_reform_validation", @@ -301,6 +303,63 @@ def tax_expenditure_reform_specs( return tuple(specs) +@dataclass(frozen=True) +class BaselineLevelSpec: + """A baseline total compared to a published actual (no counterfactual). + + Unlike a reform, a level row needs no reform simulation: the populace value + is the plain baseline weighted total of ``variable`` at ``period``, so + every level shares the one baseline simulation. Levels are only meaningful + as validation when the variable is NOT a calibration target — the shipped + config is restricted to SOI tax items outside the target set. + """ + + id: str + name: str + variable: str + period: int + benchmark_value: float + benchmark_year: str + source: str + source_url: str + description: str = "" + + def __post_init__(self) -> None: + if not self.id or not self.variable: + raise ValueError("BaselineLevelSpec requires id and variable.") + + +def _soi_baseline_levels_config_path() -> Path: + return Path(str(files("populace.build.us").joinpath("soi_baseline_levels.json"))) + + +def soi_baseline_level_specs( + path: Path | None = None, +) -> tuple[BaselineLevelSpec, ...]: + """Curated SOI-actual baseline levels from JSON config.""" + config_path = path or _soi_baseline_levels_config_path() + if not config_path.exists(): + return () + payload = json.loads(config_path.read_text()) + specs: list[BaselineLevelSpec] = [] + for raw in payload.get("levels", ()): + bench = raw.get("benchmark", {}) + specs.append( + BaselineLevelSpec( + id=raw["id"], + name=raw["name"], + variable=raw["variable"], + period=int(raw["period"]), + benchmark_value=float(bench["value"]), + benchmark_year=str(bench.get("year", "")), + source=str(bench.get("source", "")), + source_url=str(bench.get("source_url", "")), + description=str(raw.get("description", "")), + ) + ) + return tuple(specs) + + def load_default_reform_specs( *, period: int, @@ -348,6 +407,7 @@ def reform_validation_payload( simulate: SimulateFn | None = None, in_sample_estimates: dict[str, float] | None = None, in_sample_targets: dict[str, float] | None = None, + baseline_levels: Sequence[BaselineLevelSpec] = (), release_id: str | None = None, ) -> dict[str, Any]: """Score each reform on the dataset and render the JSON-stable payload. @@ -516,7 +576,43 @@ def simulated_effect( # genuinely-zero one, and so a release built with simulation skipped is # never mistaken for one where the dataset simply failed the fidelity test. # False only when out-of-sample reforms exist but no simulate() was given. - has_out_of_sample = any(not spec.in_sample for spec in specs) + # Baseline-level backtest rows (SOI actuals): no counterfactual — every + # level reads the shared baseline simulation, so together they cost one + # extra measure per line, not one simulation per line. + for level in baseline_levels: + total = ( + None if simulate is None else baseline_total(level.variable, level.period) + ) + rows.append( + { + "id": level.id, + "name": level.name, + "category": "IRS SOI actual", + "in_sample": False, + "period": level.period, + "description": level.description or None, + "jct": { + "score": _finite(level.benchmark_value), + "score_fy2027": None, + "score_type": "actual", + "window": level.benchmark_year or None, + "source": level.source or None, + "source_url": level.source_url or None, + }, + "populace": { + "budget_effect": None if total is None else _finite(total), + "period": level.period, + "window": level.benchmark_year or None, + "measure": level.variable, + "baseline_total": None if total is None else _finite(total), + "reform_total": None, + }, + } + ) + + has_out_of_sample = any(not spec.in_sample for spec in specs) or bool( + baseline_levels + ) out_of_sample_simulated = simulate is not None or not has_out_of_sample payload: dict[str, Any] = { "schema_version": REFORM_VALIDATION_SCHEMA_VERSION, diff --git a/packages/populace-build/tests/test_reform_validation.py b/packages/populace-build/tests/test_reform_validation.py index 419e71d1..c86427cc 100644 --- a/packages/populace-build/tests/test_reform_validation.py +++ b/packages/populace-build/tests/test_reform_validation.py @@ -478,3 +478,97 @@ def test_write_round_trips(tmp_path): payload = reform_validation_payload([_oos_spec(-1.0)], period=2024, simulate=None) path = write_reform_validation(payload, tmp_path / "reform_validation.json") assert json.loads(path.read_text())["reforms"][0]["id"] == "obbba_salt" + + +def test_soi_baseline_levels_load_from_default_config(): + from populace.build.us_runtime.reform_validation import soi_baseline_level_specs + + levels = soi_baseline_level_specs() + assert len(levels) >= 8 + ids = [lv.id for lv in levels] + assert len(ids) == len(set(ids)) + for lv in levels: + assert lv.benchmark_value > 0 + assert lv.benchmark_year.startswith("TY") + assert "SOI" in lv.source + assert lv.source_url + # Levels must be OUT of the calibration target set; the calibrated + # national SOI concepts are income lines, EITC, and itemized + # components — none of these variables. + assert lv.variable not in { + "adjusted_gross_income", + "eitc", + "taxable_income", + "itemized_taxable_income_deductions", + } + income_tax = next(lv for lv in levels if lv.id == "soi_income_tax_net") + assert income_tax.benchmark_value == pytest.approx(2_042_047_899_000) + + +def test_baseline_levels_share_one_simulation_and_emit_rows(): + from populace.build.us_runtime.reform_validation import BaselineLevelSpec + + calls = [] + + def simulate(reform): + calls.append(reform) + return _FakeSim({"cdcc": 3.6e9, "savers_credit": 2.2e9}) + + levels = ( + BaselineLevelSpec( + id="soi_cdcc", + name="CDCC", + variable="cdcc", + period=2024, + benchmark_value=3.47e9, + benchmark_year="TY2023", + source="IRS SOI Pub 1304 TY2023, Table 3.3", + source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls", + ), + BaselineLevelSpec( + id="soi_savers", + name="Saver's credit", + variable="savers_credit", + period=2024, + benchmark_value=2.04e9, + benchmark_year="TY2023", + source="IRS SOI Pub 1304 TY2023, Table 3.3", + source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls", + ), + ) + payload = reform_validation_payload( + (), period=2024, simulate=simulate, baseline_levels=levels + ) + # Both levels read the one shared baseline simulation. + assert calls == [None] + rows = {row["id"]: row for row in payload["reforms"]} + cdcc = rows["soi_cdcc"] + assert cdcc["category"] == "IRS SOI actual" + assert cdcc["in_sample"] is False + assert cdcc["jct"]["score"] == pytest.approx(3.47e9) + assert cdcc["jct"]["score_type"] == "actual" + assert cdcc["jct"]["window"] == "TY2023" + assert cdcc["populace"]["budget_effect"] == pytest.approx(3.6e9) + assert rows["soi_savers"]["populace"]["budget_effect"] == pytest.approx(2.2e9) + assert payload["out_of_sample_simulated"] is True + + +def test_baseline_levels_null_without_simulate(): + from populace.build.us_runtime.reform_validation import BaselineLevelSpec + + level = BaselineLevelSpec( + id="soi_cdcc", + name="CDCC", + variable="cdcc", + period=2024, + benchmark_value=3.47e9, + benchmark_year="TY2023", + source="IRS SOI", + source_url="https://example.test", + ) + payload = reform_validation_payload( + (), period=2024, simulate=None, baseline_levels=(level,) + ) + assert payload["reforms"][0]["populace"]["budget_effect"] is None + # An unsimulated backtest must mark itself, same as skipped OBBBA rows. + assert payload["out_of_sample_simulated"] is False diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index f13d48da..440f3d46 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -65,6 +65,7 @@ default_simulate_factory, load_default_reform_specs, reform_validation_payload, + soi_baseline_level_specs, write_reform_validation, ) from populace.calibrate import TargetRegistry, calibrate, calibrate_l0_refit @@ -3883,6 +3884,7 @@ def _write_reform_validation( simulate=simulate, in_sample_estimates=_in_sample_estimates(result), in_sample_targets=_in_sample_targets(result), + baseline_levels=soi_baseline_level_specs(), release_id=release_id, ) write_reform_validation(payload, release_dir / "reform_validation.json") From ebaf7e5d355b0cbf544e717abc5d39ce3d626d2e Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 1 Jul 2026 16:16:54 -0400 Subject: [PATCH 2/2] Register soi_baseline_levels.json in the us country-package manifest Co-Authored-By: Claude Fable 5 --- .../populace-build/src/populace/build/us/country_package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/populace-build/src/populace/build/us/country_package.json b/packages/populace-build/src/populace/build/us/country_package.json index af3ecc3d..7fed91f1 100644 --- a/packages/populace-build/src/populace/build/us/country_package.json +++ b/packages/populace-build/src/populace/build/us/country_package.json @@ -6,6 +6,7 @@ "fiscal_target_references.json", "obbba_reforms.json", "puf_aggregate_record_disaggregation.json", + "soi_baseline_levels.json", "source_stages.json", "support_spine.json", "tax_expenditure_reforms.json"