diff --git a/packages/populace-build/src/populace/build/us/country_package.json b/packages/populace-build/src/populace/build/us/country_package.json index af3ecc3d..7fed91f1 100644 --- a/packages/populace-build/src/populace/build/us/country_package.json +++ b/packages/populace-build/src/populace/build/us/country_package.json @@ -6,6 +6,7 @@ "fiscal_target_references.json", "obbba_reforms.json", "puf_aggregate_record_disaggregation.json", + "soi_baseline_levels.json", "source_stages.json", "support_spine.json", "tax_expenditure_reforms.json" diff --git a/packages/populace-build/src/populace/build/us/soi_baseline_levels.json b/packages/populace-build/src/populace/build/us/soi_baseline_levels.json new file mode 100644 index 00000000..02f801ac --- /dev/null +++ b/packages/populace-build/src/populace/build/us/soi_baseline_levels.json @@ -0,0 +1,128 @@ +{ + "_comment": "Out-of-sample baseline-level backtest: simulated baseline totals compared to published IRS SOI actuals for tax items that are NOT calibration targets (verified against the release target set — the calibrated national SOI concepts are income lines, EITC, and itemized components). Unlike the reform rows, these need no counterfactual: the populace value is the plain baseline weighted total of the variable at the dataset period, so all lines share one simulation. Benchmarks are tax-year actuals from the SOI Individual Complete Report (Publication 1304); the dataset period is 2024, so a one-year vintage gap (TY2023 actuals) plus population/income growth applies — populace should run slightly above the actuals.", + "schema_version": 1, + "source": { + "publication": "IRS Statistics of Income, Individual Complete Report (Publication 1304), Tax Year 2023 (rev. March 2026)", + "table_3_3": "https://www.irs.gov/pub/irs-soi/23in33ar.xls", + "table_1_4": "https://www.irs.gov/pub/irs-soi/23in14ar.xls" + }, + "levels": [ + { + "id": "soi_income_tax_net", + "name": "Total income tax (net of refundable credits)", + "variable": "income_tax", + "period": 2024, + "benchmark": { + "value": 2042047899000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (total income tax minus refundable credits)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "SOI total income tax minus refundable credits — the same net concept as populace's income_tax. Small known scope difference: the populace measure also includes qualified-retirement-plan penalty taxes and unreported payroll tax (~0.5% of the total), which SOI books outside income tax." + }, + { + "id": "soi_amt", + "name": "Alternative minimum tax", + "variable": "alternative_minimum_tax", + "period": 2024, + "benchmark": { + "value": 2752164000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 1.4 (alternative minimum tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls" + }, + "description": "Post-TCJA AMT is small (about 150,000 returns in TY2023), so this line is sensitive to the top tail of the income distribution." + }, + { + "id": "soi_cdcc", + "name": "Child and dependent care credit", + "variable": "cdcc", + "period": 2024, + "benchmark": { + "value": 3468910000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (credit for child and dependent care)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_education_credits", + "name": "Education credits (nonrefundable)", + "variable": "education_tax_credits", + "period": 2024, + "benchmark": { + "value": 7554668000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable education credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "populace's education_tax_credits is the nonrefundable American Opportunity credit plus the Lifetime Learning credit — the same scope as SOI's nonrefundable education credit column (the refundable AOTC portion is excluded on both sides)." + }, + { + "id": "soi_savers_credit", + "name": "Retirement savings contribution (saver's) credit", + "variable": "savers_credit", + "period": 2024, + "benchmark": { + "value": 2037572000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (retirement savings contribution credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_ctc_nonrefundable", + "name": "Child tax credit + ODC (nonrefundable)", + "variable": "non_refundable_ctc", + "period": 2024, + "benchmark": { + "value": 84418366000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable child and other dependent tax credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_ctc_refundable", + "name": "Additional child tax credit (refundable)", + "variable": "refundable_ctc", + "period": 2024, + "benchmark": { + "value": 34533251000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (refundable child tax credit or additional child tax credit)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + }, + { + "id": "soi_niit", + "name": "Net investment income tax", + "variable": "net_investment_income_tax", + "period": 2024, + "benchmark": { + "value": 39330017000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (net investment income tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "Concentrated in top-decile capital income, so this line tests the investment-income tail." + }, + { + "id": "soi_se_tax", + "name": "Self-employment tax", + "variable": "self_employment_tax", + "period": 2024, + "benchmark": { + "value": 81087060000, + "year": "TY2023", + "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (self-employment tax)", + "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls" + }, + "description": "" + } + ] +} diff --git a/packages/populace-build/src/populace/build/us_runtime/reform_validation.py b/packages/populace-build/src/populace/build/us_runtime/reform_validation.py index 19b518ed..c62de7b3 100644 --- a/packages/populace-build/src/populace/build/us_runtime/reform_validation.py +++ b/packages/populace-build/src/populace/build/us_runtime/reform_validation.py @@ -38,10 +38,12 @@ __all__ = [ "REFORM_VALIDATION_SCHEMA_VERSION", + "BaselineLevelSpec", "ReformValidationSpec", "in_sample_reform_specs", "out_of_sample_reform_specs", "tax_expenditure_reform_specs", + "soi_baseline_level_specs", "load_default_reform_specs", "reform_validation_payload", "write_reform_validation", @@ -301,6 +303,63 @@ def tax_expenditure_reform_specs( return tuple(specs) +@dataclass(frozen=True) +class BaselineLevelSpec: + """A baseline total compared to a published actual (no counterfactual). + + Unlike a reform, a level row needs no reform simulation: the populace value + is the plain baseline weighted total of ``variable`` at ``period``, so + every level shares the one baseline simulation. Levels are only meaningful + as validation when the variable is NOT a calibration target — the shipped + config is restricted to SOI tax items outside the target set. + """ + + id: str + name: str + variable: str + period: int + benchmark_value: float + benchmark_year: str + source: str + source_url: str + description: str = "" + + def __post_init__(self) -> None: + if not self.id or not self.variable: + raise ValueError("BaselineLevelSpec requires id and variable.") + + +def _soi_baseline_levels_config_path() -> Path: + return Path(str(files("populace.build.us").joinpath("soi_baseline_levels.json"))) + + +def soi_baseline_level_specs( + path: Path | None = None, +) -> tuple[BaselineLevelSpec, ...]: + """Curated SOI-actual baseline levels from JSON config.""" + config_path = path or _soi_baseline_levels_config_path() + if not config_path.exists(): + return () + payload = json.loads(config_path.read_text()) + specs: list[BaselineLevelSpec] = [] + for raw in payload.get("levels", ()): + bench = raw.get("benchmark", {}) + specs.append( + BaselineLevelSpec( + id=raw["id"], + name=raw["name"], + variable=raw["variable"], + period=int(raw["period"]), + benchmark_value=float(bench["value"]), + benchmark_year=str(bench.get("year", "")), + source=str(bench.get("source", "")), + source_url=str(bench.get("source_url", "")), + description=str(raw.get("description", "")), + ) + ) + return tuple(specs) + + def load_default_reform_specs( *, period: int, @@ -348,6 +407,7 @@ def reform_validation_payload( simulate: SimulateFn | None = None, in_sample_estimates: dict[str, float] | None = None, in_sample_targets: dict[str, float] | None = None, + baseline_levels: Sequence[BaselineLevelSpec] = (), release_id: str | None = None, ) -> dict[str, Any]: """Score each reform on the dataset and render the JSON-stable payload. @@ -516,7 +576,43 @@ def simulated_effect( # genuinely-zero one, and so a release built with simulation skipped is # never mistaken for one where the dataset simply failed the fidelity test. # False only when out-of-sample reforms exist but no simulate() was given. - has_out_of_sample = any(not spec.in_sample for spec in specs) + # Baseline-level backtest rows (SOI actuals): no counterfactual — every + # level reads the shared baseline simulation, so together they cost one + # extra measure per line, not one simulation per line. + for level in baseline_levels: + total = ( + None if simulate is None else baseline_total(level.variable, level.period) + ) + rows.append( + { + "id": level.id, + "name": level.name, + "category": "IRS SOI actual", + "in_sample": False, + "period": level.period, + "description": level.description or None, + "jct": { + "score": _finite(level.benchmark_value), + "score_fy2027": None, + "score_type": "actual", + "window": level.benchmark_year or None, + "source": level.source or None, + "source_url": level.source_url or None, + }, + "populace": { + "budget_effect": None if total is None else _finite(total), + "period": level.period, + "window": level.benchmark_year or None, + "measure": level.variable, + "baseline_total": None if total is None else _finite(total), + "reform_total": None, + }, + } + ) + + has_out_of_sample = any(not spec.in_sample for spec in specs) or bool( + baseline_levels + ) out_of_sample_simulated = simulate is not None or not has_out_of_sample payload: dict[str, Any] = { "schema_version": REFORM_VALIDATION_SCHEMA_VERSION, diff --git a/packages/populace-build/tests/test_reform_validation.py b/packages/populace-build/tests/test_reform_validation.py index 419e71d1..c86427cc 100644 --- a/packages/populace-build/tests/test_reform_validation.py +++ b/packages/populace-build/tests/test_reform_validation.py @@ -478,3 +478,97 @@ def test_write_round_trips(tmp_path): payload = reform_validation_payload([_oos_spec(-1.0)], period=2024, simulate=None) path = write_reform_validation(payload, tmp_path / "reform_validation.json") assert json.loads(path.read_text())["reforms"][0]["id"] == "obbba_salt" + + +def test_soi_baseline_levels_load_from_default_config(): + from populace.build.us_runtime.reform_validation import soi_baseline_level_specs + + levels = soi_baseline_level_specs() + assert len(levels) >= 8 + ids = [lv.id for lv in levels] + assert len(ids) == len(set(ids)) + for lv in levels: + assert lv.benchmark_value > 0 + assert lv.benchmark_year.startswith("TY") + assert "SOI" in lv.source + assert lv.source_url + # Levels must be OUT of the calibration target set; the calibrated + # national SOI concepts are income lines, EITC, and itemized + # components — none of these variables. + assert lv.variable not in { + "adjusted_gross_income", + "eitc", + "taxable_income", + "itemized_taxable_income_deductions", + } + income_tax = next(lv for lv in levels if lv.id == "soi_income_tax_net") + assert income_tax.benchmark_value == pytest.approx(2_042_047_899_000) + + +def test_baseline_levels_share_one_simulation_and_emit_rows(): + from populace.build.us_runtime.reform_validation import BaselineLevelSpec + + calls = [] + + def simulate(reform): + calls.append(reform) + return _FakeSim({"cdcc": 3.6e9, "savers_credit": 2.2e9}) + + levels = ( + BaselineLevelSpec( + id="soi_cdcc", + name="CDCC", + variable="cdcc", + period=2024, + benchmark_value=3.47e9, + benchmark_year="TY2023", + source="IRS SOI Pub 1304 TY2023, Table 3.3", + source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls", + ), + BaselineLevelSpec( + id="soi_savers", + name="Saver's credit", + variable="savers_credit", + period=2024, + benchmark_value=2.04e9, + benchmark_year="TY2023", + source="IRS SOI Pub 1304 TY2023, Table 3.3", + source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls", + ), + ) + payload = reform_validation_payload( + (), period=2024, simulate=simulate, baseline_levels=levels + ) + # Both levels read the one shared baseline simulation. + assert calls == [None] + rows = {row["id"]: row for row in payload["reforms"]} + cdcc = rows["soi_cdcc"] + assert cdcc["category"] == "IRS SOI actual" + assert cdcc["in_sample"] is False + assert cdcc["jct"]["score"] == pytest.approx(3.47e9) + assert cdcc["jct"]["score_type"] == "actual" + assert cdcc["jct"]["window"] == "TY2023" + assert cdcc["populace"]["budget_effect"] == pytest.approx(3.6e9) + assert rows["soi_savers"]["populace"]["budget_effect"] == pytest.approx(2.2e9) + assert payload["out_of_sample_simulated"] is True + + +def test_baseline_levels_null_without_simulate(): + from populace.build.us_runtime.reform_validation import BaselineLevelSpec + + level = BaselineLevelSpec( + id="soi_cdcc", + name="CDCC", + variable="cdcc", + period=2024, + benchmark_value=3.47e9, + benchmark_year="TY2023", + source="IRS SOI", + source_url="https://example.test", + ) + payload = reform_validation_payload( + (), period=2024, simulate=None, baseline_levels=(level,) + ) + assert payload["reforms"][0]["populace"]["budget_effect"] is None + # An unsimulated backtest must mark itself, same as skipped OBBBA rows. + assert payload["out_of_sample_simulated"] is False diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index f13d48da..440f3d46 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -65,6 +65,7 @@ default_simulate_factory, load_default_reform_specs, reform_validation_payload, + soi_baseline_level_specs, write_reform_validation, ) from populace.calibrate import TargetRegistry, calibrate, calibrate_l0_refit @@ -3883,6 +3884,7 @@ def _write_reform_validation( simulate=simulate, in_sample_estimates=_in_sample_estimates(result), in_sample_targets=_in_sample_targets(result), + baseline_levels=soi_baseline_level_specs(), release_id=release_id, ) write_reform_validation(payload, release_dir / "reform_validation.json")