Add SOI-actuals baseline backtest to reform validation

packages/populace-build/src/populace/build/us/country_package.json

@@ -6,6 +6,7 @@
     "fiscal_target_references.json",
     "obbba_reforms.json",
     "puf_aggregate_record_disaggregation.json",
+    "soi_baseline_levels.json",
     "source_stages.json",
     "support_spine.json",
     "tax_expenditure_reforms.json"

packages/populace-build/src/populace/build/us/soi_baseline_levels.json

@@ -0,0 +1,128 @@
+{
+  "_comment": "Out-of-sample baseline-level backtest: simulated baseline totals compared to published IRS SOI actuals for tax items that are NOT calibration targets (verified against the release target set — the calibrated national SOI concepts are income lines, EITC, and itemized components). Unlike the reform rows, these need no counterfactual: the populace value is the plain baseline weighted total of the variable at the dataset period, so all lines share one simulation. Benchmarks are tax-year actuals from the SOI Individual Complete Report (Publication 1304); the dataset period is 2024, so a one-year vintage gap (TY2023 actuals) plus population/income growth applies — populace should run slightly above the actuals.",
+  "schema_version": 1,
+  "source": {
+    "publication": "IRS Statistics of Income, Individual Complete Report (Publication 1304), Tax Year 2023 (rev. March 2026)",
+    "table_3_3": "https://www.irs.gov/pub/irs-soi/23in33ar.xls",
+    "table_1_4": "https://www.irs.gov/pub/irs-soi/23in14ar.xls"
+  },
+  "levels": [
+    {
+      "id": "soi_income_tax_net",
+      "name": "Total income tax (net of refundable credits)",
+      "variable": "income_tax",
+      "period": 2024,
+      "benchmark": {
+        "value": 2042047899000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (total income tax minus refundable credits)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": "SOI total income tax minus refundable credits — the same net concept as populace's income_tax. Small known scope difference: the populace measure also includes qualified-retirement-plan penalty taxes and unreported payroll tax (~0.5% of the total), which SOI books outside income tax."
+    },
+    {
+      "id": "soi_amt",
+      "name": "Alternative minimum tax",
+      "variable": "alternative_minimum_tax",
+      "period": 2024,
+      "benchmark": {
+        "value": 2752164000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 1.4 (alternative minimum tax)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls"
+      },
+      "description": "Post-TCJA AMT is small (about 150,000 returns in TY2023), so this line is sensitive to the top tail of the income distribution."
+    },
+    {
+      "id": "soi_cdcc",
+      "name": "Child and dependent care credit",
+      "variable": "cdcc",
+      "period": 2024,
+      "benchmark": {
+        "value": 3468910000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (credit for child and dependent care)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": ""
+    },
+    {
+      "id": "soi_education_credits",
+      "name": "Education credits (nonrefundable)",
+      "variable": "education_tax_credits",
+      "period": 2024,
+      "benchmark": {
+        "value": 7554668000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable education credit)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": "populace's education_tax_credits is the nonrefundable American Opportunity credit plus the Lifetime Learning credit — the same scope as SOI's nonrefundable education credit column (the refundable AOTC portion is excluded on both sides)."
+    },
+    {
+      "id": "soi_savers_credit",
+      "name": "Retirement savings contribution (saver's) credit",
+      "variable": "savers_credit",
+      "period": 2024,
+      "benchmark": {
+        "value": 2037572000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (retirement savings contribution credit)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": ""
+    },
+    {
+      "id": "soi_ctc_nonrefundable",
+      "name": "Child tax credit + ODC (nonrefundable)",
+      "variable": "non_refundable_ctc",
+      "period": 2024,
+      "benchmark": {
+        "value": 84418366000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (nonrefundable child and other dependent tax credit)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": ""
+    },
+    {
+      "id": "soi_ctc_refundable",
+      "name": "Additional child tax credit (refundable)",
+      "variable": "refundable_ctc",
+      "period": 2024,
+      "benchmark": {
+        "value": 34533251000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (refundable child tax credit or additional child tax credit)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": ""
+    },
+    {
+      "id": "soi_niit",
+      "name": "Net investment income tax",
+      "variable": "net_investment_income_tax",
+      "period": 2024,
+      "benchmark": {
+        "value": 39330017000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (net investment income tax)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": "Concentrated in top-decile capital income, so this line tests the investment-income tail."
+    },
+    {
+      "id": "soi_se_tax",
+      "name": "Self-employment tax",
+      "variable": "self_employment_tax",
+      "period": 2024,
+      "benchmark": {
+        "value": 81087060000,
+        "year": "TY2023",
+        "source": "IRS SOI Pub 1304 TY2023, Table 3.3 (self-employment tax)",
+        "source_url": "https://www.irs.gov/pub/irs-soi/23in33ar.xls"
+      },
+      "description": ""
+    }
+  ]
+}

packages/populace-build/src/populace/build/us_runtime/reform_validation.py

@@ -38,10 +38,12 @@
 
 __all__ = [
     "REFORM_VALIDATION_SCHEMA_VERSION",
+    "BaselineLevelSpec",
     "ReformValidationSpec",
     "in_sample_reform_specs",
     "out_of_sample_reform_specs",
     "tax_expenditure_reform_specs",
+    "soi_baseline_level_specs",
     "load_default_reform_specs",
     "reform_validation_payload",
     "write_reform_validation",
@@ -301,6 +303,63 @@ def tax_expenditure_reform_specs(
     return tuple(specs)
 
 
+@dataclass(frozen=True)
+class BaselineLevelSpec:
+    """A baseline total compared to a published actual (no counterfactual).
+
+    Unlike a reform, a level row needs no reform simulation: the populace value
+    is the plain baseline weighted total of ``variable`` at ``period``, so
+    every level shares the one baseline simulation. Levels are only meaningful
+    as validation when the variable is NOT a calibration target — the shipped
+    config is restricted to SOI tax items outside the target set.
+    """
+
+    id: str
+    name: str
+    variable: str
+    period: int
+    benchmark_value: float
+    benchmark_year: str
+    source: str
+    source_url: str
+    description: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.id or not self.variable:
+            raise ValueError("BaselineLevelSpec requires id and variable.")
+
+
+def _soi_baseline_levels_config_path() -> Path:
+    return Path(str(files("populace.build.us").joinpath("soi_baseline_levels.json")))
+
+
+def soi_baseline_level_specs(
+    path: Path | None = None,
+) -> tuple[BaselineLevelSpec, ...]:
+    """Curated SOI-actual baseline levels from JSON config."""
+    config_path = path or _soi_baseline_levels_config_path()
+    if not config_path.exists():
+        return ()
+    payload = json.loads(config_path.read_text())
+    specs: list[BaselineLevelSpec] = []
+    for raw in payload.get("levels", ()):
+        bench = raw.get("benchmark", {})
+        specs.append(
+            BaselineLevelSpec(
+                id=raw["id"],
+                name=raw["name"],
+                variable=raw["variable"],
+                period=int(raw["period"]),
+                benchmark_value=float(bench["value"]),
+                benchmark_year=str(bench.get("year", "")),
+                source=str(bench.get("source", "")),
+                source_url=str(bench.get("source_url", "")),
+                description=str(raw.get("description", "")),
+            )
+        )
+    return tuple(specs)
+
+
 def load_default_reform_specs(
     *,
     period: int,
@@ -348,6 +407,7 @@ def reform_validation_payload(
     simulate: SimulateFn | None = None,
     in_sample_estimates: dict[str, float] | None = None,
     in_sample_targets: dict[str, float] | None = None,
+    baseline_levels: Sequence[BaselineLevelSpec] = (),
     release_id: str | None = None,
 ) -> dict[str, Any]:
     """Score each reform on the dataset and render the JSON-stable payload.
@@ -516,7 +576,43 @@ def simulated_effect(
     # genuinely-zero one, and so a release built with simulation skipped is
     # never mistaken for one where the dataset simply failed the fidelity test.
     # False only when out-of-sample reforms exist but no simulate() was given.
-    has_out_of_sample = any(not spec.in_sample for spec in specs)
+    # Baseline-level backtest rows (SOI actuals): no counterfactual — every
+    # level reads the shared baseline simulation, so together they cost one
+    # extra measure per line, not one simulation per line.
+    for level in baseline_levels:
+        total = (
+            None if simulate is None else baseline_total(level.variable, level.period)
+        )
+        rows.append(
+            {
+                "id": level.id,
+                "name": level.name,
+                "category": "IRS SOI actual",
+                "in_sample": False,
+                "period": level.period,
+                "description": level.description or None,
+                "jct": {
+                    "score": _finite(level.benchmark_value),
+                    "score_fy2027": None,
+                    "score_type": "actual",
+                    "window": level.benchmark_year or None,
+                    "source": level.source or None,
+                    "source_url": level.source_url or None,
+                },
+                "populace": {
+                    "budget_effect": None if total is None else _finite(total),
+                    "period": level.period,
+                    "window": level.benchmark_year or None,
+                    "measure": level.variable,
+                    "baseline_total": None if total is None else _finite(total),
+                    "reform_total": None,
+                },
+            }
+        )
+
+    has_out_of_sample = any(not spec.in_sample for spec in specs) or bool(
+        baseline_levels
+    )
     out_of_sample_simulated = simulate is not None or not has_out_of_sample
     payload: dict[str, Any] = {
         "schema_version": REFORM_VALIDATION_SCHEMA_VERSION,

packages/populace-build/tests/test_reform_validation.py

@@ -478,3 +478,97 @@ def test_write_round_trips(tmp_path):
     payload = reform_validation_payload([_oos_spec(-1.0)], period=2024, simulate=None)
     path = write_reform_validation(payload, tmp_path / "reform_validation.json")
     assert json.loads(path.read_text())["reforms"][0]["id"] == "obbba_salt"
+
+
+def test_soi_baseline_levels_load_from_default_config():
+    from populace.build.us_runtime.reform_validation import soi_baseline_level_specs
+
+    levels = soi_baseline_level_specs()
+    assert len(levels) >= 8
+    ids = [lv.id for lv in levels]
+    assert len(ids) == len(set(ids))
+    for lv in levels:
+        assert lv.benchmark_value > 0
+        assert lv.benchmark_year.startswith("TY")
+        assert "SOI" in lv.source
+        assert lv.source_url
+        # Levels must be OUT of the calibration target set; the calibrated
+        # national SOI concepts are income lines, EITC, and itemized
+        # components — none of these variables.
+        assert lv.variable not in {
+            "adjusted_gross_income",
+            "eitc",
+            "taxable_income",
+            "itemized_taxable_income_deductions",
+        }
+    income_tax = next(lv for lv in levels if lv.id == "soi_income_tax_net")
+    assert income_tax.benchmark_value == pytest.approx(2_042_047_899_000)
+
+
+def test_baseline_levels_share_one_simulation_and_emit_rows():
+    from populace.build.us_runtime.reform_validation import BaselineLevelSpec
+
+    calls = []
+
+    def simulate(reform):
+        calls.append(reform)
+        return _FakeSim({"cdcc": 3.6e9, "savers_credit": 2.2e9})
+
+    levels = (
+        BaselineLevelSpec(
+            id="soi_cdcc",
+            name="CDCC",
+            variable="cdcc",
+            period=2024,
+            benchmark_value=3.47e9,
+            benchmark_year="TY2023",
+            source="IRS SOI Pub 1304 TY2023, Table 3.3",
+            source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls",
+        ),
+        BaselineLevelSpec(
+            id="soi_savers",
+            name="Saver's credit",
+            variable="savers_credit",
+            period=2024,
+            benchmark_value=2.04e9,
+            benchmark_year="TY2023",
+            source="IRS SOI Pub 1304 TY2023, Table 3.3",
+            source_url="https://www.irs.gov/pub/irs-soi/23in33ar.xls",
+        ),
+    )
+    payload = reform_validation_payload(
+        (), period=2024, simulate=simulate, baseline_levels=levels
+    )
+    # Both levels read the one shared baseline simulation.
+    assert calls == [None]
+    rows = {row["id"]: row for row in payload["reforms"]}
+    cdcc = rows["soi_cdcc"]
+    assert cdcc["category"] == "IRS SOI actual"
+    assert cdcc["in_sample"] is False
+    assert cdcc["jct"]["score"] == pytest.approx(3.47e9)
+    assert cdcc["jct"]["score_type"] == "actual"
+    assert cdcc["jct"]["window"] == "TY2023"
+    assert cdcc["populace"]["budget_effect"] == pytest.approx(3.6e9)
+    assert rows["soi_savers"]["populace"]["budget_effect"] == pytest.approx(2.2e9)
+    assert payload["out_of_sample_simulated"] is True
+
+
+def test_baseline_levels_null_without_simulate():
+    from populace.build.us_runtime.reform_validation import BaselineLevelSpec
+
+    level = BaselineLevelSpec(
+        id="soi_cdcc",
+        name="CDCC",
+        variable="cdcc",
+        period=2024,
+        benchmark_value=3.47e9,
+        benchmark_year="TY2023",
+        source="IRS SOI",
+        source_url="https://example.test",
+    )
+    payload = reform_validation_payload(
+        (), period=2024, simulate=None, baseline_levels=(level,)
+    )
+    assert payload["reforms"][0]["populace"]["budget_effect"] is None
+    # An unsimulated backtest must mark itself, same as skipped OBBBA rows.
+    assert payload["out_of_sample_simulated"] is False

tools/build_us_fiscal_refresh_release.py

@@ -65,6 +65,7 @@
     default_simulate_factory,
     load_default_reform_specs,
     reform_validation_payload,
+    soi_baseline_level_specs,
     write_reform_validation,
 )
 from populace.calibrate import TargetRegistry, calibrate, calibrate_l0_refit
@@ -3883,6 +3884,7 @@ def _write_reform_validation(
         simulate=simulate,
         in_sample_estimates=_in_sample_estimates(result),
         in_sample_targets=_in_sample_targets(result),
+        baseline_levels=soi_baseline_level_specs(),
         release_id=release_id,
     )
     write_reform_validation(payload, release_dir / "reform_validation.json")