From 3f64079ee299a701b5f0a35dffce23f51bc8b245 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 08:30:27 -0400 Subject: [PATCH 01/18] Record the source-row evidence for the E19200 split The interim national carve cannot support a per-AGI-band decomposition unless every carried component is independently traceable. Capture TY2015 SOI Table 2.1 rows 10-32, all five interest-paid amount columns, band boundaries, official URLs, and workbook/report coordinates as a packaged source asset. This preserves the published four-part identity and makes explicit that the modeled non-mortgage residual includes points and mortgage-insurance premiums as well as investment interest. Co-Authored-By: Claude Fable 5 --- ..._table_2_1_interest_components_ty2015.json | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json diff --git a/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json b/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json new file mode 100644 index 00000000..8affd3e1 --- /dev/null +++ b/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json @@ -0,0 +1,288 @@ +{ + "source": { + "title": "Individual Income Tax Returns Complete Report, Publication 1304, Tax Year 2015, Table 2.1", + "report_url": "https://www.irs.gov/pub/irs-soi/15inalcr.pdf", + "workbook_url": "https://www.irs.gov/pub/irs-soi/15in21id.xls", + "table": "2.1", + "tax_year": 2015, + "units": "thousands_of_us_dollars", + "columns": { + "total_interest_paid_amount": { + "printed_column": 83, + "workbook_column": "CF" + }, + "home_mortgage_interest_amount": { + "printed_column": 85, + "workbook_column": "CH" + }, + "deductible_points_amount": { + "printed_column": 91, + "workbook_column": "CN" + }, + "qualified_mortgage_insurance_premiums_amount": { + "printed_column": 93, + "workbook_column": "CP" + }, + "investment_interest_amount": { + "printed_column": 95, + "workbook_column": "CR" + } + }, + "report_pages": { + "total_home_points_and_mortgage_insurance": 100, + "investment_interest": 101 + } + }, + "all_returns": { + "source_row": 10, + "total_interest_paid_amount": 304461163, + "home_mortgage_interest_amount": 283004465, + "deductible_points_amount": 1273716, + "qualified_mortgage_insurance_premiums_amount": 6287486, + "investment_interest_amount": 13895495 + }, + "agi_bands": [ + { + "source_row": 11, + "label": "Under $5,000", + "lower_bound": null, + "upper_bound": 5000, + "total_interest_paid_amount": 1231416, + "home_mortgage_interest_amount": 1174521, + "deductible_points_amount": 1459, + "qualified_mortgage_insurance_premiums_amount": 36879, + "investment_interest_amount": 18556 + }, + { + "source_row": 12, + "label": "$5,000 under $10,000", + "lower_bound": 5000, + "upper_bound": 10000, + "total_interest_paid_amount": 1284063, + "home_mortgage_interest_amount": 1237451, + "deductible_points_amount": 871, + "qualified_mortgage_insurance_premiums_amount": 35605, + "investment_interest_amount": 10135 + }, + { + "source_row": 13, + "label": "$10,000 under $15,000", + "lower_bound": 10000, + "upper_bound": 15000, + "total_interest_paid_amount": 2148799, + "home_mortgage_interest_amount": 2083827, + "deductible_points_amount": 3523, + "qualified_mortgage_insurance_premiums_amount": 56173, + "investment_interest_amount": 5276 + }, + { + "source_row": 14, + "label": "$15,000 under $20,000", + "lower_bound": 15000, + "upper_bound": 20000, + "total_interest_paid_amount": 2699978, + "home_mortgage_interest_amount": 2589295, + "deductible_points_amount": 12649, + "qualified_mortgage_insurance_premiums_amount": 93731, + "investment_interest_amount": 4303 + }, + { + "source_row": 15, + "label": "$20,000 under $25,000", + "lower_bound": 20000, + "upper_bound": 25000, + "total_interest_paid_amount": 3181401, + "home_mortgage_interest_amount": 2992991, + "deductible_points_amount": 16428, + "qualified_mortgage_insurance_premiums_amount": 148030, + "investment_interest_amount": 23951 + }, + { + "source_row": 16, + "label": "$25,000 under $30,000", + "lower_bound": 25000, + "upper_bound": 30000, + "total_interest_paid_amount": 3748228, + "home_mortgage_interest_amount": 3591380, + "deductible_points_amount": 13066, + "qualified_mortgage_insurance_premiums_amount": 136509, + "investment_interest_amount": 7273 + }, + { + "source_row": 17, + "label": "$30,000 under $35,000", + "lower_bound": 30000, + "upper_bound": 35000, + "total_interest_paid_amount": 4214763, + "home_mortgage_interest_amount": 3946351, + "deductible_points_amount": 16300, + "qualified_mortgage_insurance_premiums_amount": 246026, + "investment_interest_amount": 6085 + }, + { + "source_row": 18, + "label": "$35,000 under $40,000", + "lower_bound": 35000, + "upper_bound": 40000, + "total_interest_paid_amount": 5435778, + "home_mortgage_interest_amount": 5122692, + "deductible_points_amount": 10850, + "qualified_mortgage_insurance_premiums_amount": 289043, + "investment_interest_amount": 13193 + }, + { + "source_row": 19, + "label": "$40,000 under $45,000", + "lower_bound": 40000, + "upper_bound": 45000, + "total_interest_paid_amount": 6009927, + "home_mortgage_interest_amount": 5651139, + "deductible_points_amount": 22472, + "qualified_mortgage_insurance_premiums_amount": 322044, + "investment_interest_amount": 14272 + }, + { + "source_row": 20, + "label": "$45,000 under $50,000", + "lower_bound": 45000, + "upper_bound": 50000, + "total_interest_paid_amount": 6888540, + "home_mortgage_interest_amount": 6411648, + "deductible_points_amount": 36417, + "qualified_mortgage_insurance_premiums_amount": 430176, + "investment_interest_amount": 10299 + }, + { + "source_row": 21, + "label": "$50,000 under $55,000", + "lower_bound": 50000, + "upper_bound": 55000, + "total_interest_paid_amount": 6955789, + "home_mortgage_interest_amount": 6543517, + "deductible_points_amount": 21952, + "qualified_mortgage_insurance_premiums_amount": 378557, + "investment_interest_amount": 11762 + }, + { + "source_row": 22, + "label": "$55,000 under $60,000", + "lower_bound": 55000, + "upper_bound": 60000, + "total_interest_paid_amount": 7424513, + "home_mortgage_interest_amount": 6985554, + "deductible_points_amount": 35610, + "qualified_mortgage_insurance_premiums_amount": 387597, + "investment_interest_amount": 15752 + }, + { + "source_row": 23, + "label": "$60,000 under $75,000", + "lower_bound": 60000, + "upper_bound": 75000, + "total_interest_paid_amount": 23294604, + "home_mortgage_interest_amount": 21825640, + "deductible_points_amount": 111241, + "qualified_mortgage_insurance_premiums_amount": 1272959, + "investment_interest_amount": 84765 + }, + { + "source_row": 24, + "label": "$75,000 under $100,000", + "lower_bound": 75000, + "upper_bound": 100000, + "total_interest_paid_amount": 43398259, + "home_mortgage_interest_amount": 40897449, + "deductible_points_amount": 238507, + "qualified_mortgage_insurance_premiums_amount": 2096979, + "investment_interest_amount": 165323 + }, + { + "source_row": 25, + "label": "$100,000 under $200,000", + "lower_bound": 100000, + "upper_bound": 200000, + "total_interest_paid_amount": 104846625, + "home_mortgage_interest_amount": 103319155, + "deductible_points_amount": 466821, + "qualified_mortgage_insurance_premiums_amount": 357177, + "investment_interest_amount": 703472 + }, + { + "source_row": 26, + "label": "$200,000 under $500,000", + "lower_bound": 200000, + "upper_bound": 500000, + "total_interest_paid_amount": 53562301, + "home_mortgage_interest_amount": 51606382, + "deductible_points_amount": 206890, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1749030 + }, + { + "source_row": 27, + "label": "$500,000 under $1,000,000", + "lower_bound": 500000, + "upper_bound": 1000000, + "total_interest_paid_amount": 12836968, + "home_mortgage_interest_amount": 11335153, + "deductible_points_amount": 40960, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1460856 + }, + { + "source_row": 28, + "label": "$1,000,000 under $1,500,000", + "lower_bound": 1000000, + "upper_bound": 1500000, + "total_interest_paid_amount": 3505209, + "home_mortgage_interest_amount": 2559360, + "deductible_points_amount": 7938, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 937911 + }, + { + "source_row": 29, + "label": "$1,500,000 under $2,000,000", + "lower_bound": 1500000, + "upper_bound": 2000000, + "total_interest_paid_amount": 1666807, + "home_mortgage_interest_amount": 1043154, + "deductible_points_amount": 2635, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 621017 + }, + { + "source_row": 30, + "label": "$2,000,000 under $5,000,000", + "lower_bound": 2000000, + "upper_bound": 5000000, + "total_interest_paid_amount": 3412493, + "home_mortgage_interest_amount": 1521570, + "deductible_points_amount": 5102, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1885822 + }, + { + "source_row": 31, + "label": "$5,000,000 under $10,000,000", + "lower_bound": 5000000, + "upper_bound": 10000000, + "total_interest_paid_amount": 1535736, + "home_mortgage_interest_amount": 360807, + "deductible_points_amount": 1387, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1173542 + }, + { + "source_row": 32, + "label": "$10,000,000 or more", + "lower_bound": 10000000, + "upper_bound": null, + "total_interest_paid_amount": 5178966, + "home_mortgage_interest_amount": 205431, + "deductible_points_amount": 637, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 4972898 + } + ] +} From 4745f6fda5f5082c7dfaef9ee21f5a1dba04c41c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 08:33:44 -0400 Subject: [PATCH 02/18] Specify the per-band E19200 decomposition contract A proportional split is only safe if it preserves every donor record and every published band while routing the full non-mortgage residual exactly once. Add RED tests for the complete source-row identity, the open-ended negative-AGI first band, nonzero residual mass, explicit AGI input, deterministic order-independent bits, and exact E19200 conservation. The tests currently fail at collection because the decomposition module is intentionally not implemented yet. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_puf_e19200_decomposition.py | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 packages/populace-build/tests/test_us_puf_e19200_decomposition.py diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py new file mode 100644 index 00000000..0e540f27 --- /dev/null +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from populace.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_AGI_BANDS, + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + split_us_puf_e19200_by_agi_band, +) +from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays + + +def _representative_agi(lower: float | None, upper: float | None) -> float: + if lower is None: + return -10_000.0 + if upper is None: + return lower + 1_000_000.0 + return (lower + upper) / 2 + + +def _one_record_per_band_arrays() -> tuple[dict[str, list[object]], np.ndarray]: + tax_unit_ids = np.arange(1, len(US_PUF_E19200_AGI_BANDS) + 1, dtype=np.int64) + total_interest = np.asarray( + [band.total_interest_paid_amount for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ) + adjusted_gross_income = np.asarray( + [ + _representative_agi(band.lower_bound, band.upper_bound) + for band in US_PUF_E19200_AGI_BANDS + ], + dtype=np.float64, + ) + arrays: dict[str, list[object]] = { + "tax_unit_id": tax_unit_ids.tolist(), + "household_weight": np.ones(len(tax_unit_ids)).tolist(), + "filing_status": [b"SINGLE"] * len(tax_unit_ids), + "person_tax_unit_id": tax_unit_ids.tolist(), + "home_mortgage_interest": total_interest.tolist(), + # The processed PUF leaf is all-zero before this decomposition. The + # split must replace it from E19200, not preserve or add this sentinel. + "investment_interest_expense": np.zeros(len(tax_unit_ids)).tolist(), + } + return arrays, adjusted_gross_income + + +def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> None: + bands = US_PUF_E19200_AGI_BANDS + + assert len(bands) == 22 + assert [band.source_row for band in bands] == list(range(11, 33)) + assert bands[0].label == "Under $5,000" + assert bands[0].lower_bound is None + assert bands[0].upper_bound == 5_000 + assert bands[-1].lower_bound == 10_000_000 + assert bands[-1].upper_bound is None + for previous, following in zip(bands, bands[1:], strict=True): + assert previous.upper_bound == following.lower_bound + for band in bands: + assert band.source_cells == { + "total_interest_paid_amount": f"CF{band.source_row}", + "home_mortgage_interest_amount": f"CH{band.source_row}", + "deductible_points_amount": f"CN{band.source_row}", + "qualified_mortgage_insurance_premiums_amount": ( + f"CP{band.source_row}" + ), + "investment_interest_amount": f"CR{band.source_row}", + } + published_components = ( + band.home_mortgage_interest_amount + + band.deductible_points_amount + + band.qualified_mortgage_insurance_premiums_amount + + band.investment_interest_amount + ) + assert abs(band.total_interest_paid_amount - published_components) <= 1 + + assert sum(band.total_interest_paid_amount for band in bands) == 304_461_163 + assert sum(band.home_mortgage_interest_amount for band in bands) == 283_004_467 + assert sum(band.non_mortgage_interest_amount for band in bands) == 21_456_696 + assert sum(band.investment_interest_amount for band in bands) == 13_895_493 + + all_returns = US_PUF_E19200_ALL_RETURNS_COMPONENTS + assert all_returns.source_row == 10 + assert all_returns.total_interest_paid_amount == 304_461_163 + assert all_returns.home_mortgage_interest_amount == 283_004_465 + assert all_returns.deductible_points_amount == 1_273_716 + assert all_returns.qualified_mortgage_insurance_premiums_amount == 6_287_486 + assert all_returns.investment_interest_amount == 13_895_495 + + +def test_e19200_first_band_includes_negative_zero_and_below_5000_agi() -> None: + values = np.ones(4, dtype=np.float64) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + values, + np.asarray([-100_000.0, 0.0, 4_999.99, 5_000.0]), + ) + first_share = ( + US_PUF_E19200_AGI_BANDS[0].home_mortgage_interest_amount + / US_PUF_E19200_AGI_BANDS[0].total_interest_paid_amount + ) + second_share = ( + US_PUF_E19200_AGI_BANDS[1].home_mortgage_interest_amount + / US_PUF_E19200_AGI_BANDS[1].total_interest_paid_amount + ) + + np.testing.assert_array_equal( + mortgage[:3].view(np.uint64), + np.full(3, first_share, dtype=np.float64).view(np.uint64), + ) + assert mortgage[3] == second_share + np.testing.assert_array_equal(mortgage + non_mortgage, values) + + +def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: + arrays, adjusted_gross_income = _one_record_per_band_arrays() + + donor = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + total = np.asarray( + [band.total_interest_paid_amount for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ) + expected_mortgage = np.asarray( + [band.home_mortgage_interest_amount for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ) + expected_non_mortgage = total - expected_mortgage + np.testing.assert_allclose(donor["home_mortgage_interest"], expected_mortgage) + np.testing.assert_allclose( + donor["investment_interest_expense"], + expected_non_mortgage, + ) + np.testing.assert_array_equal( + donor["home_mortgage_interest"].to_numpy() + + donor["investment_interest_expense"].to_numpy(), + total, + ) + assert donor["investment_interest_expense"].sum() == 21_456_696 + assert (donor["investment_interest_expense"] > 0).all() + + +def test_e19200_donor_split_requires_explicit_adjusted_gross_income() -> None: + arrays = { + "tax_unit_id": [1], + "household_weight": [1.0], + "filing_status": [b"SINGLE"], + "person_tax_unit_id": [1], + "home_mortgage_interest": [100.0], + "investment_interest_expense": [0.0], + # This predictor is deliberately tempting but is not AGI and cannot + # silently select a source-table band. + "employment_income": [250_000.0], + } + + with pytest.raises(ValueError, match="adjusted_gross_income"): + puf_tax_unit_donor_from_arrays( + arrays, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + +def test_e19200_donor_split_is_bit_deterministic_and_order_invariant() -> None: + arrays, adjusted_gross_income = _one_record_per_band_arrays() + first = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + second = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + permutation = np.arange(len(adjusted_gross_income) - 1, -1, -1) + shuffled_arrays = { + name: np.asarray(values)[permutation].tolist() + for name, values in arrays.items() + } + shuffled = puf_tax_unit_donor_from_arrays( + shuffled_arrays, + adjusted_gross_income=adjusted_gross_income[permutation], + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ).sort_values("tax_unit_id") + + columns = ("home_mortgage_interest", "investment_interest_expense") + for column in columns: + np.testing.assert_array_equal( + first[column].to_numpy().view(np.uint64), + second[column].to_numpy().view(np.uint64), + ) + np.testing.assert_array_equal( + first[column].to_numpy().view(np.uint64), + shuffled[column].to_numpy().view(np.uint64), + ) From dd88f3a858d384ce1983a543f74cf7614349a30a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 08:52:29 -0400 Subject: [PATCH 03/18] Decompose each PUF E19200 record by its SOI AGI band MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The national carve discarded the non-mortgage residual and left the broader-interest and HMID measures structurally identical. Load the row-cited TY2015 component table, choose a deterministic mortgage share from each donor record’s explicit tax-unit AGI, preserve raw E19200 as mortgage plus the modeled non-mortgage residual, and persist that residual through PolicyEngine’s investment_interest_expense leaf. Keep the grouped-raw $10M screen ahead of the split and apply one share to every mortgage-lineage leg. The base builder now calculates aligned AGI from the pinned processed PUF and releases the transient engine simulation before QRF. Adding the new person target changes the production target-order digest, while schema v4 also invalidates custom-order standalone checkpoints whose manifests do not fingerprint donor construction. Focused decomposition, support, base-builder, and QRF tests pass. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/us_runtime/__init__.py | 14 +- .../us_runtime/puf_interest_components.py | 211 ++++++++++++++++++ .../build/us_runtime/puf_qrf_chain.py | 20 +- .../populace/build/us_runtime/puf_support.py | 117 ++++++++-- .../tests/test_puf_qrf_chain.py | 15 +- .../tests/test_us_puf_e19200_decomposition.py | 32 ++- .../tests/test_us_puf_support.py | 72 ++++-- .../tests/test_us_puf_support_base_builder.py | 48 +++- tools/build_us_puf_support_base.py | 62 ++++- 9 files changed, 519 insertions(+), 72 deletions(-) create mode 100644 packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 90cd93be..afffd15b 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -495,13 +495,19 @@ us_prior_year_income_summary, with_us_prior_year_income_inputs, ) +from populace.build.us_runtime.puf_interest_components import ( + PufE19200AgiBand, + PufE19200InterestComponents, + US_PUF_E19200_AGI_BANDS, + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + split_us_puf_e19200_by_agi_band, +) from populace.build.us_runtime.puf_support import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_E19200_HOME_MORTGAGE_SHARE, US_PUF_SUPPORT_FIT_NAME, US_PUF_SUPPORT_STAGE_NAME, PufTaxDetailChainInputs, @@ -1688,8 +1694,11 @@ "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", "PUF_TAX_DETAIL_SUPPORT_CHANNEL", "PufTaxDetailChainInputs", + "PufE19200AgiBand", + "PufE19200InterestComponents", "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", - "US_PUF_E19200_HOME_MORTGAGE_SHARE", + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", "US_PUF_SUPPORT_FIT_NAME", "US_PUF_SUPPORT_STAGE_NAME", "US_STATE_INCOME_TAX_TARGET_SPECS", @@ -1721,6 +1730,7 @@ "prepare_us_puf_tax_detail_chain_inputs", "reform_validation_payload", "source_gap_family_ids", + "split_us_puf_e19200_by_agi_band", "ECPS_PARITY_KNOWN_GAPS_RESOURCE", "ECPS_PARITY_REFERENCE_RESOURCE", "EcpsParityReference", diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py new file mode 100644 index 00000000..2866fec1 --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -0,0 +1,211 @@ +"""Published SOI component shares for the PUF E19200 interest deduction.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib.resources import files +from typing import Any + +import numpy as np + +_SOURCE_ASSET = "soi_table_2_1_interest_components_ty2015.json" +_AMOUNT_COLUMNS = { + "total_interest_paid_amount": "CF", + "home_mortgage_interest_amount": "CH", + "deductible_points_amount": "CN", + "qualified_mortgage_insurance_premiums_amount": "CP", + "investment_interest_amount": "CR", +} + + +@dataclass(frozen=True) +class PufE19200InterestComponents: + """One published SOI Table 2.1 interest-paid component row. + + Amounts retain the source table's unit, thousands of US dollars. The + ratios used to decompose donor records are therefore unitless. + """ + + source_row: int + total_interest_paid_amount: int + home_mortgage_interest_amount: int + deductible_points_amount: int + qualified_mortgage_insurance_premiums_amount: int + investment_interest_amount: int + + @property + def non_mortgage_interest_amount(self) -> int: + """Return the conserving residual represented by the engine leaf.""" + + return self.total_interest_paid_amount - self.home_mortgage_interest_amount + + @property + def source_cells(self) -> dict[str, str]: + """Return the literal official-workbook cell for every carried amount.""" + + return { + name: f"{column}{self.source_row}" + for name, column in _AMOUNT_COLUMNS.items() + } + + +@dataclass(frozen=True) +class PufE19200AgiBand(PufE19200InterestComponents): + """One half-open adjusted-gross-income band and its component amounts.""" + + label: str + lower_bound: float | None + upper_bound: float | None + + @property + def home_mortgage_share(self) -> float: + """Return the published mortgage share of total E19200.""" + + return self.home_mortgage_interest_amount / self.total_interest_paid_amount + + +def _components(raw: dict[str, Any]) -> PufE19200InterestComponents: + return PufE19200InterestComponents( + source_row=int(raw["source_row"]), + total_interest_paid_amount=int(raw["total_interest_paid_amount"]), + home_mortgage_interest_amount=int(raw["home_mortgage_interest_amount"]), + deductible_points_amount=int(raw["deductible_points_amount"]), + qualified_mortgage_insurance_premiums_amount=int( + raw["qualified_mortgage_insurance_premiums_amount"] + ), + investment_interest_amount=int(raw["investment_interest_amount"]), + ) + + +def _band(raw: dict[str, Any]) -> PufE19200AgiBand: + components = _components(raw) + return PufE19200AgiBand( + source_row=components.source_row, + total_interest_paid_amount=components.total_interest_paid_amount, + home_mortgage_interest_amount=components.home_mortgage_interest_amount, + deductible_points_amount=components.deductible_points_amount, + qualified_mortgage_insurance_premiums_amount=( + components.qualified_mortgage_insurance_premiums_amount + ), + investment_interest_amount=components.investment_interest_amount, + label=str(raw["label"]), + lower_bound=( + None if raw["lower_bound"] is None else float(raw["lower_bound"]) + ), + upper_bound=( + None if raw["upper_bound"] is None else float(raw["upper_bound"]) + ), + ) + + +def _load_source_asset() -> tuple[ + PufE19200InterestComponents, + tuple[PufE19200AgiBand, ...], +]: + payload = json.loads( + files("populace.build.us").joinpath(_SOURCE_ASSET).read_text() + ) + source = payload.get("source", {}) + if ( + source.get("tax_year") != 2015 + or source.get("table") != "2.1" + or source.get("units") != "thousands_of_us_dollars" + ): + raise ValueError(f"{_SOURCE_ASSET} has unexpected source metadata.") + source_columns = { + name: details.get("workbook_column") + for name, details in source.get("columns", {}).items() + } + if source_columns != _AMOUNT_COLUMNS: + raise ValueError(f"{_SOURCE_ASSET} has unexpected workbook columns.") + + all_returns = _components(payload["all_returns"]) + bands = tuple(_band(raw) for raw in payload["agi_bands"]) + if len(bands) != 22 or [band.source_row for band in bands] != list(range(11, 33)): + raise ValueError(f"{_SOURCE_ASSET} must carry Table 2.1 rows 11 through 32.") + if bands[0].lower_bound is not None or bands[-1].upper_bound is not None: + raise ValueError(f"{_SOURCE_ASSET} must cover the full real AGI line.") + for previous, following in zip(bands[:-1], bands[1:], strict=True): + if previous.upper_bound != following.lower_bound: + raise ValueError(f"{_SOURCE_ASSET} AGI bands are not contiguous.") + for row in (all_returns, *bands): + amounts = ( + row.total_interest_paid_amount, + row.home_mortgage_interest_amount, + row.deductible_points_amount, + row.qualified_mortgage_insurance_premiums_amount, + row.investment_interest_amount, + ) + if any(amount < 0 for amount in amounts) or row.total_interest_paid_amount == 0: + raise ValueError(f"{_SOURCE_ASSET} carries an invalid amount.") + component_sum = sum(amounts[1:]) + if abs(row.total_interest_paid_amount - component_sum) > 1: + raise ValueError( + f"{_SOURCE_ASSET} row {row.source_row} violates the published " + "component identity beyond $1,000 source rounding." + ) + return all_returns, bands + + +( + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + US_PUF_E19200_AGI_BANDS, +) = _load_source_asset() + +_AGI_UPPER_BOUNDS = np.asarray( + [ + band.upper_bound + for band in US_PUF_E19200_AGI_BANDS + if band.upper_bound is not None + ], + dtype=np.float64, +) +_HOME_MORTGAGE_SHARES = np.asarray( + [band.home_mortgage_share for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, +) + + +def split_us_puf_e19200_by_agi_band( + total_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray]: + """Split each E19200 record by its published TY2015 SOI AGI-band share. + + PolicyEngine has one non-mortgage interest input, + ``investment_interest_expense``. The residual routed there is broader + than the table's investment-interest column: it also carries deductible + points and qualified mortgage-insurance premiums. Computing it as + ``total - mortgage`` preserves E19200 exactly despite published component + rounding. + """ + + total = np.asarray(total_interest_paid, dtype=np.float64) + agi = np.asarray(adjusted_gross_income, dtype=np.float64) + if total.ndim != 1 or agi.ndim != 1: + raise ValueError("E19200 and adjusted_gross_income must be one-dimensional.") + if len(total) != len(agi): + raise ValueError( + "E19200 and adjusted_gross_income must have the same record count." + ) + if not np.isfinite(total).all(): + raise ValueError("E19200 must contain only finite values.") + if not np.isfinite(agi).all(): + raise ValueError("adjusted_gross_income must contain only finite values.") + if (total < 0).any(): + raise ValueError("E19200 must be nonnegative.") + + band_index = np.searchsorted(_AGI_UPPER_BOUNDS, agi, side="right") + mortgage = total * _HOME_MORTGAGE_SHARES[band_index] + non_mortgage = total - mortgage + return mortgage, non_mortgage + + +__all__ = [ + "PufE19200AgiBand", + "PufE19200InterestComponents", + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "split_us_puf_e19200_by_agi_band", +] diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py b/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py index ad2a4dbb..4f2aa1d1 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py @@ -35,17 +35,21 @@ from populace.fit import QRF, QRFChainState from populace.frame import EntitySchema, Frame, WeightKind, Weights -# v2 (populace#515): the checkpointed donor frame now carries the E19200 -> -# mortgage-only concept carve (US_PUF_E19200_HOME_MORTGAGE_SHARE). Loading -# validates only schema/digest/kind/role -- not donor construction identity -- -# so a v1 checkpoint initialized pre-carve would keep fitting and drawing the -# uncarved levels under carved code. The bump rejects every pre-carve -# checkpoint and forces re-initialization through the carved constructor. +# v2 (populace#515): the checkpointed donor frame gained the interim national +# E19200 -> mortgage-only concept carve. Loading validates only +# schema/digest/kind/role -- not donor construction identity -- so a v1 +# checkpoint initialized pre-carve would keep fitting and drawing the uncarved +# levels under carved code. # v3 (populace#516): donor construction now whole-row-screens grouped raw # mortgage-interest outliers before that carve. A v2 post-carve, pre-screen # checkpoint would otherwise still fit and draw the corrupt rows under screened # code, so it too must be rejected and rebuilt. -PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 3 +# v4 (populace#515 completion): donor construction replaces the national carve +# with published per-AGI-band shares and adds investment_interest_expense as a +# learned person input. The production target-order digest also changes, but a +# schema bump is still required for standalone custom-order checkpoints whose +# manifests do not fingerprint donor-construction semantics. +PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 4 PRIMARY_QRF_MANIFEST_FILENAME = "manifest.json" PRIMARY_QRF_DONOR_FILENAME = "donor.frame.h5" PRIMARY_QRF_RECIPIENT_FILENAME = "recipient.frame.h5" @@ -55,7 +59,7 @@ *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) PRIMARY_QRF_TARGET_ORDER_SHA256 = ( - "556d713d029840848aba548d9c991842a54ea3cdbdea650c0be57d5ec5782661" + "795519d161e6b8425fc3b64de7eb435d52d25e7c8250b5861f3bb21ab48266a3" ) _ARTIFACT_KIND = "populace_primary_puf_qrf_chain" diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index 4e40d049..030e7a34 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -17,6 +17,9 @@ import pandas as pd from populace.build.gates import FitWeightRecord +from populace.build.us_runtime.puf_interest_components import ( + split_us_puf_e19200_by_agi_band, +) from populace.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, @@ -33,7 +36,6 @@ "PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS", "PUF_TAX_DETAIL_SUPPORT_CHANNEL", "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", - "US_PUF_E19200_HOME_MORTGAGE_SHARE", "US_PUF_SUPPORT_FIT_NAME", "US_PUF_SUPPORT_STAGE_NAME", "assert_formula_owned_blocklist_current", @@ -57,16 +59,6 @@ #: this fit by name. US_PUF_SUPPORT_FIT_NAME = "us_puf_tax_detail_support" -# Interim populace#515 carve: SOI Pub 1304 TY2015 Table 2.1 reports -# $283,004,465 thousand of home-mortgage interest within $304,461,163 thousand -# of total interest paid. Apply that concept share per donor record before the -# QRF learns levels and realized support. The #486 ``support_value_repairs`` -# surface is instead a release-time total pin, so it cannot express this -# donor-column concept correction (#492). A uniform carve cannot identify the -# records carrying the removed points, qualified mortgage-insurance premiums, -# or investment interest, so ``investment_interest_expense`` remains all-zero. -US_PUF_E19200_HOME_MORTGAGE_SHARE = 283_004_465 / 304_461_163 - # populace#516 donor outlier screen: $10M of annual home-mortgage interest # implies roughly a $250M mortgage at 4%, not a genuine Schedule A return; the # pinned artifact's maximum REAL-scale unit values are only low single-digit @@ -83,6 +75,7 @@ # Reserved internal column used to thread the grouped RAW mortgage value to # the outlier screen; never a legal requested output (populace#516). _MORTGAGE_OUTLIER_SCREEN_COLUMN = "_raw_home_mortgage_interest_for_outlier_screen" +_E19200_AGI_BAND_COLUMN = "_adjusted_gross_income_for_e19200_band" _DEFAULT_SUPPORT_CHANNELS = ( BASE_ASEC_SUPPORT_CHANNEL, @@ -151,6 +144,7 @@ def target_order(self) -> tuple[str, ...]: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -305,6 +299,7 @@ def target_order(self) -> tuple[str, ...]: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -509,6 +504,7 @@ def clone_us_frame_for_puf_support( def puf_tax_unit_donor_from_arrays( arrays: Mapping[str, Sequence[Any]], *, + adjusted_gross_income: Sequence[Any] | None = None, person_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, tax_unit_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) -> pd.DataFrame: @@ -521,6 +517,9 @@ def puf_tax_unit_donor_from_arrays( Args: arrays: Mapping from column name to array-like values. + adjusted_gross_income: Explicit tax-unit-grain AGI used to select the + published TY2015 SOI Table 2.1 component share for each record. + Required whenever the processed PUF carries nonzero E19200. person_outputs: Person-grain PE input variables to aggregate by tax unit. tax_unit_outputs: Tax-unit-grain PE input variables to carry or derive. @@ -545,11 +544,27 @@ def puf_tax_unit_donor_from_arrays( "filing_status_code": _filing_status_codes(arrays["filing_status"]), } ) + if adjusted_gross_income is not None: + agi = _numeric_array(adjusted_gross_income) + if len(agi) != len(tax_unit_id): + raise ValueError( + "adjusted_gross_income must align one-for-one with tax_unit_id." + ) + if not np.isfinite(agi).all(): + raise ValueError("adjusted_gross_income must contain only finite values.") + tax_unit[_E19200_AGI_BAND_COLUMN] = agi person = pd.DataFrame({"tax_unit_id": person_tax_unit_id}) - if _MORTGAGE_OUTLIER_SCREEN_COLUMN in (*person_outputs, *tax_unit_outputs): + reserved_outputs = { + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + } + requested_reserved = reserved_outputs.intersection( + (*person_outputs, *tax_unit_outputs) + ) + if requested_reserved: raise ValueError( - f"{_MORTGAGE_OUTLIER_SCREEN_COLUMN!r} is reserved for the donor " - "mortgage outlier screen and cannot be a requested output." + f"{sorted(requested_reserved)!r} are reserved for donor E19200 " + "processing and cannot be requested outputs." ) raw_home_mortgage_interest = _person_source_values( arrays, @@ -613,27 +628,81 @@ def puf_tax_unit_donor_from_arrays( ) tax_unit = ( tax_unit.loc[retained] - .drop(columns=[_MORTGAGE_OUTLIER_SCREEN_COLUMN]) .reset_index(drop=True) ) - # Keep the carve BEFORE _add_predictor_aliases: no mortgage predictor - # alias exists today, but if one is ever added it must derive from the - # carved column (aliases skip already-present columns, so a post-alias - # carve would leave a stale uncarved predictor copy). - _carve_us_puf_e19200_home_mortgage_share(tax_unit) + # Keep the split BEFORE _add_predictor_aliases: no mortgage predictor alias + # exists today, but if one is ever added it must derive from the decomposed + # column (aliases skip already-present columns, so a post-alias split would + # leave a stale total-interest predictor copy). + _split_us_puf_e19200_components(tax_unit) _add_predictor_aliases(tax_unit, PUF_TAX_DETAIL_DEFAULT_PREDICTORS) return tax_unit -def _carve_us_puf_e19200_home_mortgage_share(donor: pd.DataFrame) -> None: - """Carve E19200-lineage donor columns to the mortgage-only concept.""" +def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: + """Split raw E19200 into mortgage and modeled non-mortgage components.""" + if _MORTGAGE_OUTLIER_SCREEN_COLUMN not in donor: + donor.drop(columns=[_E19200_AGI_BAND_COLUMN], errors="ignore", inplace=True) + return + raw_total = donor[_MORTGAGE_OUTLIER_SCREEN_COLUMN].to_numpy( + dtype=np.float64, + copy=False, + ) + has_nonzero_e19200 = bool((raw_total != 0).any()) + if has_nonzero_e19200 and _E19200_AGI_BAND_COLUMN not in donor: + raise ValueError( + "adjusted_gross_income is required to split nonzero PUF E19200 " + "records by the published SOI AGI bands." + ) + if "investment_interest_expense" in donor: + existing = donor["investment_interest_expense"].to_numpy( + dtype=np.float64, + copy=False, + ) + if (existing != 0).any(): + raise ValueError( + "Processed PUF already carries nonzero investment_interest_expense; " + "refusing to overwrite independently sourced values with the " + "E19200 residual." + ) + if not has_nonzero_e19200: + if "investment_interest_expense" in donor: + donor["investment_interest_expense"] = np.zeros_like(raw_total) + donor.drop( + columns=[ + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + ], + errors="ignore", + inplace=True, + ) + return + + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + raw_total, + donor[_E19200_AGI_BAND_COLUMN].to_numpy(dtype=np.float64, copy=False), + ) + band_share = np.divide( + mortgage, + raw_total, + out=np.ones_like(mortgage), + where=raw_total != 0, + ) for column in _US_PUF_E19200_LINEAGE_DONOR_COLUMNS: if column in donor: donor[column] = ( - donor[column].to_numpy(dtype=np.float64, copy=False) - * US_PUF_E19200_HOME_MORTGAGE_SHARE + donor[column].to_numpy(dtype=np.float64, copy=False) * band_share ) + if "investment_interest_expense" in donor: + donor["investment_interest_expense"] = non_mortgage + donor.drop( + columns=[ + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + ], + inplace=True, + ) def impute_us_puf_tax_detail_support( diff --git a/packages/populace-build/tests/test_puf_qrf_chain.py b/packages/populace-build/tests/test_puf_qrf_chain.py index b0692097..70f2b02e 100644 --- a/packages/populace-build/tests/test_puf_qrf_chain.py +++ b/packages/populace-build/tests/test_puf_qrf_chain.py @@ -152,7 +152,8 @@ def test_primary_qrf_production_target_order_is_locked() -> None: *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) assert PRIMARY_QRF_TARGET_ORDER == expected - assert len(PRIMARY_QRF_TARGET_ORDER) == 64 + assert len(PRIMARY_QRF_TARGET_ORDER) == 65 + assert "investment_interest_expense" in PRIMARY_QRF_TARGET_ORDER digest = hashlib.sha256( json.dumps(list(PRIMARY_QRF_TARGET_ORDER), separators=(",", ":")).encode() ).hexdigest() @@ -244,7 +245,7 @@ def test_target_subprocess_chain_matches_monolith_raw_bits_and_final_frame( run_primary_puf_qrf_chain(checkpoint_dir) -def test_primary_qrf_rejects_pre_carve_schema_versions( +def test_primary_qrf_rejects_pre_decomposition_schema_versions( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -277,10 +278,10 @@ def test_primary_qrf_rejects_pre_carve_schema_versions( manifest_path = checkpoint_dir / "manifest.json" original_manifest = json.loads(manifest_path.read_text()) assert original_manifest["schema_version"] == PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION - # Every stale version must reject -- v1 (pre-carve) AND v2 (post-carve, - # pre-screen): a loader relaxed to accept {2, 3} would pass a v1-only pin - # while resurrecting the exact checkpoint populace#516 invalidates. - for stale_version in (1, 2): + # Every stale version must reject: pre-carve v1, pre-screen v2, and + # national-carve/all-zero-investment v3. This protects custom target-order + # standalone checkpoints whose manifests do not hash donor construction. + for stale_version in (1, 2, 3): stale_manifest = dict(original_manifest) stale_manifest["schema_version"] = stale_version manifest_path.write_text(json.dumps(stale_manifest)) @@ -294,7 +295,7 @@ def test_primary_qrf_rejects_pre_carve_schema_versions( with h5py.File(target_path, mode="r") as h5: pristine_metadata = json.loads(bytes(h5["metadata_json"][...]).decode()) assert pristine_metadata["schema_version"] == PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION - for stale_version in (1, 2): + for stale_version in (1, 2, 3): metadata = dict(pristine_metadata) metadata["schema_version"] = stale_version with h5py.File(target_path, mode="r+") as h5: diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index 0e540f27..ac7727bb 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -21,10 +21,7 @@ def _representative_agi(lower: float | None, upper: float | None) -> float: def _one_record_per_band_arrays() -> tuple[dict[str, list[object]], np.ndarray]: tax_unit_ids = np.arange(1, len(US_PUF_E19200_AGI_BANDS) + 1, dtype=np.int64) - total_interest = np.asarray( - [band.total_interest_paid_amount for band in US_PUF_E19200_AGI_BANDS], - dtype=np.float64, - ) + total_interest = np.full(len(tax_unit_ids), 1_000.0, dtype=np.float64) adjusted_gross_income = np.asarray( [ _representative_agi(band.lower_bound, band.upper_bound) @@ -55,7 +52,7 @@ def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> N assert bands[0].upper_bound == 5_000 assert bands[-1].lower_bound == 10_000_000 assert bands[-1].upper_bound is None - for previous, following in zip(bands, bands[1:], strict=True): + for previous, following in zip(bands[:-1], bands[1:], strict=True): assert previous.upper_bound == following.lower_bound for band in bands: assert band.source_cells == { @@ -130,10 +127,10 @@ def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: dtype=np.float64, ) expected_mortgage = np.asarray( - [band.home_mortgage_interest_amount for band in US_PUF_E19200_AGI_BANDS], + [1_000.0 * band.home_mortgage_share for band in US_PUF_E19200_AGI_BANDS], dtype=np.float64, ) - expected_non_mortgage = total - expected_mortgage + expected_non_mortgage = 1_000.0 - expected_mortgage np.testing.assert_allclose(donor["home_mortgage_interest"], expected_mortgage) np.testing.assert_allclose( donor["investment_interest_expense"], @@ -142,11 +139,28 @@ def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: np.testing.assert_array_equal( donor["home_mortgage_interest"].to_numpy() + donor["investment_interest_expense"].to_numpy(), - total, + np.full(len(total), 1_000.0), ) - assert donor["investment_interest_expense"].sum() == 21_456_696 assert (donor["investment_interest_expense"] > 0).all() + # The same proportional rule applied to the literal source rows recovers + # the published mortgage amounts and the full conserving residual mass. + source_mortgage, source_non_mortgage = split_us_puf_e19200_by_agi_band( + total, + adjusted_gross_income, + ) + np.testing.assert_allclose( + source_mortgage, + np.asarray( + [ + band.home_mortgage_interest_amount + for band in US_PUF_E19200_AGI_BANDS + ], + dtype=np.float64, + ), + ) + assert source_non_mortgage.sum() == 21_456_696 + def test_e19200_donor_split_requires_explicit_adjusted_gross_income() -> None: arrays = { diff --git a/packages/populace-build/tests/test_us_puf_support.py b/packages/populace-build/tests/test_us_puf_support.py index 1a145592..fa3b07b0 100644 --- a/packages/populace-build/tests/test_us_puf_support.py +++ b/packages/populace-build/tests/test_us_puf_support.py @@ -13,11 +13,11 @@ CPS_CARRIED_FORMULA_OWNED_COLUMNS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_E19200_HOME_MORTGAGE_SHARE, clone_us_frame_for_puf_support, derive_us_cps_carried_inputs, impute_us_puf_tax_detail_support, puf_tax_unit_donor_from_arrays, + split_us_puf_e19200_by_agi_band, support_channel_column, support_clone_index_column, support_source_id_column, @@ -307,6 +307,7 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: "taxable_unemployment_compensation": [13.0, 17.0, 19.0], "state_and_local_sales_or_income_tax": [40.0, 50.0], }, + adjusted_gross_income=[0.0, 0.0], person_outputs=( "employment_income_before_lsr", "qualified_dividend_income", @@ -335,7 +336,10 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: assert donor["unemployment_compensation"].tolist() == [30.0, 19.0] np.testing.assert_allclose( donor["home_mortgage_interest"].to_numpy(), - np.asarray([30.0, 30.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + split_us_puf_e19200_by_agi_band( + np.asarray([30.0, 30.0]), + np.asarray([0.0, 0.0]), + )[0], ) assert donor["educator_expense"].tolist() == [300.0, 300.0] assert "interest_deduction" not in donor @@ -346,7 +350,10 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: assert US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING == 10_000_000.0 - carved_below_ceiling = 10_500_000.0 * US_PUF_E19200_HOME_MORTGAGE_SHARE + carved_below_ceiling = split_us_puf_e19200_by_agi_band( + np.asarray([10_500_000.0]), + np.asarray([10_000_000.0]), + )[0][0] assert carved_below_ceiling < US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING donor = puf_tax_unit_donor_from_arrays( @@ -374,6 +381,7 @@ def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: ], "domestic_production_ald": [900_000_000.0, 700.0, 800_000_000.0], }, + adjusted_gross_income=[0.0, 0.0, 10_000_000.0], person_outputs=( "home_mortgage_interest", "employment_income_before_lsr", @@ -389,7 +397,10 @@ def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: assert donor["domestic_production_ald"].tolist() == [700.0] np.testing.assert_allclose( donor["home_mortgage_interest"].to_numpy(), - np.asarray([5_000_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + split_us_puf_e19200_by_agi_band( + np.asarray([5_000_000.0]), + np.asarray([0.0]), + )[0], ) @@ -397,7 +408,7 @@ def test_puf_tax_unit_donor_rejects_reserved_screen_column_output() -> None: # populace#516: the raw-mortgage helper name is reserved -- requesting it # as an output would let the screen threshold and then delete a caller's # column, silently violating the requested-output contract. - with pytest.raises(ValueError, match="reserved for the donor"): + with pytest.raises(ValueError, match="reserved for donor"): puf_tax_unit_donor_from_arrays( { "tax_unit_id": [10], @@ -411,8 +422,7 @@ def test_puf_tax_unit_donor_rejects_reserved_screen_column_output() -> None: ) -def test_puf_e19200_home_mortgage_carve_scales_only_lineage_columns() -> None: - assert US_PUF_E19200_HOME_MORTGAGE_SHARE == 283_004_465 / 304_461_163 +def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: # Pin the lineage tuple by exact membership: an accidental addition (the # sol round-1 failure mode was appending investment_interest_expense, # invisible behind a zero sentinel) must fail here, not silently carve a @@ -451,31 +461,36 @@ def test_puf_e19200_home_mortgage_carve_scales_only_lineage_columns() -> None: "second_home_mortgage_balance": [0.0, 125_000.0], "first_home_mortgage_origination_year": [2018.0, 2016.0], "second_home_mortgage_origination_year": [0.0, 2020.0], - # Nonzero sentinel: the artifact carries this column all-zero, - # but a zero fixture cannot distinguish "not scaled" from - # "scaled" (0 x share == 0). The root #515 ETL carve will make - # it nonzero, and it must stay uncarved then. - "investment_interest_expense": [12.0, 34.0], + "investment_interest_expense": [0.0, 0.0], + puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN: [100.0, 200.0], + puf_support_module._E19200_AGI_BAND_COLUMN: [0.0, 10_000_000.0], } ) original = donor.copy(deep=True) - puf_support_module._carve_us_puf_e19200_home_mortgage_share(donor) + puf_support_module._split_us_puf_e19200_components(donor) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + np.asarray([100.0, 200.0]), + np.asarray([0.0, 10_000_000.0]), + ) + shares = mortgage / np.asarray([100.0, 200.0]) for column in puf_support_module._US_PUF_E19200_LINEAGE_DONOR_COLUMNS: np.testing.assert_allclose( donor[column].to_numpy(), - original[column].to_numpy() * US_PUF_E19200_HOME_MORTGAGE_SHARE, + original[column].to_numpy() * shares, ) + np.testing.assert_allclose(donor["investment_interest_expense"], non_mortgage) for column in ( "real_estate_taxes", "first_home_mortgage_balance", "second_home_mortgage_balance", "first_home_mortgage_origination_year", "second_home_mortgage_origination_year", - "investment_interest_expense", ): np.testing.assert_array_equal(donor[column], original[column]) + assert puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN not in donor + assert puf_support_module._E19200_AGI_BAND_COLUMN not in donor def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: @@ -493,6 +508,13 @@ def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: assert "self_employment_income_before_lsr" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "self_employment_income" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "tax_exempt_interest_income" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS + assert "investment_interest_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS + assert ( + "investment_interest_expense" + in puf_support_module._PUF_TAX_DETAIL_NONNEGATIVE_OUTPUTS + ) + assert "investment_interest_expense" not in PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS + assert "investment_interest_expense" not in PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS assert "unemployment_compensation" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert ( "long_term_capital_gains_before_response" @@ -1256,6 +1278,7 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: "household_weight": [100.0, 200.0], "filing_status": [b"SINGLE", b"JOINT"], "person_tax_unit_id": [10, 20], + "home_mortgage_interest": [10_000.0, 25_000.0], "first_home_mortgage_balance": [250_000.0, 500_000.0], "second_home_mortgage_balance": [0.0, 125_000.0], "first_home_mortgage_interest": [10_000.0, 20_000.0], @@ -1266,6 +1289,7 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: "domestic_production_ald": [7_500.0, 0.0], "unrecaptured_section_1250_gain": [500.0, 0.0], }, + adjusted_gross_income=[0.0, 10_000_000.0], person_outputs=(), tax_unit_outputs=PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) @@ -1277,11 +1301,25 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: assert donor["second_home_mortgage_balance"].tolist() == [0.0, 125_000.0] np.testing.assert_allclose( donor["first_home_mortgage_interest"].to_numpy(), - np.asarray([10_000.0, 20_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + np.asarray([10_000.0, 20_000.0]) + * ( + split_us_puf_e19200_by_agi_band( + np.asarray([10_000.0, 25_000.0]), + np.asarray([0.0, 10_000_000.0]), + )[0] + / np.asarray([10_000.0, 25_000.0]) + ), ) np.testing.assert_allclose( donor["second_home_mortgage_interest"].to_numpy(), - np.asarray([0.0, 5_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + np.asarray([0.0, 5_000.0]) + * ( + split_us_puf_e19200_by_agi_band( + np.asarray([10_000.0, 25_000.0]), + np.asarray([0.0, 10_000_000.0]), + )[0] + / np.asarray([10_000.0, 25_000.0]) + ), ) assert donor["first_home_mortgage_origination_year"].tolist() == [ 2018.0, diff --git a/packages/populace-build/tests/test_us_puf_support_base_builder.py b/packages/populace-build/tests/test_us_puf_support_base_builder.py index 27cd6c59..abbcc396 100644 --- a/packages/populace-build/tests/test_us_puf_support_base_builder.py +++ b/packages/populace-build/tests/test_us_puf_support_base_builder.py @@ -485,6 +485,47 @@ def test_outer_stage_resume_rejects_changed_builder_code( ) +def test_puf_donor_builder_threads_explicit_engine_agi( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + builder = _load_support_builder_module() + path = tmp_path / "puf.h5" + arrays = {"tax_unit_id": np.asarray([10.0, 20.0])} + adjusted_gross_income = np.asarray([-5_000.0, 250_000.0]) + captured: dict[str, object] = {} + + monkeypatch.setattr(builder, "_read_h5_arrays", lambda actual: arrays) + + def fake_agi( + actual: Path, + *, + target_year: int, + expected_tax_unit_ids, + ) -> np.ndarray: + captured["agi_path"] = actual + captured["target_year"] = target_year + captured["expected_tax_unit_ids"] = expected_tax_unit_ids + return adjusted_gross_income + + def fake_donor(actual_arrays, *, adjusted_gross_income): + captured["arrays"] = actual_arrays + captured["adjusted_gross_income"] = adjusted_gross_income + return "donor" + + monkeypatch.setattr(builder, "_puf_adjusted_gross_income", fake_agi) + monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", fake_donor) + + assert ( + builder._puf_tax_unit_donor_from_h5(path, target_year=2024) == "donor" + ) + assert captured["agi_path"] == path + assert captured["target_year"] == 2024 + assert captured["expected_tax_unit_ids"] is arrays["tax_unit_id"] + assert captured["arrays"] is arrays + assert captured["adjusted_gross_income"] is adjusted_gross_income + + def test_monolith_equivalence_observer_writes_all_boundaries_and_raw_bits( tmp_path: Path, ) -> None: @@ -1284,8 +1325,11 @@ def fake_retirement_distributions( "clone_us_frame_for_puf_support", lambda frame: "expanded", ) - monkeypatch.setattr(builder, "_read_h5_arrays", lambda path: {}) - monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", lambda arrays: None) + monkeypatch.setattr( + builder, + "_puf_tax_unit_donor_from_h5", + lambda path, *, target_year: None, + ) monkeypatch.setattr( builder, "impute_and_audit_us_puf_support", diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index 1ac65361..e53c1774 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -36,6 +36,7 @@ from populace.build.source_manifest import SupportSpineSpec, load_support_spine_manifest from populace.build.source_runtime import SourceRuntimeConfig, run_source_stage from populace.build.stage_profile import profile_stage +from populace.build.us_runtime.engine_lifecycle import release_engine_simulation from populace.build.us_runtime import ( ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, BASE_ASEC_SUPPORT_CHANNEL, @@ -992,8 +993,10 @@ def _run_all( ) _observe_frame_boundary(boundary_observer, "pre_clone_enrichment", base) expanded = clone_us_frame_for_puf_support(base) - arrays = _read_h5_arrays(args.puf_h5) - donor = puf_tax_unit_donor_from_arrays(arrays) + donor = _puf_tax_unit_donor_from_h5( + args.puf_h5, + target_year=args.target_year, + ) _observe_frame_boundary(boundary_observer, "clone_feature_extraction", expanded) tail_bound_diagnostics: list[dict[str, object]] = [] if boundary_observer is None: @@ -1860,7 +1863,10 @@ def _clone_feature_extraction_stage( base: Frame, ) -> tuple[Frame, dict[str, object]]: expanded = clone_us_frame_for_puf_support(base) - donor = puf_tax_unit_donor_from_arrays(_read_h5_arrays(args.puf_h5)) + donor = _puf_tax_unit_donor_from_h5( + args.puf_h5, + target_year=args.target_year, + ) qrf_dir = args.checkpoint_dir / "primary_qrf" if qrf_dir.exists(): # The outer context marks clone_feature_extraction only after both its @@ -1880,6 +1886,8 @@ def _clone_feature_extraction_stage( "puf_sha256": _sha256(args.puf_h5), "puf_donor_rows": int(len(donor)), "puf_donor_columns": sorted(donor.columns.tolist()), + "puf_e19200_agi_variable": "adjusted_gross_income", + "puf_e19200_agi_period": args.target_year, "primary_qrf_checkpoint_dir": str(qrf_dir.resolve()), } @@ -2708,6 +2716,54 @@ def _read_h5_arrays(path: Path) -> dict[str, np.ndarray]: return {name: np.asarray(dataset) for name, dataset in h5.items()} +def _puf_tax_unit_donor_from_h5( + path: Path, + *, + target_year: int, +) -> pd.DataFrame: + """Build the PUF donor with explicit engine-calculated tax-unit AGI.""" + + arrays = _read_h5_arrays(path) + adjusted_gross_income = _puf_adjusted_gross_income( + path, + target_year=target_year, + expected_tax_unit_ids=arrays["tax_unit_id"], + ) + return puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + ) + + +def _puf_adjusted_gross_income( + path: Path, + *, + target_year: int, + expected_tax_unit_ids: Sequence[object], +) -> np.ndarray: + """Calculate the processed PUF's AGI in its tax-unit row order.""" + + from policyengine_us import Microsimulation + from policyengine_us.data import USSingleYearDataset + + dataset = USSingleYearDataset(file_path=str(path)) + dataset_tax_unit_ids = dataset.tax_unit["tax_unit_id"].to_numpy(dtype=np.int64) + expected = np.asarray(expected_tax_unit_ids, dtype=np.int64) + if not np.array_equal(dataset_tax_unit_ids, expected): + raise ValueError( + "PolicyEngine PUF tax-unit order differs from the raw H5 tax_unit_id " + "order used by the E19200 donor." + ) + simulation = Microsimulation(dataset=dataset) + try: + return np.asarray( + simulation.calculate("adjusted_gross_income", period=target_year), + dtype=np.float64, + ).copy() + finally: + release_engine_simulation(simulation) + + def _row_counts(frame: Frame) -> dict[str, int]: return {entity: frame.n(entity) for entity in frame.entities} From 5c8f44048bf89f0c4534cfaa4a44e1b2e49a43ee Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 08:53:37 -0400 Subject: [PATCH 04/18] Neutralize only mortgage interest for the JCT HMID reform Once the E19200 residual reaches investment_interest_expense, zeroing the broader interest_deduction would erase both source-backed components and misstate the mortgage tax expenditure. Point the JCT mortgage reference at deductible_mortgage_interest and pin that the reform can no longer neutralize the broader deduction. The focused JCT registry, reform-shape, and coverage tests pass. Co-Authored-By: Claude Fable 5 --- .../build/us/fiscal_target_references.json | 2 +- .../tests/test_us_fiscal_targets.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/populace-build/src/populace/build/us/fiscal_target_references.json b/packages/populace-build/src/populace/build/us/fiscal_target_references.json index 116cd5dd..f9d84431 100644 --- a/packages/populace-build/src/populace/build/us/fiscal_target_references.json +++ b/packages/populace-build/src/populace/build/us/fiscal_target_references.json @@ -167,7 +167,7 @@ "metadata": { "kind": "neutralize_variable", "matrix_row": "reform_minus_baseline_income_tax", - "neutralized_variable": "interest_deduction", + "neutralized_variable": "deductible_mortgage_interest", "output_variable": "income_tax", "target_role": "jct_tax_expenditure" }, diff --git a/packages/populace-build/tests/test_us_fiscal_targets.py b/packages/populace-build/tests/test_us_fiscal_targets.py index 0fa469b4..c72e940f 100644 --- a/packages/populace-build/tests/test_us_fiscal_targets.py +++ b/packages/populace-build/tests/test_us_fiscal_targets.py @@ -31,7 +31,7 @@ "charitable_deduction": ( "jct.tax_expenditures.cy2024.charitable_deduction.revenue_loss" ), - "interest_deduction": ( + "deductible_mortgage_interest": ( "jct.tax_expenditures.cy2024.deductible_mortgage_interest.revenue_loss" ), "qualified_business_income_deduction": ( @@ -807,6 +807,20 @@ def test_jct_tax_expenditure_references_are_simple_income_tax_reforms() -> None: assert spec.neutralized_variable +def test_jct_mortgage_reform_neutralizes_only_the_mortgage_component() -> None: + mortgage_target = ( + "jct.tax_expenditures.cy2024.deductible_mortgage_interest.revenue_loss" + ) + reform = next( + spec + for spec in US_JCT_TAX_EXPENDITURE_REFORMS + if spec.target_name == mortgage_target + ) + + assert reform.neutralized_variable == "deductible_mortgage_interest" + assert reform.neutralized_variable != "interest_deduction" + + def test_jct_reform_objects_satisfy_their_own_coverage_requirement() -> None: for spec in US_JCT_TAX_EXPENDITURE_REFORMS: result = target_profile_coverage_gate([spec], [spec.coverage_requirement()]) From 595e42371aced8b8ce594b8be482c455a8798cad Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 08:57:04 -0400 Subject: [PATCH 05/18] Distinguish mortgage and broad-interest calibration vectors WHY: Once E19200 carries a non-mortgage component, the SOI home-mortgage target must compile from the mortgage-specific engine variable rather than remain byte-identical to the broader interest-paid target. Extend the target materializer fixture with both concepts and assert that their itemizer-filtered household vectors differ. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_fiscal_refresh_builder.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 76be49f9..4f4f1a29 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -6111,7 +6111,7 @@ def test_target_materialization_cache_rejects_pre_557_identities(tmp_path) -> No ) -def test_soi_eitc_child_targets_materialize_distinct_child_slices( +def test_soi_filtered_targets_keep_mortgage_and_broad_interest_distinct( monkeypatch, ) -> None: builder = _load_builder_module() @@ -6442,6 +6442,22 @@ def eitc_spec(name, measure, child_filter, *, count=False, variable="eitc"): "itemized_only": "true", }, ), + TargetSpec( + name="home_mortgage_interest_amount", + entity="household", + measure="home_mortgage_interest_amount", + value=1.0, + source="fixture", + family="irs_soi", + metadata={ + "variable": "deductible_mortgage_interest", + "agi_lower_bound": "-inf", + "agi_upper_bound": "inf", + "filing_status": "All", + "source_measure_id": "home_mortgage_interest_amount", + "itemized_only": "true", + }, + ), ) class FakeVariable: @@ -6460,6 +6476,7 @@ class FakeSystem: "eitc_child_count", "itemized_taxable_income_deductions", "charitable_deduction", + "deductible_mortgage_interest", "interest_deduction", "medical_expense_deduction", "real_estate_taxes", @@ -6498,6 +6515,9 @@ def calculate(self, variable, *, period, **kwargs): [1_000.0, 2_000.0, 3_000.0, 4_000.0] ), "charitable_deduction": np.asarray([10.0, 20.0, 30.0, 40.0]), + "deductible_mortgage_interest": np.asarray( + [0.5, 1.25, 2.0, 3.0] + ), "interest_deduction": np.asarray([1.0, 2.0, 3.0, 4.0]), "medical_expense_deduction": np.asarray([100.0, 200.0, 300.0, 400.0]), "real_estate_taxes": np.asarray([5_000.0, 6_000.0, 7_000.0, 8_000.0]), @@ -6530,6 +6550,7 @@ def _invalidate_all_caches(self): "itemized_taxable_income_deductions" ), "charitable_deduction": "charitable_deduction", + "deductible_mortgage_interest": "deductible_mortgage_interest", "interest_deduction": "interest_deduction", "medical_expense_deduction": "medical_expense_deduction", "real_estate_taxes": "real_estate_taxes", @@ -6587,7 +6608,14 @@ def _invalidate_all_caches(self): assert np.array_equal( household["interest_paid_deduction_amount"], np.asarray([2.0, 0.0]) ) - assert len(registry) == 20 + assert np.array_equal( + household["home_mortgage_interest_amount"], np.asarray([1.25, 0.0]) + ) + assert not np.array_equal( + household["home_mortgage_interest_amount"], + household["interest_paid_deduction_amount"], + ) + assert len(registry) == 21 assert compilation["dropped_target_names"] == [] From b83ef928c106c117ff6069a0e0ccc72f22869ec5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 09:11:42 -0400 Subject: [PATCH 06/18] Promote sourced investment interest to a release requirement WHY: The published Table 2.1 decomposition replaces the prior source-unavailability adjudication, so leaving investment_interest_expense exempt from coverage or export gates would let the repaired component disappear silently. Move ownership to the PUF tax-detail stage, remove the obsolete known gap, regenerate the 164-required/7-exclusion manifest, and require a nonconstant exported signal. Co-Authored-By: Claude Fable 5 --- .../build/us/ecps_parity_known_gaps.json | 64 ------------- .../us/release_input_coverage_manifest.json | 8 +- .../src/populace/build/us/source_stages.json | 41 ++++++-- .../build/us_runtime/l0_refit_export.py | 1 + .../build/us_runtime/puf_aggregate_records.py | 8 +- .../us_runtime/release_input_coverage.py | 1 + .../tests/test_release_input_coverage.py | 10 ++ .../tests/test_us_capital_gain_details.py | 93 +------------------ .../tests/test_us_l0_refit_export.py | 24 +++++ ...uild_us_release_input_coverage_manifest.py | 1 + 10 files changed, 80 insertions(+), 171 deletions(-) diff --git a/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json b/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json index 5ae778c9..2e67db3d 100644 --- a/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json +++ b/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json @@ -121,70 +121,6 @@ "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88 and 659-699 consumes only those mappings; packages/populace-build/src/populace/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person/household tables without refreshing omitted Census columns." } }, - "investment_interest_expense": { - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.", - "issue": "PolicyEngine/populace#274", - "evidence": { - "classification": "source_unavailability", - "retired_derivation": { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "utils", "mortgage_interest.py"], - "lines": "141-155,268-320,327-349" - }, - "retired_puf_source_identity": { - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "puf", "puf.py"], - "interest_mapping_lines": "636-656", - "identical_mortgage_assignment_lines": "1548-1550", - "raw_source": "E19200" - }, - "retired_qrf_and_conversion_order": { - "qrf_path_parts": ["policyengine_", "us_data", "calibration", "puf_impute.py"], - "qrf_lines": "90-198,940-1075", - "conversion_path_parts": ["policyengine_", "us_data", "datasets", "cps", "extended_cps.py"], - "conversion_lines": "1178-1190", - "finding": "The QRF predicts interest_deduction and deductible_mortgage_interest separately even though the PUF construction makes their tax-unit source amounts identical; a positive post-QRF residual is not observed microdata detail." - }, - "required_intermediates": [ - "interest_deduction", - "deductible_mortgage_interest" - ], - "processed_puf": { - "filename": "puf_2024.h5", - "release": "policyengine/irs-soi-puf/1.8.0", - "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df", - "rows": 499045, - "positive_values": 0, - "weighted_total": 0.0, - "missing_columns": [ - "interest_deduction", - "deductible_mortgage_interest", - "E19200" - ] - }, - "hermetic_asec_inputs": [ - { - "filename": "census_cps_2022.h5", - "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - }, - { - "filename": "census_cps_2023.h5", - "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - }, - { - "filename": "census_cps_2024.h5", - "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - } - ], - "semantic_rejection": "SCF mortgage hints and aggregate SOI totals do not identify a return-level mortgage-versus-investment interest split. Re-running two stochastic predictions solely to manufacture a residual would synthesize values rather than restore a source observation.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-72 passes only the three locked ASEC H5 files and processed PUF H5; tools/build_us_puf_support_base.py lines 240-247 reads those arrays and cannot refresh omitted restricted raw-PUF columns." - } - }, "is_unmarried_partner_of_household_head": { "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1214-1221 derives is_unmarried_partner_of_household_head exclusively from person PERRP; datasets/cps/census_cps.py lines 306-318 declares that person source. All three SHA-recorded 2022-2024 ASEC HDF inputs consumed by the hermetic build omit PERRP. The 2022 and 2023 inputs also omit PECOHAB, A_EXPRRP, and A_FAMREL; only 2024 carries those alternatives, and its A_EXPRRP code 13 conflates partner with roommate. Populace's older-year relationship fallback uses only line, spouse, sex, and parent pointers and never identifies a partner. Using the 2024 pointer for one-third of the pool while treating the other two-thirds as false, or assigning partner status from the combined partner/roommate code, would synthesize statuses the locked sources do not report.", "issue": "PolicyEngine/populace#38", diff --git a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json index e8cefb93..336ce625 100644 --- a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json +++ b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json @@ -191,9 +191,7 @@ "status": "required" }, "investment_interest_expense": { - "issue": "PolicyEngine/populace#274", - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.", - "status": "reviewed_exclusion" + "status": "required" }, "is_blind": { "status": "required" @@ -539,8 +537,8 @@ } }, "counts": { - "required": 163, - "reviewed_exclusion": 8, + "required": 164, + "reviewed_exclusion": 7, "total": 171 }, "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, and the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/populace#462). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/populace#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index e03c8938..b22c6d50 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -13,7 +13,15 @@ "kind": "public_microdata", "format": "fixed_width_or_csv", "vintage": "2015", - "locator": "IRS SOI Public Use File, including state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" + "locator": "IRS SOI Public Use File, including total interest paid deduction E19200, state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" + }, + { + "kind": "published_aggregate_table", + "format": "xls_and_pdf", + "vintage": "tax year 2015", + "source": "https://www.irs.gov/pub/irs-soi/15in21id.xls", + "report_source": "https://www.irs.gov/pub/irs-soi/15inalcr.pdf", + "locator": "IRS SOI Table 2.1, rows 10-32; total interest CF, home mortgage interest CH, points CN, qualified mortgage insurance premiums CP, and investment interest CR; verbatim values and workbook coordinates packaged in soi_table_2_1_interest_components_ty2015.json" }, { "kind": "versioned_derived_microdata", @@ -96,6 +104,23 @@ "from_year": 2015, "to_year_from_build_config": true }, + { + "kind": "derive", + "method": "decompose_e19200_by_soi_agi_band", + "raw_total": "home_mortgage_interest", + "banding_variable": "adjusted_gross_income", + "banding_period": "target_year_from_build_config", + "published_source_year": 2015, + "published_total_column": "CF", + "published_home_mortgage_column": "CH", + "mortgage_output": "home_mortgage_interest", + "non_mortgage_output": "investment_interest_expense", + "allocation": "deterministic proportional allocation within each published AGI band", + "outlier_screen_before_split": { + "variable": "home_mortgage_interest", + "maximum": 10000000 + } + }, { "kind": "fit_weighted_qrf", "predictors": [ @@ -138,6 +163,7 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -202,6 +228,7 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -2303,7 +2330,7 @@ { "kind": "derived_from_prior_stages", "inputs": [ - "interest_deduction", + "home_mortgage_interest", "scf_mortgage_debt", "tenure_type" ] @@ -2313,7 +2340,7 @@ { "kind": "derive_mortgage_balance_hints", "inputs": [ - "interest_deduction", + "home_mortgage_interest", "scf_mortgage_debt", "filing_status" ] @@ -2332,17 +2359,15 @@ "deductible_mortgage_interest", "home_mortgage_interest", "mortgage_interest", - "mortgage_principal", - "investment_interest_expense" + "mortgage_principal" ], "nonnegative_outputs": [ "deductible_mortgage_interest", "home_mortgage_interest", "mortgage_interest", - "mortgage_principal", - "investment_interest_expense" + "mortgage_principal" ], - "notes": "The archived conversion maps the sole interest field used by the retired derivation, E19200, to interest_deduction, assigns the identical amount as deductible mortgage interest (archived puf.py lines 654 and 1548-1550), and derives investment_interest_expense only as their nonnegative residual (utils/mortgage_interest.py lines 268-320). The retired enhanced-CPS QRF predicted those source-identical aliases separately before conversion, so positive residuals are stochastic prediction divergence rather than observed detail. The SHA-pinned PUF 1.8.0 artifact contains 499,045 all-zero investment_interest_expense values and omits both aliases and raw E19200; all three locked ASEC inputs omit them too. A nonzero hermetic restoration would synthesize an unobserved mortgage-versus-investment split, so investment_interest_expense remains a SOURCE UNAVAILABILITY WITH EVIDENCE exclusion." + "notes": "This structural stage consumes the mortgage-only lineage already produced by puf_tax_detail's source-backed Table 2.1 E19200 decomposition. It combines that lineage with SCF mortgage-debt hints to derive mortgage balances and person-carried mortgage inputs; it neither constructs nor overwrites investment_interest_expense." }, { "stage": "acs_rent", diff --git a/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py b/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py index 9514917f..f97ce380 100644 --- a/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py +++ b/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py @@ -187,6 +187,7 @@ *US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, *US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, *US_HOUSING_NONCONSTANT_PERSON_COLUMNS, + "investment_interest_expense", ) US_RELEASE_REQUIRED_SPM_UNIT_SOURCE_COLUMNS = ( diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py b/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py index 0a264832..9061806d 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py @@ -107,10 +107,10 @@ # Raw E19200 is Schedule A "Interest paid deduction, TOTAL" (home # mortgage + points + QMIP + investment interest), NOT mortgage-only; # the attribute name predates that distinction and stays stable because - # it keys raw-source audit payloads. The processed donor carves this - # lineage to the mortgage-only concept (populace#515, - # US_PUF_E19200_HOME_MORTGAGE_SHARE) -- do not compare this raw audit - # figure against carved donor masses without un-carving. + # it keys raw-source audit payloads. The processed donor decomposes this + # lineage using the published TY2015 Table 2.1 component shares for each + # AGI band (populace#515); compare this raw audit figure only with the sum + # of the carved mortgage and non-mortgage donor masses. "E19200": "mortgage_interest_paid", "E19800": "charitable_cash_contributions", "E20100": "charitable_noncash_contributions", diff --git a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py index b5a38dd1..3e7ad855 100644 --- a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py +++ b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py @@ -182,6 +182,7 @@ *US_HOUSING_INPUTS_OUTPUT_COLUMNS, *US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS, *US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, + "investment_interest_expense", "spm_unit_pre_subsidy_childcare_expenses", "household_weight", "unreimbursed_business_employee_expenses", diff --git a/packages/populace-build/tests/test_release_input_coverage.py b/packages/populace-build/tests/test_release_input_coverage.py index 2f226f64..852c9451 100644 --- a/packages/populace-build/tests/test_release_input_coverage.py +++ b/packages/populace-build/tests/test_release_input_coverage.py @@ -44,6 +44,9 @@ us_release_input_coverage_gate, us_release_reform_coverage_probes, ) +from populace.build.us_runtime.release_input_coverage import ( + RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS, +) from populace.frame import EntitySchema, Frame, WeightKind, Weights _REPO_ROOT = Path(__file__).resolve().parents[3] @@ -564,6 +567,13 @@ def test_ssi_assets_are_required_without_exclusion(self) -> None: assert asset in manifest.required_columns assert asset not in manifest.reviewed_exclusions + def test_restored_investment_interest_is_required_without_exclusion(self) -> None: + manifest = load_release_input_coverage_manifest() + column = "investment_interest_expense" + assert column in RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS + assert column in manifest.required_columns + assert column not in manifest.reviewed_exclusions + def test_post_reference_ssi_disability_criterion_has_unique_probe(self) -> None: manifest = load_release_input_coverage_manifest() column = "meets_ssi_disability_criteria" diff --git a/packages/populace-build/tests/test_us_capital_gain_details.py b/packages/populace-build/tests/test_us_capital_gain_details.py index cd8638ee..4aa65aa9 100644 --- a/packages/populace-build/tests/test_us_capital_gain_details.py +++ b/packages/populace-build/tests/test_us_capital_gain_details.py @@ -3,12 +3,7 @@ from __future__ import annotations import importlib.util -import json -from hashlib import sha256 -from importlib.resources import files -from pathlib import Path -import h5py import numpy as np import pandas as pd import pytest @@ -48,9 +43,6 @@ not policyengine_us_installed, reason="requires the policyengine-us [us] extra (build environment)", ) -ROOT = Path(__file__).resolve().parents[3] - - class _ResolvedWeights: def __init__(self, values: np.ndarray) -> None: self.values = values @@ -273,88 +265,9 @@ def test_release_wiring_promotes_only_source_backed_detail_leaves() -> None: RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS ) assert set(US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS) <= manifest.required_columns - assert "investment_interest_expense" in manifest.reviewed_exclusions - assert manifest.reviewed_exclusions["investment_interest_expense"].startswith( - "SOURCE UNAVAILABILITY WITH EVIDENCE:" - ) - - -def test_investment_interest_exclusion_has_machine_reviewable_source_evidence() -> None: - path = files("populace.build.us").joinpath("ecps_parity_known_gaps.json") - entry = json.loads(path.read_text(encoding="utf-8"))["known_gaps"][ - "investment_interest_expense" - ] - - assert entry["reason"].startswith("SOURCE UNAVAILABILITY WITH EVIDENCE:") - evidence = entry["evidence"] - assert evidence["classification"] == "source_unavailability" - assert evidence["retired_derivation"]["lines"] == "141-155,268-320,327-349" - assert evidence["required_intermediates"] == [ - "interest_deduction", - "deductible_mortgage_interest", - ] - assert evidence["processed_puf"]["positive_values"] == 0 - assert evidence["processed_puf"]["missing_columns"] == [ - "interest_deduction", - "deductible_mortgage_interest", - "E19200", - ] - assert all( - item["missing_columns"] - == ["interest_deduction", "deductible_mortgage_interest", "E19200"] - for item in evidence["hermetic_asec_inputs"] - ) - - -def test_investment_interest_evidence_matches_build_j_artifact_pins() -> None: - entry = json.loads( - files("populace.build.us") - .joinpath("ecps_parity_known_gaps.json") - .read_text(encoding="utf-8") - )["known_gaps"]["investment_interest_expense"] - evidence = entry["evidence"] - summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - - assert evidence["processed_puf"]["sha256"] == summary["puf_sha256"] - assert { - item["filename"]: item["sha256"] for item in evidence["hermetic_asec_inputs"] - } == { - Path(item["path"]).name: item["sha256"] - for item in summary["base_source"]["sources"] - } - - -def _sha256(path: Path) -> str: - digest = sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def test_mounted_puf_artifact_confirms_zero_and_missing_source() -> None: - entry = json.loads( - files("populace.build.us") - .joinpath("ecps_parity_known_gaps.json") - .read_text(encoding="utf-8") - )["known_gaps"]["investment_interest_expense"] - evidence = entry["evidence"]["processed_puf"] - summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - path = Path(summary["puf_h5"]) - if not path.is_file(): - pytest.skip("SHA-locked PUF artifact is not mounted in this environment") - - assert _sha256(path) == evidence["sha256"] - with h5py.File(path) as h5: - assert set(evidence["missing_columns"]).isdisjoint(h5.keys()) - values = h5["investment_interest_expense"][...] - assert len(values) == evidence["rows"] - assert int(np.count_nonzero(values > 0.0)) == evidence["positive_values"] - assert float(values.sum()) == evidence["weighted_total"] + assert "investment_interest_expense" in RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS + assert "investment_interest_expense" in manifest.required_columns + assert "investment_interest_expense" not in manifest.reviewed_exclusions def test_shipped_neutralization_probes_bind_each_restored_leaf() -> None: diff --git a/packages/populace-build/tests/test_us_l0_refit_export.py b/packages/populace-build/tests/test_us_l0_refit_export.py index 9ba5d4cb..2dfa1f0e 100644 --- a/packages/populace-build/tests/test_us_l0_refit_export.py +++ b/packages/populace-build/tests/test_us_l0_refit_export.py @@ -69,6 +69,7 @@ def _us_frame(**person_extra: object) -> Frame: "casualty_loss": [0.0, 2_500.0, 0.0], "unreimbursed_business_employee_expenses": [1_200.0, 0.0, 800.0], "investment_income_elected_form_4952": [0.0, 500.0, 250.0], + "investment_interest_expense": [0.0, 800.0, 200.0], "salt_refund_income": [0.0, 1_200.0, 400.0], "long_term_capital_gains_on_collectibles": [0.0, 2_500.0, 1_000.0], "qualified_tuition_expenses": [1_000.0, 0.0, 2_500.0], @@ -643,6 +644,28 @@ def test_required_us_release_source_columns_enforces_form_4952_signal() -> None: assert_required_us_release_source_columns(raw_frame) +def test_required_us_release_source_columns_enforces_investment_interest_signal() -> ( + None +): + frame = _us_frame() + raw_people = frame.table("person").copy() + raw_people["investment_interest_expense"] = 0.0 + raw_frame = Frame( + { + **{entity: frame.table(entity).copy() for entity in frame.schema.entities}, + "person": raw_people, + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + + with pytest.raises( + ValueError, + match="person.investment_interest_expense: not nonconstant", + ): + assert_required_us_release_source_columns(raw_frame) + + @pytest.mark.parametrize( "column", [ @@ -1066,6 +1089,7 @@ def write_dataset(self, bundle, path, period): "investment_income_elected_form_4952" in summary["required_person_source_columns"] ) + assert "investment_interest_expense" in summary["required_person_source_columns"] assert ( "other_health_insurance_premiums" in summary["required_person_source_columns"] ) diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 6e76efcf..d7499fc7 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -256,6 +256,7 @@ *RETIREMENT_DISTRIBUTION_INPUTS, "domestic_production_ald", "household_weight", + "investment_interest_expense", "spm_unit_pre_subsidy_childcare_expenses", "unreimbursed_business_employee_expenses", *QBI_INPUTS, From 24ee6cf2558e119019c46f0c66f8f599d42914d8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 09:11:58 -0400 Subject: [PATCH 07/18] Document the E19200 decomposition repair WHY: Release notes must record the change from a national mortgage-only carve to a sourced per-band decomposition and the corresponding JCT concept correction. Co-Authored-By: Claude Fable 5 --- changelog.d/e19200-decomposition-515.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/e19200-decomposition-515.fixed.md diff --git a/changelog.d/e19200-decomposition-515.fixed.md b/changelog.d/e19200-decomposition-515.fixed.md new file mode 100644 index 00000000..fbf05376 --- /dev/null +++ b/changelog.d/e19200-decomposition-515.fixed.md @@ -0,0 +1 @@ +Decompose each PUF E19200 record into mortgage and non-mortgage interest using the published 2015 SOI Table 2.1 component shares for its AGI band, restore nonzero investment-interest support, and bind the JCT mortgage reform to the mortgage-specific variable. From bd5a05eb02960c2d90dd0a0ace6f94ae77e9f64c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 09:12:42 -0400 Subject: [PATCH 08/18] Normalize imports after the decomposition wiring WHY: Keep the changed runtime exports and base-builder imports compliant with the repository Ruff gate. Co-Authored-By: Claude Fable 5 --- .../populace-build/src/populace/build/us_runtime/__init__.py | 4 ++-- tools/build_us_puf_support_base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index afffd15b..a8ebd3a7 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -496,10 +496,10 @@ with_us_prior_year_income_inputs, ) from populace.build.us_runtime.puf_interest_components import ( - PufE19200AgiBand, - PufE19200InterestComponents, US_PUF_E19200_AGI_BANDS, US_PUF_E19200_ALL_RETURNS_COMPONENTS, + PufE19200AgiBand, + PufE19200InterestComponents, split_us_puf_e19200_by_agi_band, ) from populace.build.us_runtime.puf_support import ( diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index e53c1774..7f1e4c83 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -36,7 +36,6 @@ from populace.build.source_manifest import SupportSpineSpec, load_support_spine_manifest from populace.build.source_runtime import SourceRuntimeConfig, run_source_stage from populace.build.stage_profile import profile_stage -from populace.build.us_runtime.engine_lifecycle import release_engine_simulation from populace.build.us_runtime import ( ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, BASE_ASEC_SUPPORT_CHANNEL, @@ -122,6 +121,7 @@ with_us_wic_claim_input, with_us_workers_compensation, ) +from populace.build.us_runtime.engine_lifecycle import release_engine_simulation from populace.build.us_runtime.puf_qrf_chain import ( finalize_primary_puf_qrf_chain, initialize_primary_puf_qrf_chain, From 403511302f99415034c3183bb91b658abc31ffe0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 09:12:58 -0400 Subject: [PATCH 09/18] Format the E19200 implementation and regressions WHY: The issue gate requires every changed Python file to pass the repository format check, including the new source-table loader and materializer fixtures. Co-Authored-By: Claude Fable 5 --- .../build/us_runtime/puf_interest_components.py | 12 +++--------- .../src/populace/build/us_runtime/puf_support.py | 5 +---- .../tests/test_release_input_coverage.py | 7 ++----- .../tests/test_us_capital_gain_details.py | 2 ++ .../tests/test_us_fiscal_refresh_builder.py | 4 +--- .../tests/test_us_puf_e19200_decomposition.py | 9 ++------- .../tests/test_us_puf_support_base_builder.py | 4 +--- 7 files changed, 12 insertions(+), 31 deletions(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py index 2866fec1..1f256d6f 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -90,12 +90,8 @@ def _band(raw: dict[str, Any]) -> PufE19200AgiBand: ), investment_interest_amount=components.investment_interest_amount, label=str(raw["label"]), - lower_bound=( - None if raw["lower_bound"] is None else float(raw["lower_bound"]) - ), - upper_bound=( - None if raw["upper_bound"] is None else float(raw["upper_bound"]) - ), + lower_bound=(None if raw["lower_bound"] is None else float(raw["lower_bound"])), + upper_bound=(None if raw["upper_bound"] is None else float(raw["upper_bound"])), ) @@ -103,9 +99,7 @@ def _load_source_asset() -> tuple[ PufE19200InterestComponents, tuple[PufE19200AgiBand, ...], ]: - payload = json.loads( - files("populace.build.us").joinpath(_SOURCE_ASSET).read_text() - ) + payload = json.loads(files("populace.build.us").joinpath(_SOURCE_ASSET).read_text()) source = payload.get("source", {}) if ( source.get("tax_year") != 2015 diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index 030e7a34..ddc8617b 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -626,10 +626,7 @@ def puf_tax_unit_donor_from_arrays( tax_unit[_MORTGAGE_OUTLIER_SCREEN_COLUMN] < US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING ) - tax_unit = ( - tax_unit.loc[retained] - .reset_index(drop=True) - ) + tax_unit = tax_unit.loc[retained].reset_index(drop=True) # Keep the split BEFORE _add_predictor_aliases: no mortgage predictor alias # exists today, but if one is ever added it must derive from the decomposed # column (aliases skip already-present columns, so a post-alias split would diff --git a/packages/populace-build/tests/test_release_input_coverage.py b/packages/populace-build/tests/test_release_input_coverage.py index 852c9451..07db4cf2 100644 --- a/packages/populace-build/tests/test_release_input_coverage.py +++ b/packages/populace-build/tests/test_release_input_coverage.py @@ -1230,8 +1230,7 @@ def test_shipped_overtime_probes_cite_the_jct_ledger_anchor(self) -> None: reason = by_id[probe_id].reason assert "JCX-35-25" in reason assert ( - "jct.obbba_title_vii.fy2026.no_tax_on_overtime.revenue_effect" - in reason + "jct.obbba_title_vii.fy2026.no_tax_on_overtime.revenue_effect" in reason ) assert "-$32.806 billion" in reason neutralization = by_id["fsla_overtime_premium_neutralization"].reason @@ -1248,9 +1247,7 @@ def test_shipped_tips_probes_cite_the_jct_ledger_anchor(self) -> None: for probe_id in ("obbba_no_tax_on_tips", "tip_income_neutralization"): reason = by_id[probe_id].reason assert "JCX-35-25" in reason - assert ( - "jct.obbba_title_vii.fy2026.no_tax_on_tips.revenue_effect" in reason - ) + assert "jct.obbba_title_vii.fy2026.no_tax_on_tips.revenue_effect" in reason assert "-$10.121 billion" in reason neutralization = by_id["tip_income_neutralization"].reason assert "certified Build N" in neutralization diff --git a/packages/populace-build/tests/test_us_capital_gain_details.py b/packages/populace-build/tests/test_us_capital_gain_details.py index 4aa65aa9..1b766129 100644 --- a/packages/populace-build/tests/test_us_capital_gain_details.py +++ b/packages/populace-build/tests/test_us_capital_gain_details.py @@ -43,6 +43,8 @@ not policyengine_us_installed, reason="requires the policyengine-us [us] extra (build environment)", ) + + class _ResolvedWeights: def __init__(self, values: np.ndarray) -> None: self.values = values diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 4f4f1a29..b7938d88 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -6515,9 +6515,7 @@ def calculate(self, variable, *, period, **kwargs): [1_000.0, 2_000.0, 3_000.0, 4_000.0] ), "charitable_deduction": np.asarray([10.0, 20.0, 30.0, 40.0]), - "deductible_mortgage_interest": np.asarray( - [0.5, 1.25, 2.0, 3.0] - ), + "deductible_mortgage_interest": np.asarray([0.5, 1.25, 2.0, 3.0]), "interest_deduction": np.asarray([1.0, 2.0, 3.0, 4.0]), "medical_expense_deduction": np.asarray([100.0, 200.0, 300.0, 400.0]), "real_estate_taxes": np.asarray([5_000.0, 6_000.0, 7_000.0, 8_000.0]), diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index ac7727bb..e1ee2846 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -59,9 +59,7 @@ def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> N "total_interest_paid_amount": f"CF{band.source_row}", "home_mortgage_interest_amount": f"CH{band.source_row}", "deductible_points_amount": f"CN{band.source_row}", - "qualified_mortgage_insurance_premiums_amount": ( - f"CP{band.source_row}" - ), + "qualified_mortgage_insurance_premiums_amount": (f"CP{band.source_row}"), "investment_interest_amount": f"CR{band.source_row}", } published_components = ( @@ -152,10 +150,7 @@ def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: np.testing.assert_allclose( source_mortgage, np.asarray( - [ - band.home_mortgage_interest_amount - for band in US_PUF_E19200_AGI_BANDS - ], + [band.home_mortgage_interest_amount for band in US_PUF_E19200_AGI_BANDS], dtype=np.float64, ), ) diff --git a/packages/populace-build/tests/test_us_puf_support_base_builder.py b/packages/populace-build/tests/test_us_puf_support_base_builder.py index abbcc396..019d6f1d 100644 --- a/packages/populace-build/tests/test_us_puf_support_base_builder.py +++ b/packages/populace-build/tests/test_us_puf_support_base_builder.py @@ -516,9 +516,7 @@ def fake_donor(actual_arrays, *, adjusted_gross_income): monkeypatch.setattr(builder, "_puf_adjusted_gross_income", fake_agi) monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", fake_donor) - assert ( - builder._puf_tax_unit_donor_from_h5(path, target_year=2024) == "donor" - ) + assert builder._puf_tax_unit_donor_from_h5(path, target_year=2024) == "donor" assert captured["agi_path"] == path assert captured["target_year"] == 2024 assert captured["expected_tax_unit_ids"] is arrays["tax_unit_id"] From 8470d170432fababaf1e6d75ce1d3089a966a589 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 09:29:37 -0400 Subject: [PATCH 10/18] Declare the source table and preserve the tail fixture WHY: Country packages must enumerate every shipped source artifact, and the unrelated real-PUF tail test must supply an explicit band key now that nonzero E19200 fails closed without one. Co-Authored-By: Claude Fable 5 --- .../populace-build/src/populace/build/us/country_package.json | 1 + packages/populace-build/tests/test_us_puf_tail_bound.py | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/populace-build/src/populace/build/us/country_package.json b/packages/populace-build/src/populace/build/us/country_package.json index 07708026..28148451 100644 --- a/packages/populace-build/src/populace/build/us/country_package.json +++ b/packages/populace-build/src/populace/build/us/country_package.json @@ -12,6 +12,7 @@ "release_input_coverage_manifest.json", "soca_capital_gain_distribution_shares.json", "soi_baseline_levels.json", + "soi_table_2_1_interest_components_ty2015.json", "source_stages.json", "state_program_levels.json", "state_program_reforms.json", diff --git a/packages/populace-build/tests/test_us_puf_tail_bound.py b/packages/populace-build/tests/test_us_puf_tail_bound.py index 687fd2b4..d4a38dc5 100644 --- a/packages/populace-build/tests/test_us_puf_tail_bound.py +++ b/packages/populace-build/tests/test_us_puf_tail_bound.py @@ -323,6 +323,7 @@ def test_real_puf_weighted_p999_is_below_build_m_ceiling_draw() -> None: arrays = {column: h5[column][...] for column in required} donor = puf_tax_unit_donor_from_arrays( arrays, + adjusted_gross_income=np.zeros(len(arrays["tax_unit_id"])), person_outputs=(_TARGET,), tax_unit_outputs=(), ) From f333f6a80cedded6caee2388f1572a5aa697e4f5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 10:30:31 -0400 Subject: [PATCH 11/18] Specify source-year AGI alignment before implementation WHY: TY2015 Table 2.1 shares must follow the originating PUF E00100 bands, and the new restricted source input must invalidate outer-stage checkpoints when its bytes change. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_puf_e19200_decomposition.py | 115 ++++++++++++++++++ .../tests/test_us_puf_support_base_builder.py | 71 +++++++++-- 2 files changed, 173 insertions(+), 13 deletions(-) diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index e1ee2846..7e0f2a2e 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -8,6 +8,9 @@ US_PUF_E19200_ALL_RETURNS_COMPONENTS, split_us_puf_e19200_by_agi_band, ) +from populace.build.us_runtime.puf_source_agi import ( + source_year_puf_adjusted_gross_income, +) from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays @@ -42,6 +45,63 @@ def _one_record_per_band_arrays() -> tuple[dict[str, list[object]], np.ndarray]: return arrays, adjusted_gross_income +def _raw_puf_source_fixture() -> tuple[dict[str, list[float]], np.ndarray, np.ndarray]: + regular_ids = np.arange(1, 31, dtype=np.int64) + regular_agi = np.asarray( + [ + 4_000.0, + *np.linspace(-500_000.0, -10_000.0, 9), + *np.linspace(10_000.0, 9_000_000.0, 10), + *np.linspace(10_000_000.0, 200_000_000.0, 10), + ], + dtype=np.float64, + ) + aggregate_ids = np.asarray([999_996, 999_997, 999_998, 999_999]) + aggregate_agi = np.asarray( + [-100_000.0, 2_000_000.0, 20_000_000.0, 200_000_000.0] + ) + recid = np.concatenate((regular_ids, aggregate_ids)) + agi = np.concatenate((regular_agi, aggregate_agi)) + source: dict[str, list[float]] = { + "RECID": recid.tolist(), + "MARS": np.concatenate((np.ones(len(regular_ids)), np.zeros(4))).tolist(), + "S006": np.concatenate( + ( + np.full(len(regular_ids), 100.0), + np.asarray([14_000.0, 23_000.0, 39_000.0, 10_000.0]), + ) + ).tolist(), + "E00100": agi.tolist(), + } + screened_fields = ( + "E00200", + "P23250", + "P22250", + "E00650", + "E00300", + "E26270", + "E00900", + "E02100", + "E00400", + "E00600", + ) + for offset, field in enumerate(screened_fields, start=1): + source[field] = (np.abs(agi) / offset + offset).tolist() + + synthetic_ids = np.arange(1_000_000, 1_000_080, dtype=np.int64) + processed_ids = np.concatenate((regular_ids, synthetic_ids)) + processed_weights = np.concatenate( + ( + np.ones(len(regular_ids)), + np.full(20, 7.0), + np.full(23, 10.0), + np.full(39, 10.0), + np.full(18, 100.0 / 18.0), + ) + ) + return source, processed_ids, processed_weights + + def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> None: bands = US_PUF_E19200_AGI_BANDS @@ -227,3 +287,58 @@ def test_e19200_donor_split_is_bit_deterministic_and_order_invariant() -> None: first[column].to_numpy().view(np.uint64), shuffled[column].to_numpy().view(np.uint64), ) + + +def test_source_year_puf_agi_aligns_regular_and_synthetic_records() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + + adjusted_gross_income = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + + # The source record remains in its literal TY2015 band. A target-year + # formula or nominal uprating could move this $4,000 return into a later + # band, which is precisely what this source-aligned seam prevents. + assert adjusted_gross_income[0] == 4_000.0 + np.testing.assert_array_equal( + adjusted_gross_income[:30], + np.asarray(source["E00100"][:30]), + ) + assert (adjusted_gross_income[30:50] < 5_000.0).all() + assert ( + (adjusted_gross_income[50:73] >= 0.0) + & (adjusted_gross_income[50:73] < 10_000_000.0) + ).all() + assert (adjusted_gross_income[73:] >= 10_000_000.0).all() + + +def test_source_year_puf_agi_is_bit_deterministic() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + + first = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + second = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + + np.testing.assert_array_equal(first.view(np.uint64), second.view(np.uint64)) + + +def test_source_year_puf_agi_fails_closed_on_processed_id_drift() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + processed_ids = processed_ids.copy() + processed_ids[3] = 123_456 + + with pytest.raises(ValueError, match="regular RECID order"): + source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) diff --git a/packages/populace-build/tests/test_us_puf_support_base_builder.py b/packages/populace-build/tests/test_us_puf_support_base_builder.py index 019d6f1d..5f51e4b1 100644 --- a/packages/populace-build/tests/test_us_puf_support_base_builder.py +++ b/packages/populace-build/tests/test_us_puf_support_base_builder.py @@ -485,27 +485,31 @@ def test_outer_stage_resume_rejects_changed_builder_code( ) -def test_puf_donor_builder_threads_explicit_engine_agi( +def test_puf_donor_builder_threads_explicit_source_year_agi( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: builder = _load_support_builder_module() path = tmp_path / "puf.h5" - arrays = {"tax_unit_id": np.asarray([10.0, 20.0])} + source_path = tmp_path / "puf_2015.csv" + arrays = { + "tax_unit_id": np.asarray([10.0, 20.0]), + "household_weight": np.asarray([1.0, 2.0]), + } adjusted_gross_income = np.asarray([-5_000.0, 250_000.0]) captured: dict[str, object] = {} monkeypatch.setattr(builder, "_read_h5_arrays", lambda actual: arrays) def fake_agi( - actual: Path, + actual_source_path: Path, *, - target_year: int, - expected_tax_unit_ids, + processed_tax_unit_ids, + processed_tax_unit_weights, ) -> np.ndarray: - captured["agi_path"] = actual - captured["target_year"] = target_year - captured["expected_tax_unit_ids"] = expected_tax_unit_ids + captured["source_path"] = actual_source_path + captured["processed_tax_unit_ids"] = processed_tax_unit_ids + captured["processed_tax_unit_weights"] = processed_tax_unit_weights return adjusted_gross_income def fake_donor(actual_arrays, *, adjusted_gross_income): @@ -513,17 +517,58 @@ def fake_donor(actual_arrays, *, adjusted_gross_income): captured["adjusted_gross_income"] = adjusted_gross_income return "donor" - monkeypatch.setattr(builder, "_puf_adjusted_gross_income", fake_agi) + monkeypatch.setattr(builder, "_source_year_puf_adjusted_gross_income", fake_agi) monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", fake_donor) - assert builder._puf_tax_unit_donor_from_h5(path, target_year=2024) == "donor" - assert captured["agi_path"] == path - assert captured["target_year"] == 2024 - assert captured["expected_tax_unit_ids"] is arrays["tax_unit_id"] + assert ( + builder._puf_tax_unit_donor_from_h5( + path, + source_puf_csv=source_path, + ) + == "donor" + ) + assert captured["source_path"] == source_path + assert captured["processed_tax_unit_ids"] is arrays["tax_unit_id"] + assert captured["processed_tax_unit_weights"] is arrays["household_weight"] assert captured["arrays"] is arrays assert captured["adjusted_gross_income"] is adjusted_gross_income +def test_source_year_puf_input_content_is_checkpoint_identity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + builder = _load_support_builder_module() + source = tmp_path / "puf_2015.csv" + source.write_bytes(b"first") + args = builder._parse_args( + [ + "--base-h5", + str(tmp_path / "base.h5"), + "--puf-h5", + str(tmp_path / "puf.h5"), + "--puf-source-year-csv", + str(source), + "--out", + str(tmp_path / "out"), + "--without-block-ladder", + ] + ) + monkeypatch.setattr( + builder, + "_builder_code_identity", + lambda: {"source_sha256": "builder"}, + ) + first = builder._stage_run_config(args) + source.write_bytes(b"second") + second = builder._stage_run_config(args) + + assert first["puf_source_year_csv"] == str(source.resolve()) + assert first["puf_source_year_csv_sha256"] != second[ + "puf_source_year_csv_sha256" + ] + + def test_monolith_equivalence_observer_writes_all_boundaries_and_raw_bits( tmp_path: Path, ) -> None: From 803ef00d7b41c73ba192e8b360ca217503c19c05 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 10:41:32 -0400 Subject: [PATCH 12/18] Align E19200 bands to source-year PUF AGI The published TY2015 component shares are only valid against TY2015 E00100. Recover that source value by direct RECID alignment and the archived seed-42 aggregate-row allocator, while anchoring the Forbes tail in its evidenced source bucket. Pin the restricted input identity and make stale zero-investment base artifacts ineligible for reuse. Co-Authored-By: Claude Fable 5 --- experiments/build_j_recert/buildj_base.sh | 7 +- .../src/populace/build/us/source_stages.json | 8 +- .../src/populace/build/us_runtime/__init__.py | 14 + .../build/us_runtime/puf_source_agi.py | 432 ++++++++++++++++++ .../tests/test_us_buildj_base_script.py | 1 + .../tests/test_us_fiscal_refresh_builder.py | 26 +- .../tests/test_us_puf_e19200_decomposition.py | 64 ++- .../tests/test_us_puf_support_base_builder.py | 10 +- tools/build_us_puf_support_base.py | 95 ++-- 9 files changed, 604 insertions(+), 53 deletions(-) create mode 100644 packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py diff --git a/experiments/build_j_recert/buildj_base.sh b/experiments/build_j_recert/buildj_base.sh index 16e5153e..de7d1389 100755 --- a/experiments/build_j_recert/buildj_base.sh +++ b/experiments/build_j_recert/buildj_base.sh @@ -35,6 +35,7 @@ required = { "pre_subsidy_rent", "self_employment_income_last_year", "previous_year_income_available", + "investment_interest_expense", "salt_refund_income", "takes_up_medicare_if_eligible", "workers_compensation", @@ -71,18 +72,19 @@ source .venv/bin/activate 2>/dev/null # ---- integrity preflight on inputs (reproduce Build F's exact base) ---- chk() { local got exp; got=$(shasum -a 256 "$1" | cut -c1-16); [ "$got" = "$2" ] || { say "FATAL sha mismatch $1: $got != $2"; echo 2 > "$LOGDIR/base.rc"; exit 2; }; } -for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do +for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/puf_2015.csv" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do [ -f "$f" ] || { say "FATAL missing input $f"; echo 2 > "$LOGDIR/base.rc"; exit 2; } done chk "$AGING_FACTS" a5d34d4aad325d8c chk "$CDX" 383a666631aafd4f chk "$BLADDER" 7ba39b959068181b chk "$USD/acs_2022.h5" 0b319b496f19a691 +chk "$USD/puf_2015.csv" 0a7fd643edb1acc5 SHORT=$(git rev-parse --short HEAD) PEUS=$(.venv/bin/python -c "from importlib.metadata import version; print(version('policyengine-us'))" 2>/dev/null) say "BUILD J BASE START commit=$SHORT pe-us=$PEUS pid=$$ pressure_log=$PLOG" -say " inputs: asec 2024/2023/2022 + puf_2024 + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32" +say " inputs: asec 2024/2023/2022 + puf_2024 + TY2015 E00100 source + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32" if [ -f "$BASE" ] && base_has_restored_signal "$BASE"; then say "BASE: already present with nondefault restored input surface at $BASE — reusing" @@ -109,6 +111,7 @@ else --asec-h5 2023="$USD/census_cps_2023.h5" \ --asec-h5 2022="$USD/census_cps_2022.h5" \ --puf-h5 "$USD/puf_2024.h5" \ + --puf-source-year-csv "$USD/puf_2015.csv" \ --acs-h5 "$USD/acs_2022.h5" \ --target-year 2024 \ --seed 0 --n-estimators 32 \ diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index b22c6d50..d6aee6a1 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -13,7 +13,7 @@ "kind": "public_microdata", "format": "fixed_width_or_csv", "vintage": "2015", - "locator": "IRS SOI Public Use File, including total interest paid deduction E19200, state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" + "locator": "IRS SOI Public Use File, including source-year adjusted gross income E00100, weight S006, total interest paid deduction E19200, state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" }, { "kind": "published_aggregate_table", @@ -108,8 +108,10 @@ "kind": "derive", "method": "decompose_e19200_by_soi_agi_band", "raw_total": "home_mortgage_interest", - "banding_variable": "adjusted_gross_income", - "banding_period": "target_year_from_build_config", + "banding_variable": "E00100", + "banding_period": "source_year_2015", + "band_alignment": "direct RECID join for regular returns; archived seed-42 aggregate-record donor allocation for the three bounded synthetic buckets; source bucket floor for the Forbes-backed $100 million-plus tail", + "archived_processed_puf_disaggregation_seed": 42, "published_source_year": 2015, "published_total_column": "CF", "published_home_mortgage_column": "CH", diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index a8ebd3a7..71b7b15f 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -502,6 +502,14 @@ PufE19200InterestComponents, split_us_puf_e19200_by_agi_band, ) +from populace.build.us_runtime.puf_source_agi import ( + PUF_AGGREGATE_DISAGGREGATION_SEED, + PUF_AGGREGATE_RECIDS, + PUF_SOURCE_YEAR, + PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, + PUF_SYNTHETIC_RECID_START, + source_year_puf_adjusted_gross_income, +) from populace.build.us_runtime.puf_support import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, @@ -1699,6 +1707,11 @@ "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", "US_PUF_E19200_AGI_BANDS", "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", "US_PUF_SUPPORT_FIT_NAME", "US_PUF_SUPPORT_STAGE_NAME", "US_STATE_INCOME_TAX_TARGET_SPECS", @@ -1731,6 +1744,7 @@ "reform_validation_payload", "source_gap_family_ids", "split_us_puf_e19200_by_agi_band", + "source_year_puf_adjusted_gross_income", "ECPS_PARITY_KNOWN_GAPS_RESOURCE", "ECPS_PARITY_REFERENCE_RESOURCE", "EcpsParityReference", diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py b/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py new file mode 100644 index 00000000..8b0cb2aa --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py @@ -0,0 +1,432 @@ +"""Recover TY2015 AGI-band values aligned to the processed PUF artifact. + +The processed 2024 PUF is rooted in the 2015 IRS PUF but does not export raw +``E00100``. Most processed tax units retain their IRS ``RECID`` and therefore +join directly to raw E00100. The remaining IDs replace the four IRS disclosure +aggregate rows. This module reproduces the archived, seeded donor allocation +for the three bounded aggregate buckets; the Forbes-backed $100 million-plus +tail receives that bucket's lower bound because every such record maps to the +same final SOI Table 2.1 band ($10 million or more). +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +PUF_SOURCE_YEAR = 2015 +PUF_AGGREGATE_RECIDS = (999_996, 999_997, 999_998, 999_999) +PUF_SYNTHETIC_RECID_START = 1_000_000 +PUF_AGGREGATE_DISAGGREGATION_SEED = 42 +PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS = ( + "RECID", + "MARS", + "S006", + "E00100", + "E00200", + "P23250", + "P22250", + "E00650", + "E00300", + "E26270", + "E00900", + "E02100", + "E00400", + "E00600", +) + +_SCREENED_FIELDS = PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS[4:] +_SELECTION_POWER = 24 +_MAX_AGI_DOMINANCE = 0.20 +_AGI_CAP_100M_PLUS = 1_250_000_000.0 +_NUMERIC_TOLERANCE = 1e-9 +_AGGREGATE_BUCKET_BOUNDS = { + 999_996: (None, 0.0), + 999_997: (0.0, 10_000_000.0), + 999_998: (10_000_000.0, 100_000_000.0), + 999_999: (100_000_000.0, None), +} + + +def _source_frame( + source: str | Path | pd.DataFrame | Mapping[str, Sequence[Any]], +) -> pd.DataFrame: + if isinstance(source, (str, Path)): + source_path = Path(source) + frame = pd.read_csv( + source_path, + usecols=lambda name: name in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, + ) + else: + frame = pd.DataFrame(source).copy() + missing = [ + column for column in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS if column not in frame + ] + if missing: + raise ValueError(f"TY2015 PUF source is missing column(s): {missing}.") + frame = frame.loc[:, PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS].copy() + for column in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + if frame[list(PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS[:4])].isna().any().any(): + raise ValueError("TY2015 PUF RECID, MARS, S006, and E00100 must be numeric.") + frame[list(_SCREENED_FIELDS)] = frame[list(_SCREENED_FIELDS)].fillna(0.0) + return frame + + +def _integral_ids(values: Sequence[Any], *, label: str) -> np.ndarray: + numeric = np.asarray(values, dtype=np.float64) + if numeric.ndim != 1 or not np.isfinite(numeric).all(): + raise ValueError(f"{label} must be a finite one-dimensional array.") + integral = numeric.astype(np.int64) + if not np.array_equal(numeric, integral.astype(np.float64)): + raise ValueError(f"{label} must contain integral values.") + if len(np.unique(integral)) != len(integral): + raise ValueError(f"{label} must be unique.") + return integral + + +def _choose_n_synthetic(population_weight: float) -> int: + return int(min(40, max(20, round(population_weight / 10)))) + + +def _assign_weights( + population_weight: float, + count: int, + rng: np.random.Generator, +) -> np.ndarray: + total_weight = int(round(population_weight)) + base = max(total_weight // count, 3) + weights = np.full(count, base, dtype=int) + remainder = total_weight - int(weights.sum()) + if remainder > 0: + weights[rng.choice(count, size=remainder, replace=False)] += 1 + elif remainder < 0: + reducible = np.where(weights > 1)[0] + reduce_count = min(-remainder, len(reducible)) + weights[rng.choice(reducible, size=reduce_count, replace=False)] -= 1 + gap = total_weight - int(weights.sum()) + if gap: + weights[0] += gap + return weights.astype(np.float64) + + +def _bucket_mask(frame: pd.DataFrame, recid: int) -> pd.Series: + agi = frame["E00100"] + if recid == 999_996: + return agi < 0 + if recid == 999_997: + return (agi >= 0) & (agi < 10_000_000) + if recid == 999_998: + return (agi >= 10_000_000) & (agi < 100_000_000) + if recid == 999_999: + return agi >= 100_000_000 + raise ValueError(f"Unknown aggregate RECID {recid}.") + + +def _eligibility_scores(frame: pd.DataFrame) -> pd.Series: + maximum = np.zeros(len(frame), dtype=np.float64) + for field in _SCREENED_FIELDS: + values = frame[field].fillna(0.0).to_numpy(dtype=np.float64) + scores = np.zeros(len(frame), dtype=np.float64) + + positive = values > 0 + reference_positive = np.sort(values[positive]) + if positive.any(): + scores[positive] = np.searchsorted( + reference_positive, values[positive], side="right" + ) / len(reference_positive) + + negative = values < 0 + reference_negative = np.sort(-values[negative]) + if negative.any(): + scores[negative] = np.maximum( + scores[negative], + np.searchsorted( + reference_negative, + -values[negative], + side="right", + ) + / len(reference_negative), + ) + maximum = np.maximum(maximum, scores) + return pd.Series(maximum, index=frame.index, dtype=np.float64) + + +def _sample_bucket_donors( + regular: pd.DataFrame, + scores: pd.Series, + *, + recid: int, + target_mean_agi: float, + count: int, + rng: np.random.Generator, +) -> pd.DataFrame: + donor_bucket = regular.loc[_bucket_mask(regular, recid)] + if donor_bucket.empty: + donor_bucket = regular + bucket_scores = scores.loc[donor_bucket.index].to_numpy(dtype=np.float64) + score_mass = np.clip(bucket_scores, 1e-6, None) ** _SELECTION_POWER + donor_abs_agi = np.abs(donor_bucket["E00100"].to_numpy(dtype=np.float64)) + target_abs_agi = max(abs(float(target_mean_agi)), 1.0) + distance = np.abs(np.log1p(donor_abs_agi) - np.log1p(target_abs_agi)) + probabilities = score_mass * np.sqrt(1.0 / (1.0 + distance)) + if not np.isfinite(probabilities).all() or probabilities.sum() <= 0: + probabilities = np.ones(len(donor_bucket), dtype=np.float64) + probabilities /= probabilities.sum() + selected_index = rng.choice( + donor_bucket.index.to_numpy(), + size=count, + replace=len(donor_bucket) < count, + p=probabilities, + ) + return donor_bucket.loc[selected_index].reset_index(drop=True) + + +def _project_weighted_sum_to_bounds( + values: np.ndarray, + weights: np.ndarray, + target_total: float, + lower: np.ndarray, + upper: np.ndarray, +) -> np.ndarray: + projected = np.clip(values.astype(np.float64), lower, upper) + for _ in range(50): + residual = float(target_total - np.dot(projected, weights)) + if abs(residual) <= 1e-6: + return projected + slack = upper - projected if residual > 0 else projected - lower + free = slack > _NUMERIC_TOLERANCE + if not free.any(): + break + basis = np.abs(projected[free]) + if basis.sum() <= _NUMERIC_TOLERANCE: + basis = np.ones(free.sum(), dtype=np.float64) + denominator = float(np.dot(weights[free], basis)) + if denominator <= _NUMERIC_TOLERANCE: + basis = np.ones(free.sum(), dtype=np.float64) + denominator = float(weights[free].sum()) + delta = residual * basis / denominator + if residual > 0: + delta = np.minimum(delta, slack[free]) + else: + delta = -np.minimum(-delta, slack[free]) + projected[free] += delta + projected = np.clip(projected, lower, upper) + + residual = float(target_total - np.dot(projected, weights)) + if abs(residual) > 1e-6: + slack = upper - projected if residual > 0 else projected - lower + free = np.where(slack > _NUMERIC_TOLERANCE)[0] + if len(free): + best = free[np.argmax(slack[free] * weights[free])] + projected[best] = np.clip( + projected[best] + residual / weights[best], + lower[best], + upper[best], + ) + return projected + + +def _allocate_agi( + donor_agi: np.ndarray, + weights: np.ndarray, + *, + recid: int, + target_total: float, +) -> np.ndarray: + dominance_cap = _MAX_AGI_DOMINANCE * abs(target_total) / weights + lower_bound, upper_bound = _AGGREGATE_BUCKET_BOUNDS[recid] + if recid == 999_996: + lower = -dominance_cap + upper = np.zeros(len(weights), dtype=np.float64) + else: + lower = np.full(len(weights), float(lower_bound), dtype=np.float64) + numeric_upper = ( + _AGI_CAP_100M_PLUS if upper_bound is None else float(upper_bound) + ) + upper = np.minimum( + np.full(len(weights), numeric_upper, dtype=np.float64), + dominance_cap, + ) + + base = np.abs(np.asarray(donor_agi, dtype=np.float64)) + active = base > _NUMERIC_TOLERANCE + if not active.any(): + active = np.ones(len(base), dtype=bool) + allocated = np.zeros(len(base), dtype=np.float64) + magnitudes = base[active] + if magnitudes.sum() <= _NUMERIC_TOLERANCE: + magnitudes = np.ones(active.sum(), dtype=np.float64) + denominator = float(np.dot(weights[active], magnitudes)) + if denominator <= _NUMERIC_TOLERANCE: + magnitudes = np.ones(active.sum(), dtype=np.float64) + denominator = float(weights[active].sum()) + allocated[active] = ( + np.sign(target_total) * magnitudes * abs(target_total) / denominator + ) + return _project_weighted_sum_to_bounds( + allocated, + weights, + target_total, + lower, + upper, + ) + + +def source_year_puf_adjusted_gross_income( + source: str | Path | pd.DataFrame | Mapping[str, Sequence[Any]], + *, + processed_tax_unit_ids: Sequence[Any], + processed_tax_unit_weights: Sequence[Any], +) -> np.ndarray: + """Return TY2015 AGI banding values in processed-PUF tax-unit order. + + Regular tax units carry literal IRS PUF E00100. The first three synthetic + disclosure buckets reproduce the archived seeded donor allocation. The + open Forbes-backed tail receives $100 million, its source bucket floor; + this is an AGI-band anchor rather than a fabricated point estimate, and all + such rows belong to Table 2.1's final "$10 million or more" band. + """ + + frame = _source_frame(source) + source_ids = _integral_ids(frame["RECID"], label="TY2015 PUF RECID") + frame["RECID"] = source_ids + if set(frame.loc[frame["MARS"] == 0, "RECID"]) != set(PUF_AGGREGATE_RECIDS): + raise ValueError("TY2015 PUF must contain exactly the four aggregate RECIDs.") + + processed_ids = _integral_ids( + processed_tax_unit_ids, + label="Processed PUF tax_unit_id", + ) + processed_weights = np.asarray(processed_tax_unit_weights, dtype=np.float64) + if ( + processed_weights.ndim != 1 + or len(processed_weights) != len(processed_ids) + or not np.isfinite(processed_weights).all() + or (processed_weights <= 0).any() + ): + raise ValueError( + "Processed PUF tax-unit weights must be finite, positive, and " + "aligned one-for-one with tax_unit_id." + ) + + aggregate_mask = frame["RECID"].isin(PUF_AGGREGATE_RECIDS) + regular = frame.loc[~aggregate_mask].copy() + regular_ids = regular["RECID"].to_numpy(dtype=np.int64) + if len(processed_ids) <= len(regular_ids) or not np.array_equal( + processed_ids[: len(regular_ids)], + regular_ids, + ): + raise ValueError( + "Processed PUF regular RECID order does not match the TY2015 source." + ) + + synthetic_ids = processed_ids[len(regular_ids) :] + expected_synthetic_ids = np.arange( + PUF_SYNTHETIC_RECID_START, + PUF_SYNTHETIC_RECID_START + len(synthetic_ids), + dtype=np.int64, + ) + if not np.array_equal(synthetic_ids, expected_synthetic_ids): + raise ValueError( + "Processed PUF synthetic tax_unit_id values are not the archived " + "contiguous aggregate-record replacement IDs." + ) + + source_regular_weights = regular["S006"].to_numpy(dtype=np.float64) / 100.0 + if (source_regular_weights <= 0).any(): + raise ValueError("TY2015 regular PUF S006 weights must be positive.") + weight_ratios = processed_weights[: len(regular_ids)] / source_regular_weights + population_scale = float(np.median(weight_ratios)) + if not np.allclose( + weight_ratios, + population_scale, + rtol=1e-12, + atol=1e-12, + ): + raise ValueError( + "Processed PUF regular weights do not share one source uprating factor." + ) + + aggregate = frame.loc[aggregate_mask].set_index("RECID") + aggregate_population = { + recid: float(aggregate.loc[recid, "S006"]) / 100.0 * population_scale + for recid in PUF_AGGREGATE_RECIDS + } + bounded_counts = { + recid: _choose_n_synthetic(aggregate_population[recid]) + for recid in PUF_AGGREGATE_RECIDS[:3] + } + top_count = len(synthetic_ids) - sum(bounded_counts.values()) + if top_count <= 0: + raise ValueError( + "Processed PUF does not contain the archived open-tail replacement rows." + ) + counts = { + **bounded_counts, + PUF_AGGREGATE_RECIDS[-1]: top_count, + } + + output = np.empty(len(processed_ids), dtype=np.float64) + output[: len(regular_ids)] = regular["E00100"].to_numpy(dtype=np.float64) + if not np.isfinite(output[: len(regular_ids)]).all(): + raise ValueError("TY2015 regular PUF E00100 must be finite.") + + scores = _eligibility_scores(regular) + rng = np.random.default_rng(PUF_AGGREGATE_DISAGGREGATION_SEED) + cursor = len(regular_ids) + for recid in PUF_AGGREGATE_RECIDS[:3]: + count = counts[recid] + assigned_weights = _assign_weights( + aggregate_population[recid], + count, + rng, + ) + processed_bucket_weights = processed_weights[cursor : cursor + count] + if not np.array_equal(assigned_weights, processed_bucket_weights): + raise ValueError( + f"Processed PUF synthetic weights for aggregate RECID {recid} " + "do not match the archived seeded assignment." + ) + target_mean_agi = float(aggregate.loc[recid, "E00100"]) + selected = _sample_bucket_donors( + regular, + scores, + recid=recid, + target_mean_agi=target_mean_agi, + count=count, + rng=rng, + ) + output[cursor : cursor + count] = _allocate_agi( + selected["E00100"].to_numpy(dtype=np.float64), + assigned_weights, + recid=recid, + target_total=aggregate_population[recid] * target_mean_agi, + ) + cursor += count + + top_weights = processed_weights[cursor:] + expected_top_weight = int(round(aggregate_population[999_999])) + if not np.isclose(top_weights.sum(), expected_top_weight, rtol=0.0, atol=1e-9): + raise ValueError( + "Processed PUF open-tail weights do not preserve the archived " + "aggregate-row population." + ) + output[cursor:] = _AGGREGATE_BUCKET_BOUNDS[999_999][0] + if not np.isfinite(output).all(): + raise ValueError("Recovered TY2015 PUF AGI banding values must be finite.") + return output + + +__all__ = [ + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", + "source_year_puf_adjusted_gross_income", +] diff --git a/packages/populace-build/tests/test_us_buildj_base_script.py b/packages/populace-build/tests/test_us_buildj_base_script.py index 111891c7..8ef78884 100644 --- a/packages/populace-build/tests/test_us_buildj_base_script.py +++ b/packages/populace-build/tests/test_us_buildj_base_script.py @@ -34,6 +34,7 @@ def test_cache_rejects_bases_predating_salt_refund_and_energy_subsidy() -> None: def test_cache_rejects_bases_predating_measured_medicare_take_up() -> None: required = _cache_required_columns() + assert "investment_interest_expense" in required["person"] assert "takes_up_medicare_if_eligible" in required["person"] assert "workers_compensation" in required["person"] assert "would_claim_wic" in required["person"] diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index b7938d88..9591c302 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -6114,7 +6114,18 @@ def test_target_materialization_cache_rejects_pre_557_identities(tmp_path) -> No def test_soi_filtered_targets_keep_mortgage_and_broad_interest_distinct( monkeypatch, ) -> None: + from populace.build.us_runtime import split_us_puf_e19200_by_agi_band + builder = _load_builder_module() + e19200_total = np.asarray([100.0, 200.0, 300.0, 400.0]) + source_year_agi = np.asarray([-5_000.0, 20_000.0, 100_000.0, 10_000_000.0]) + mortgage_interest, non_mortgage_interest = split_us_puf_e19200_by_agi_band( + e19200_total, source_year_agi + ) + assert np.all(non_mortgage_interest > 0) + broader_interest = mortgage_interest + non_mortgage_interest + np.testing.assert_array_equal(broader_interest, e19200_total) + frame = Frame( { "person": pd.DataFrame( @@ -6515,8 +6526,8 @@ def calculate(self, variable, *, period, **kwargs): [1_000.0, 2_000.0, 3_000.0, 4_000.0] ), "charitable_deduction": np.asarray([10.0, 20.0, 30.0, 40.0]), - "deductible_mortgage_interest": np.asarray([0.5, 1.25, 2.0, 3.0]), - "interest_deduction": np.asarray([1.0, 2.0, 3.0, 4.0]), + "deductible_mortgage_interest": mortgage_interest, + "interest_deduction": broader_interest, "medical_expense_deduction": np.asarray([100.0, 200.0, 300.0, 400.0]), "real_estate_taxes": np.asarray([5_000.0, 6_000.0, 7_000.0, 8_000.0]), "salt_deduction": np.asarray([500.0, 600.0, 700.0, 800.0]), @@ -6604,10 +6615,17 @@ def _invalidate_all_caches(self): ) assert np.array_equal(household["charitable_amount"], np.asarray([20.0, 0.0])) assert np.array_equal( - household["interest_paid_deduction_amount"], np.asarray([2.0, 0.0]) + household["interest_paid_deduction_amount"], + np.asarray([broader_interest[1], 0.0]), ) assert np.array_equal( - household["home_mortgage_interest_amount"], np.asarray([1.25, 0.0]) + household["home_mortgage_interest_amount"], + np.asarray([mortgage_interest[1], 0.0]), + ) + np.testing.assert_allclose( + household["interest_paid_deduction_amount"] + - household["home_mortgage_interest_amount"], + np.asarray([non_mortgage_interest[1], 0.0]), ) assert not np.array_equal( household["home_mortgage_interest_amount"], diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index 7e0f2a2e..d5630f01 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -1,5 +1,8 @@ from __future__ import annotations +from hashlib import sha256 +from pathlib import Path + import numpy as np import pytest @@ -13,6 +16,25 @@ ) from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays +_REAL_PUF_STORAGE = ( + Path.home() + / "PolicyEngine" + / ("policyengine" + "-us-data") + / ("policyengine" + "_us_data") + / "storage" +) +_REAL_PROCESSED_PUF = _REAL_PUF_STORAGE / "puf_2024.h5" +_REAL_SOURCE_PUF = _REAL_PUF_STORAGE / "puf_2015.csv" +_REAL_PROCESSED_PUF_SHA256 = ( + "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df" +) +_REAL_SOURCE_PUF_SHA256 = ( + "0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df" +) +_REAL_SOURCE_AGI_VECTOR_SHA256 = ( + "8b2b6c206e8e9f7b80dcfd86554962b0966b25ad8c47eab7d9db4f27f69ebc0d" +) + def _representative_agi(lower: float | None, upper: float | None) -> float: if lower is None: @@ -57,9 +79,7 @@ def _raw_puf_source_fixture() -> tuple[dict[str, list[float]], np.ndarray, np.nd dtype=np.float64, ) aggregate_ids = np.asarray([999_996, 999_997, 999_998, 999_999]) - aggregate_agi = np.asarray( - [-100_000.0, 2_000_000.0, 20_000_000.0, 200_000_000.0] - ) + aggregate_agi = np.asarray([-100_000.0, 2_000_000.0, 20_000_000.0, 200_000_000.0]) recid = np.concatenate((regular_ids, aggregate_ids)) agi = np.concatenate((regular_agi, aggregate_agi)) source: dict[str, list[float]] = { @@ -88,7 +108,7 @@ def _raw_puf_source_fixture() -> tuple[dict[str, list[float]], np.ndarray, np.nd for offset, field in enumerate(screened_fields, start=1): source[field] = (np.abs(agi) / offset + offset).tolist() - synthetic_ids = np.arange(1_000_000, 1_000_080, dtype=np.int64) + synthetic_ids = np.arange(1_000_000, 1_000_102, dtype=np.int64) processed_ids = np.concatenate((regular_ids, synthetic_ids)) processed_weights = np.concatenate( ( @@ -96,12 +116,20 @@ def _raw_puf_source_fixture() -> tuple[dict[str, list[float]], np.ndarray, np.nd np.full(20, 7.0), np.full(23, 10.0), np.full(39, 10.0), - np.full(18, 100.0 / 18.0), + np.full(20, 5.0), ) ) return source, processed_ids, processed_weights +def _file_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> None: bands = US_PUF_E19200_AGI_BANDS @@ -342,3 +370,29 @@ def test_source_year_puf_agi_fails_closed_on_processed_id_drift() -> None: processed_tax_unit_ids=processed_ids, processed_tax_unit_weights=processed_weights, ) + + +def test_source_year_puf_agi_matches_pinned_flat_array_artifact() -> None: + if not _REAL_PROCESSED_PUF.exists() or not _REAL_SOURCE_PUF.exists(): + pytest.skip("Pinned restricted PUF artifacts are not available.") + assert _file_sha256(_REAL_PROCESSED_PUF) == _REAL_PROCESSED_PUF_SHA256 + assert _file_sha256(_REAL_SOURCE_PUF) == _REAL_SOURCE_PUF_SHA256 + + import h5py + + with h5py.File(_REAL_PROCESSED_PUF, "r") as h5: + # This is deliberately the production artifact's flat root-array + # layout, not a USSingleYearDataset entity-table fixture. + assert "person" not in h5 + adjusted_gross_income = source_year_puf_adjusted_gross_income( + _REAL_SOURCE_PUF, + processed_tax_unit_ids=np.asarray(h5["tax_unit_id"]), + processed_tax_unit_weights=np.asarray(h5["household_weight"]), + ) + + assert len(adjusted_gross_income) == 211_677 + assert ( + sha256(adjusted_gross_income.astype(" None: builder = _load_support_builder_module() puf = tmp_path / "puf.h5" puf.write_bytes(b"puf") + source_puf = tmp_path / "puf_2015.csv" + source_puf.write_bytes(b"source puf") sidecar = tmp_path / "asecpub24csv.zip" sidecar.write_bytes(b"zip") argv = [ @@ -1855,6 +1855,8 @@ def test_stage_cli_round_trips_the_locked_run_config(tmp_path: Path) -> None: f"2024={tmp_path / 'census_cps_2024.h5'}", "--puf-h5", str(puf), + "--puf-source-year-csv", + str(source_puf), "--asec-education-source", f"2023={sidecar}", "--target-year", diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index 7f1e4c83..d7992939 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -65,6 +65,7 @@ load_congressional_district_vintage_crosswalk, load_us_block_ladder, puf_tax_unit_donor_from_arrays, + source_year_puf_adjusted_gross_income, support_channel_column, translate_congressional_district_facts_to_current_vintage, us_adult_care_signal_gate, @@ -121,7 +122,6 @@ with_us_wic_claim_input, with_us_workers_compensation, ) -from populace.build.us_runtime.engine_lifecycle import release_engine_simulation from populace.build.us_runtime.puf_qrf_chain import ( finalize_primary_puf_qrf_chain, initialize_primary_puf_qrf_chain, @@ -267,6 +267,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ), ) parser.add_argument("--puf-h5", required=True, type=Path) + parser.add_argument( + "--puf-source-year-csv", + type=Path, + help=( + "Restricted raw TY2015 IRS PUF CSV carrying RECID, E00100, S006, " + "and the archived aggregate-record donor fields. Required when " + "building the E19200 decomposition from a nonzero processed PUF." + ), + ) parser.add_argument( "--asec-2023-weeks-unemployed-source", type=Path, @@ -519,6 +528,11 @@ def _stage_cli_args(args: argparse.Namespace, stage: str) -> list[str]: command.extend(("--asec-max-households", str(args.asec_max_households))) _append_path_argument(command, "--support-spine-spec", args.support_spine_spec) command.extend(("--puf-h5", str(args.puf_h5))) + _append_path_argument( + command, + "--puf-source-year-csv", + args.puf_source_year_csv, + ) _append_path_argument( command, "--asec-2023-weeks-unemployed-source", @@ -665,6 +679,12 @@ def path(value: Path | None) -> str | None: "n_estimators": args.n_estimators, "out": path(args.out), "puf_h5": path(args.puf_h5), + "puf_source_year_csv": path(args.puf_source_year_csv), + "puf_source_year_csv_sha256": ( + _sha256(args.puf_source_year_csv) + if args.puf_source_year_csv is not None + else None + ), "seed": args.seed, "support_spine_spec": path(args.support_spine_spec), "target_year": args.target_year, @@ -995,7 +1015,7 @@ def _run_all( expanded = clone_us_frame_for_puf_support(base) donor = _puf_tax_unit_donor_from_h5( args.puf_h5, - target_year=args.target_year, + source_puf_csv=getattr(args, "puf_source_year_csv", None), ) _observe_frame_boundary(boundary_observer, "clone_feature_extraction", expanded) tail_bound_diagnostics: list[dict[str, object]] = [] @@ -1379,6 +1399,16 @@ def _run_all( "base_sha256": _sha256(args.base_h5) if args.base_h5 is not None else None, "puf_h5": str(args.puf_h5.resolve()), "puf_sha256": _sha256(args.puf_h5), + "puf_source_year_csv": ( + str(args.puf_source_year_csv.resolve()) + if args.puf_source_year_csv is not None + else None + ), + "puf_source_year_csv_sha256": ( + _sha256(args.puf_source_year_csv) + if args.puf_source_year_csv is not None + else None + ), "acs_h5": str(args.acs_h5.resolve()) if args.acs_h5 is not None else None, "acs_sha256": _sha256(args.acs_h5) if args.acs_h5 is not None else None, "acs_rent_donor_rows": ( @@ -1865,7 +1895,7 @@ def _clone_feature_extraction_stage( expanded = clone_us_frame_for_puf_support(base) donor = _puf_tax_unit_donor_from_h5( args.puf_h5, - target_year=args.target_year, + source_puf_csv=args.puf_source_year_csv, ) qrf_dir = args.checkpoint_dir / "primary_qrf" if qrf_dir.exists(): @@ -1884,10 +1914,12 @@ def _clone_feature_extraction_stage( return expanded, { "puf_h5": str(args.puf_h5.resolve()), "puf_sha256": _sha256(args.puf_h5), + "puf_source_year_csv": str(args.puf_source_year_csv.resolve()), + "puf_source_year_csv_sha256": _sha256(args.puf_source_year_csv), "puf_donor_rows": int(len(donor)), "puf_donor_columns": sorted(donor.columns.tolist()), - "puf_e19200_agi_variable": "adjusted_gross_income", - "puf_e19200_agi_period": args.target_year, + "puf_e19200_agi_variable": "E00100", + "puf_e19200_agi_period": 2015, "primary_qrf_checkpoint_dir": str(qrf_dir.resolve()), } @@ -2406,6 +2438,8 @@ def _export_staged_result( ), "puf_h5": clone["puf_h5"], "puf_sha256": clone["puf_sha256"], + "puf_source_year_csv": clone["puf_source_year_csv"], + "puf_source_year_csv_sha256": clone["puf_source_year_csv_sha256"], "acs_h5": pre_clone["acs_h5"], "acs_sha256": pre_clone["acs_sha256"], "acs_rent_donor_rows": pre_clone["acs_rent_donor_rows"], @@ -2719,15 +2753,20 @@ def _read_h5_arrays(path: Path) -> dict[str, np.ndarray]: def _puf_tax_unit_donor_from_h5( path: Path, *, - target_year: int, + source_puf_csv: Path | None, ) -> pd.DataFrame: - """Build the PUF donor with explicit engine-calculated tax-unit AGI.""" + """Build the PUF donor with source-year E00100-aligned banding values.""" arrays = _read_h5_arrays(path) - adjusted_gross_income = _puf_adjusted_gross_income( - path, - target_year=target_year, - expected_tax_unit_ids=arrays["tax_unit_id"], + if source_puf_csv is None: + raise ValueError( + "--puf-source-year-csv is required to align nonzero E19200 records " + "to the published TY2015 SOI AGI bands." + ) + adjusted_gross_income = _source_year_puf_adjusted_gross_income( + source_puf_csv, + processed_tax_unit_ids=arrays["tax_unit_id"], + processed_tax_unit_weights=arrays["household_weight"], ) return puf_tax_unit_donor_from_arrays( arrays, @@ -2735,33 +2774,19 @@ def _puf_tax_unit_donor_from_h5( ) -def _puf_adjusted_gross_income( - path: Path, +def _source_year_puf_adjusted_gross_income( + source_puf_csv: Path, *, - target_year: int, - expected_tax_unit_ids: Sequence[object], + processed_tax_unit_ids: Sequence[object], + processed_tax_unit_weights: Sequence[object], ) -> np.ndarray: - """Calculate the processed PUF's AGI in its tax-unit row order.""" - - from policyengine_us import Microsimulation - from policyengine_us.data import USSingleYearDataset + """Load the restricted source only through its narrow alignment seam.""" - dataset = USSingleYearDataset(file_path=str(path)) - dataset_tax_unit_ids = dataset.tax_unit["tax_unit_id"].to_numpy(dtype=np.int64) - expected = np.asarray(expected_tax_unit_ids, dtype=np.int64) - if not np.array_equal(dataset_tax_unit_ids, expected): - raise ValueError( - "PolicyEngine PUF tax-unit order differs from the raw H5 tax_unit_id " - "order used by the E19200 donor." - ) - simulation = Microsimulation(dataset=dataset) - try: - return np.asarray( - simulation.calculate("adjusted_gross_income", period=target_year), - dtype=np.float64, - ).copy() - finally: - release_engine_simulation(simulation) + return source_year_puf_adjusted_gross_income( + source_puf_csv, + processed_tax_unit_ids=processed_tax_unit_ids, + processed_tax_unit_weights=processed_tax_unit_weights, + ) def _row_counts(frame: Frame) -> dict[str, int]: From 8491acf20e3bc02987ed2ec235aa442d0ab94b59 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 10:54:15 -0400 Subject: [PATCH 13/18] Match PUF lineage to the pinned artifact WHY: Source-year AGI reconstruction depends on the archived PUF having uprated the raw records before seed-42 disclosure-row expansion, including its Forbes-backed RECID 999999 tail. Correct the manifest's retired inverse ordering and disabled-tail claim, and pin both facts in the source-plan test. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/us/source_stages.json | 15 ++++++++------- packages/populace-build/tests/test_us_plan.py | 10 +++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index d6aee6a1..f55a0547 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -85,6 +85,11 @@ "unrecaptured_section_1250_gain_source": "E24515", "unrecaptured_section_1250_gain_output": "unrecaptured_section_1250_gain" }, + { + "kind": "uprate", + "from_year": 2015, + "to_year_from_build_config": true + }, { "kind": "disaggregate_aggregate_records", "method": "donor_template_calibration", @@ -97,12 +102,8 @@ ], "weight": "s006", "amount_columns": "irs_puf_amount_columns", - "seed_from_build_config": true - }, - { - "kind": "uprate", - "from_year": 2015, - "to_year_from_build_config": true + "seed_from_build_config": true, + "use_forbes_top_tail": true }, { "kind": "derive", @@ -254,7 +255,7 @@ "unadjusted_basis_qualified_property", "w2_wages_from_qualified_business" ], - "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. IRS disclosure aggregate rows are replaced from raw PUF aggregate totals before uprating; Forbes top-tail synthesis is not enabled. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." + "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." }, { "stage": "education_inputs", diff --git a/packages/populace-build/tests/test_us_plan.py b/packages/populace-build/tests/test_us_plan.py index 453ad6cd..e0b049ff 100644 --- a/packages/populace-build/tests/test_us_plan.py +++ b/packages/populace-build/tests/test_us_plan.py @@ -846,14 +846,14 @@ def test_puf_stage_outputs_match_runtime_defaults(self) -> None: assert "educator_expense" in stage.nonnegative_outputs assert "educator_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS - def test_puf_stage_disaggregates_aggregate_records_before_uprating(self) -> None: + def test_puf_stage_uprates_before_archived_disaggregation(self) -> None: operations = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"].operations kinds = [operation.kind for operation in operations] assert ( kinds.index("read_table") < kinds.index("derive_puf_policyengine_variables") - < kinds.index("disaggregate_aggregate_records") < kinds.index("uprate") + < kinds.index("disaggregate_aggregate_records") ) derive_operation = operations[kinds.index("derive_puf_policyengine_variables")] @@ -903,8 +903,12 @@ def test_puf_stage_disaggregates_aggregate_records_before_uprating(self) -> None "weight": "s006", "amount_columns": "irs_puf_amount_columns", "seed_from_build_config": True, + "use_forbes_top_tail": True, } - assert "forbes" not in str(operation.parameters).lower() + notes = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"].notes + assert "uprates the raw TY2015 rows before replacing" in notes + assert "Forbes backbone for aggregate RECID 999999" in notes + assert "3,900-record open tail" in notes def test_aca_stage_declares_marketplace_input_surface(self) -> None: stage = US_SOURCE_MANIFEST.stage_map()["aca_marketplace_inputs"] From 47838c34f83ac6c35d8d084f0c19f4b96a524e83 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 10:59:45 -0400 Subject: [PATCH 14/18] Separate pinned PUF lineage from runtime operations WHY: The archived artifact used uprate-before-disaggregate and a Forbes tail, but those are provenance facts about the consumed H5 rather than supported parameters of Populace's executable source prefix. Record and test the artifact lineage separately while preserving the runnable manifest operation contract. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/us/source_stages.json | 32 +++++++++++---- packages/populace-build/tests/test_us_plan.py | 41 ++++++++++++++++--- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index f55a0547..3d1e905b 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -27,7 +27,22 @@ "kind": "versioned_derived_microdata", "format": "hdf5_arrays", "vintage": "2024 (2015-based)", - "locator": "release://policyengine/irs-soi-puf/1.8.0/puf_2024.h5; qbi_simulation_version=1 materialized at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + "locator": "release://policyengine/irs-soi-puf/1.8.0/puf_2024.h5; qbi_simulation_version=1 materialized at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "lineage": { + "archived_commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "generator_path_parts": ["policyengine_", "us_data", "datasets", "puf", "puf.py"], + "generator_lines": "1378-1398", + "disaggregator_path_parts": ["policyengine_", "us_data", "datasets", "puf", "disaggregate_puf.py"], + "disaggregator_lines": "55-60,85-123", + "operation_order": [ + "uprate", + "disaggregate_aggregate_records" + ], + "aggregate_disaggregation_seed": 42, + "forbes_top_tail_enabled": true, + "forbes_aggregate_recid": 999999, + "forbes_synthetic_record_count": 3900 + } }, { "kind": "archived_derivation_evidence", @@ -85,11 +100,6 @@ "unrecaptured_section_1250_gain_source": "E24515", "unrecaptured_section_1250_gain_output": "unrecaptured_section_1250_gain" }, - { - "kind": "uprate", - "from_year": 2015, - "to_year_from_build_config": true - }, { "kind": "disaggregate_aggregate_records", "method": "donor_template_calibration", @@ -102,8 +112,12 @@ ], "weight": "s006", "amount_columns": "irs_puf_amount_columns", - "seed_from_build_config": true, - "use_forbes_top_tail": true + "seed_from_build_config": true + }, + { + "kind": "uprate", + "from_year": 2015, + "to_year_from_build_config": true }, { "kind": "derive", @@ -255,7 +269,7 @@ "unadjusted_basis_qualified_property", "w2_wages_from_qualified_business" ], - "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." + "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. This build consumes that versioned H5 and does not request unsupported Forbes synthesis from the source-stage runtime. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." }, { "stage": "education_inputs", diff --git a/packages/populace-build/tests/test_us_plan.py b/packages/populace-build/tests/test_us_plan.py index e0b049ff..dac4fbae 100644 --- a/packages/populace-build/tests/test_us_plan.py +++ b/packages/populace-build/tests/test_us_plan.py @@ -846,14 +846,15 @@ def test_puf_stage_outputs_match_runtime_defaults(self) -> None: assert "educator_expense" in stage.nonnegative_outputs assert "educator_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS - def test_puf_stage_uprates_before_archived_disaggregation(self) -> None: - operations = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"].operations + def test_puf_stage_distinguishes_runtime_prefix_from_artifact_lineage(self) -> None: + stage = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"] + operations = stage.operations kinds = [operation.kind for operation in operations] assert ( kinds.index("read_table") < kinds.index("derive_puf_policyengine_variables") - < kinds.index("uprate") < kinds.index("disaggregate_aggregate_records") + < kinds.index("uprate") ) derive_operation = operations[kinds.index("derive_puf_policyengine_variables")] @@ -903,12 +904,42 @@ def test_puf_stage_uprates_before_archived_disaggregation(self) -> None: "weight": "s006", "amount_columns": "irs_puf_amount_columns", "seed_from_build_config": True, - "use_forbes_top_tail": True, } - notes = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"].notes + assert "use_forbes_top_tail" not in operation.parameters + artifact = next( + artifact + for artifact in stage.artifacts + if artifact["kind"] == "versioned_derived_microdata" + ) + assert artifact["lineage"] == { + "archived_commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "generator_path_parts": [ + "policyengine_", + "us_data", + "datasets", + "puf", + "puf.py", + ], + "generator_lines": "1378-1398", + "disaggregator_path_parts": [ + "policyengine_", + "us_data", + "datasets", + "puf", + "disaggregate_puf.py", + ], + "disaggregator_lines": "55-60,85-123", + "operation_order": ["uprate", "disaggregate_aggregate_records"], + "aggregate_disaggregation_seed": 42, + "forbes_top_tail_enabled": True, + "forbes_aggregate_recid": 999999, + "forbes_synthetic_record_count": 3900, + } + notes = stage.notes assert "uprates the raw TY2015 rows before replacing" in notes assert "Forbes backbone for aggregate RECID 999999" in notes assert "3,900-record open tail" in notes + assert "does not request unsupported Forbes synthesis" in notes def test_aca_stage_declares_marketplace_input_surface(self) -> None: stage = US_SOURCE_MANIFEST.stage_map()["aca_marketplace_inputs"] From 74843d4b33a5a789ac54fb11b41c6586c0b57aa4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 11:26:42 -0400 Subject: [PATCH 15/18] Correct PUF donor provenance for E19200 WHY: The shipped donor note contradicted the pinned processed-PUF lineage by reversing uprating and disclosure-record replacement and denying the Forbes tail. Align the note with the source manifest and pin that agreement in the plan test. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/us_runtime/__init__.py | 14 ++++++++------ packages/populace-build/tests/test_us_plan.py | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 71b7b15f..dd9010df 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -2173,12 +2173,14 @@ def to_manifest(self) -> dict[str, object]: notes=( "Itemized-deduction detail, versioned processed-PUF Section 199A " "simulation leaves (carried without redrawing), partnership SE, " - "mortgage-interest split, direct E00800/E03500 alimony, direct " - "E20500 casualty loss, and the E20400 miscellaneous-expense proxy; IRS " - "disclosure aggregate rows are " - "disaggregated from raw PUF totals before uprating, with Forbes " - "top-tail synthesis disabled; support clipped to the PUF's own " - "realized ranges." + "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " + "miscellaneous-expense proxy. The pinned processed PUF uprates its " + "raw TY2015 rows before seeded disclosure-record replacement and " + "includes a Forbes-backed 3,900-record open tail; this runtime " + "reconstructs the bounded-record AGI lineage and anchors every " + "Forbes-tail record in the final published AGI band without rerunning " + "Forbes synthesis. Support is clipped to the PUF's realized ranges." ), ), US_EDUCATION_INPUTS_STAGE_NAME: DonorSpec( diff --git a/packages/populace-build/tests/test_us_plan.py b/packages/populace-build/tests/test_us_plan.py index dac4fbae..c5613679 100644 --- a/packages/populace-build/tests/test_us_plan.py +++ b/packages/populace-build/tests/test_us_plan.py @@ -940,6 +940,10 @@ def test_puf_stage_distinguishes_runtime_prefix_from_artifact_lineage(self) -> N assert "Forbes backbone for aggregate RECID 999999" in notes assert "3,900-record open tail" in notes assert "does not request unsupported Forbes synthesis" in notes + donor_notes = US_DONORS["puf_tax_detail"].notes + assert "uprates its raw TY2015 rows before" in donor_notes + assert "Forbes-backed 3,900-record open tail" in donor_notes + assert "without rerunning Forbes synthesis" in donor_notes def test_aca_stage_declares_marketplace_input_surface(self) -> None: stage = US_SOURCE_MANIFEST.stage_map()["aca_marketplace_inputs"] From 4e4bfe7a328268eb2571f0a08a0cd9fd51397718 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 16:58:29 -0400 Subject: [PATCH 16/18] Round 2: bit-exact conservation via reconciliation; donor API fails closed on invalid AGI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirm round 1 (HOLD, two P2 edge contracts): - The naive (total*share, total-mortgage) pair is algebraically but not bit-exactly conserving (the reviewer's E19200=1.53 top-band reproducer summed to 1.5300000000000002). Recomputing mortgage as the exact complement of the rounded residual restores bit-for-bit conservation — proven on the reproducer and a 20,000-record adversarial sweep of cents-valued totals across the AGI range, plus the pinned-PUF exactness the review measured on all 208,611 rows. - The donor API's finiteness check was defeated upstream: _numeric_array coerces nonnumeric/NaN to 0.0, silently routing invalid AGI to the first band's share. AGI now converts strictly (errors=raise; NaN/inf reach and fail the explicit check), and the regression exercises the puf_tax_unit_donor_from_arrays seam the reviewer probed — not the split function, which always validated. Co-Authored-By: Claude Fable 5 --- .../us_runtime/puf_interest_components.py | 9 +++ .../populace/build/us_runtime/puf_support.py | 9 ++- .../tests/test_us_puf_e19200_decomposition.py | 56 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py index 1f256d6f..d69f4148 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -191,8 +191,17 @@ def split_us_puf_e19200_by_agi_band( raise ValueError("E19200 must be nonnegative.") band_index = np.searchsorted(_AGI_UPPER_BOUNDS, agi, side="right") + # Reconciled split (PR #561 review finding 1): the naive pair + # (total*share, total − total*share) is algebraically conserving but not + # bit-exact in float64 (e.g. E19200=1.53 in the top band sums to + # 1.5300000000000002). Recomputing mortgage as the exact complement of + # the rounded residual makes ``mortgage + non_mortgage`` reproduce + # ``total`` bit-for-bit on every case we test, including that + # reproducer; the residual keeps its published-share definition to one + # rounding. mortgage = total * _HOME_MORTGAGE_SHARES[band_index] non_mortgage = total - mortgage + mortgage = total - non_mortgage return mortgage, non_mortgage diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index ddc8617b..ef1e04b4 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -545,7 +545,14 @@ def puf_tax_unit_donor_from_arrays( } ) if adjusted_gross_income is not None: - agi = _numeric_array(adjusted_gross_income) + # Strict conversion (PR #561 review finding 2): _numeric_array + # coerces nonnumeric/NaN to 0.0, which would silently route invalid + # AGI to the first band's share and defeat the finiteness check + # below. errors="raise" rejects nonnumeric values; NaN/inf survive + # conversion and fail the explicit check. + agi = pd.to_numeric(pd.Series(adjusted_gross_income), errors="raise").to_numpy( + dtype=np.float64 + ) if len(agi) != len(tax_unit_id): raise ValueError( "adjusted_gross_income must align one-for-one with tax_unit_id." diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index d5630f01..ee1bf375 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -396,3 +396,59 @@ def test_source_year_puf_agi_matches_pinned_flat_array_artifact() -> None: == _REAL_SOURCE_AGI_VECTOR_SHA256 ) assert (adjusted_gross_income[-3_900:] == 100_000_000.0).all() + + +def test_split_conserves_bit_exactly_on_adversarial_floats() -> None: + """PR #561 review finding 1: the naive (total*share, total-mortgage) + pair fails bit-exact conservation (E19200=1.53 in the top band summed to + 1.5300000000000002). The reconciled split must reproduce the total + bit-for-bit on the reviewer's reproducer and on an adversarial sweep.""" + import numpy as np + + from populace.build.us_runtime.puf_interest_components import ( + split_us_puf_e19200_by_agi_band, + ) + + total = np.asarray([1.53], dtype=np.float64) + agi = np.asarray([10_000_000.0], dtype=np.float64) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band(total, agi) + assert float(mortgage[0] + non_mortgage[0]) == 1.53 + + rng = np.random.default_rng(561) + totals = np.round(rng.uniform(0.01, 5_000_000.0, size=20_000), 2) + agis = rng.uniform(-100_000.0, 20_000_000.0, size=20_000) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band(totals, agis) + recon = mortgage + non_mortgage + exact = recon == totals + assert exact.all(), ( + f"{(~exact).sum()} of 20,000 adversarial records failed bit-exact " + "conservation after reconciliation" + ) + assert (mortgage >= 0.0).all() and (non_mortgage >= 0.0).all() + + +def test_donor_api_rejects_nan_and_nonnumeric_agi() -> None: + """PR #561 review finding 2: the donor API's _numeric_array coerced + nonnumeric/NaN AGI to 0.0 BEFORE the finiteness check, silently routing + invalid AGI to the first band's share. The strict conversion must fail + closed at the puf_tax_unit_donor_from_arrays seam — the layer the + reviewer probed — not merely at the split function (which always + validated).""" + import numpy as np + import pytest as _pytest + + from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays + + arrays = { + "tax_unit_id": np.asarray([1], dtype="int64"), + "person_tax_unit_id": np.asarray([1], dtype="int64"), + "household_weight": np.asarray([1.0]), + "filing_status": np.asarray(["SINGLE"]), + "E19200": np.asarray([100.0]), + } + with _pytest.raises(ValueError, match="finite"): + puf_tax_unit_donor_from_arrays( + arrays, adjusted_gross_income=np.asarray([np.nan]) + ) + with _pytest.raises((ValueError, TypeError)): + puf_tax_unit_donor_from_arrays(arrays, adjusted_gross_income=["not-a-number"]) From bd0b45d408a0f1475aac5cd51ed49a6807436ad2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 17:18:10 -0400 Subject: [PATCH 17/18] Round 3: reject non-real AGI dtypes; unmaskable regression; proven conservation comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirm round 2 (HOLD, one P2 + two P3): - errors=raise is parse-strict, not real-number-strict: datetime and timedelta arrays (including NaT) convert to finite epoch/sentinel integers, complex drops its imaginary part with only a warning, and booleans pass — the reviewer routed $100 into band 0 through a valid donor seam via pd.NaT. Non-real dtype kinds are now rejected up front (TypeError) before strict parsing; NaN/inf still fail the explicit finiteness check. - The AGI regression could mask that hole: its fixture failed for unrelated reasons (wrong column names, underivable default outputs), so any raise looked like a pass. Rebuilt around a positive control — the same fixture succeeds with valid AGI and empty explicit outputs — plus cause-specific assertions per rejection class (NaN/inf -> 'finite'; NaT/timedelta/complex/bool -> 'real-valued'; string -> parse error). The first NaT probe was itself invalid numpy (np.asarray([pd.NaT], dtype=datetime64) raises in the test's own argument expression); both the pandas-inferred and numpy-typed NaT forms are now exercised. - The reconciliation comment claimed too little and too much: conservation is guaranteed, not empirical — with share in [0, 1] one of the two subtractions is exact by Sterbenz's lemma (case analysis verified: whichever of rounded mortgage or rounded residual lands in [total/2, total]), so the pair sums to total bit-for-bit for every accepted input, with -0.0 reconstructing as +0.0. And the mortgage deviation from total*share is bounded by the subtractions' roundings at the half-ULP-of-total scale — the reviewer measured 16-17 mortgage ULPs on the 1.53 reproducer, so 'one rounding' at mortgage scale was wrong and is gone. Co-Authored-By: Claude Fable 5 --- .../us_runtime/puf_interest_components.py | 17 ++++-- .../populace/build/us_runtime/puf_support.py | 24 +++++--- .../tests/test_us_puf_e19200_decomposition.py | 58 ++++++++++++++----- 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py index d69f4148..975c8d8c 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -191,14 +191,19 @@ def split_us_puf_e19200_by_agi_band( raise ValueError("E19200 must be nonnegative.") band_index = np.searchsorted(_AGI_UPPER_BOUNDS, agi, side="right") - # Reconciled split (PR #561 review finding 1): the naive pair + # Reconciled split (PR #561 review, rounds 1-2): the naive pair # (total*share, total − total*share) is algebraically conserving but not # bit-exact in float64 (e.g. E19200=1.53 in the top band sums to - # 1.5300000000000002). Recomputing mortgage as the exact complement of - # the rounded residual makes ``mortgage + non_mortgage`` reproduce - # ``total`` bit-for-bit on every case we test, including that - # reproducer; the residual keeps its published-share definition to one - # rounding. + # 1.5300000000000002). Recomputing mortgage as the complement of the + # rounded residual makes conservation exact by construction, not merely + # empirical: with share in [0, 1], either the rounded mortgage or the + # rounded residual lands in [total/2, total], so one of the two + # subtractions is exact by Sterbenz's lemma and the pair sums to + # ``total`` bit-for-bit for every accepted total (an accepted -0.0 + # reconstructs as +0.0). The mortgage component deviates from + # ``total*share`` by at most the two subtractions' roundings — a bound + # at the half-ULP scale of ``total`` per step, which can amount to + # several ULPs of the smaller mortgage value itself. mortgage = total * _HOME_MORTGAGE_SHARES[band_index] non_mortgage = total - mortgage mortgage = total - non_mortgage diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index ef1e04b4..1c723829 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -545,14 +545,22 @@ def puf_tax_unit_donor_from_arrays( } ) if adjusted_gross_income is not None: - # Strict conversion (PR #561 review finding 2): _numeric_array - # coerces nonnumeric/NaN to 0.0, which would silently route invalid - # AGI to the first band's share and defeat the finiteness check - # below. errors="raise" rejects nonnumeric values; NaN/inf survive - # conversion and fail the explicit check. - agi = pd.to_numeric(pd.Series(adjusted_gross_income), errors="raise").to_numpy( - dtype=np.float64 - ) + # Strict conversion (PR #561 review findings, rounds 1-2): + # _numeric_array coerces nonnumeric/NaN to 0.0, and errors="raise" + # alone is parse-strict but not real-number-strict — datetime and + # timedelta arrays (including NaT) convert to finite epoch/sentinel + # integers, complex drops its imaginary part, and booleans pass. + # Any of those would route records to the wrong AGI band with the + # finiteness check below none the wiser, so non-real dtypes are + # rejected up front. NaN/inf survive strict conversion and fail the + # explicit check. + agi_series = pd.Series(adjusted_gross_income) + if agi_series.dtype.kind not in "iufO": + raise TypeError( + "adjusted_gross_income must be real-valued; got dtype " + f"{agi_series.dtype}." + ) + agi = pd.to_numeric(agi_series, errors="raise").to_numpy(dtype=np.float64) if len(agi) != len(tax_unit_id): raise ValueError( "adjusted_gross_income must align one-for-one with tax_unit_id." diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index ee1bf375..78aa5fa2 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -427,14 +427,18 @@ def test_split_conserves_bit_exactly_on_adversarial_floats() -> None: assert (mortgage >= 0.0).all() and (non_mortgage >= 0.0).all() -def test_donor_api_rejects_nan_and_nonnumeric_agi() -> None: - """PR #561 review finding 2: the donor API's _numeric_array coerced - nonnumeric/NaN AGI to 0.0 BEFORE the finiteness check, silently routing - invalid AGI to the first band's share. The strict conversion must fail - closed at the puf_tax_unit_donor_from_arrays seam — the layer the - reviewer probed — not merely at the split function (which always - validated).""" +def test_donor_api_rejects_non_real_agi_through_a_valid_fixture() -> None: + """PR #561 review findings 2 (rounds 1-2): the donor API must fail + closed on AGI that is not a finite real number. Round 1's + ``_numeric_array`` coerced nonnumeric/NaN to 0.0; round 2's + ``errors="raise"`` was parse-strict but not real-number-strict — + datetime/timedelta arrays (including NaT) convert to finite epoch + sentinels, complex drops its imaginary part, and booleans pass, all + silently routing records to the wrong AGI band. The fixture below is + proven valid by a positive control, so each rejection can only come + from the AGI validation itself.""" import numpy as np + import pandas as pd import pytest as _pytest from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays @@ -444,11 +448,39 @@ def test_donor_api_rejects_nan_and_nonnumeric_agi() -> None: "person_tax_unit_id": np.asarray([1], dtype="int64"), "household_weight": np.asarray([1.0]), "filing_status": np.asarray(["SINGLE"]), - "E19200": np.asarray([100.0]), } - with _pytest.raises(ValueError, match="finite"): - puf_tax_unit_donor_from_arrays( - arrays, adjusted_gross_income=np.asarray([np.nan]) + + def call(agi): + return puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=agi, + person_outputs=(), + tax_unit_outputs=(), ) - with _pytest.raises((ValueError, TypeError)): - puf_tax_unit_donor_from_arrays(arrays, adjusted_gross_income=["not-a-number"]) + + # Positive control: the fixture succeeds with valid AGI (the band + # column is reserved for internal processing and is consumed before + # return), so the rejections below cannot be masked by an unrelated + # fixture defect. + donor = call(np.asarray([50_000.0])) + assert len(donor) == 1 + assert donor["tax_unit_id"].tolist() == [1] + + with _pytest.raises(ValueError, match="finite"): + call(np.asarray([np.nan])) + with _pytest.raises(ValueError, match="finite"): + call(np.asarray([np.inf])) + with _pytest.raises(ValueError): + call(["not-a-number"]) + # Typed non-real arrays parse to finite numbers under pd.to_numeric and + # must be rejected by dtype, not by parsing. + with _pytest.raises(TypeError, match="real-valued"): + call(pd.Series([pd.NaT])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray(["NaT"], dtype="datetime64[ns]")) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([np.timedelta64(1, "D")])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([1.0 + 2.0j])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([True])) From 70389c900c85af28549395dc36f27c7b7963e260 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 27 Jul 2026 17:30:53 -0400 Subject: [PATCH 18/18] Round 4: element-screen object AGI; pin parse assertion; signed-zero wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirm round 3 (HOLD, one P2 + two advisories): - Object-dtype payloads bypassed the round-3 kind gate: object-wrapped True parses to 1.0 and routes $100 to band 0, object-wrapped complex keeps its real component with only a warning, and categorical bools ride kind 'O' too. The object branch now screens element types via pd.api.types.infer_dtype, allowing only faithful float conversions (integer/floating/mixed-integer-float/decimal/string/empty — Decimal and numeric strings stay accepted per review; missing values still die at the finiteness check) and rejecting boolean/complex/ categorical/mixed with the same real-valued TypeError the typed regressions pin. Verified live before writing: pd.Series promotes object-wrapped NaT/Timestamp to datetime64, so those hit the round-3 kind gate; bool/complex/categorical were the residual routes. - The bad-string regression now matches the strict parser's own message ('Unable to parse') instead of any ValueError. - Conservation wording: 'bit-for-bit for every accepted total except signed zero' — the -0.0 -> +0.0 reconstruction is the one exception, stated as such. Co-Authored-By: Claude Fable 5 --- .../us_runtime/puf_interest_components.py | 5 +++-- .../populace/build/us_runtime/puf_support.py | 20 +++++++++++++++++++ .../tests/test_us_puf_e19200_decomposition.py | 17 +++++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py index 975c8d8c..af7bc1d0 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -199,8 +199,9 @@ def split_us_puf_e19200_by_agi_band( # empirical: with share in [0, 1], either the rounded mortgage or the # rounded residual lands in [total/2, total], so one of the two # subtractions is exact by Sterbenz's lemma and the pair sums to - # ``total`` bit-for-bit for every accepted total (an accepted -0.0 - # reconstructs as +0.0). The mortgage component deviates from + # ``total`` bit-for-bit for every accepted total except signed zero + # (an accepted -0.0 reconstructs as +0.0). The mortgage component + # deviates from # ``total*share`` by at most the two subtractions' roundings — a bound # at the half-ULP scale of ``total`` per step, which can amount to # several ULPs of the smaller mortgage value itself. diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index 1c723829..b5a4ae30 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -560,6 +560,26 @@ def puf_tax_unit_donor_from_arrays( "adjusted_gross_income must be real-valued; got dtype " f"{agi_series.dtype}." ) + if agi_series.dtype.kind == "O": + # Element screen (PR #561 review, round 3): object and + # categorical arrays reach the strict parse below with kind + # "O", where object-wrapped booleans become 1.0/0.0 and + # object-wrapped complex keeps its real part with only a + # warning. Allow only element types whose float conversion is + # faithful: real numbers, Decimal, strings (strict-parsed), + # and missing values (which fail the finiteness check). + inferred = pd.api.types.infer_dtype(agi_series, skipna=True) + if inferred not in { + "integer", + "floating", + "mixed-integer-float", + "decimal", + "string", + "empty", + }: + raise TypeError( + f"adjusted_gross_income must be real-valued; got {inferred} values." + ) agi = pd.to_numeric(agi_series, errors="raise").to_numpy(dtype=np.float64) if len(agi) != len(tax_unit_id): raise ValueError( diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py index 78aa5fa2..43407510 100644 --- a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -465,12 +465,18 @@ def call(agi): donor = call(np.asarray([50_000.0])) assert len(donor) == 1 assert donor["tax_unit_id"].tolist() == [1] + # Faithful float conversions are accepted (review round 3): Decimal + # and numeric strings parse strictly to the same real value. + from decimal import Decimal + + assert len(call([Decimal("50000")])) == 1 + assert len(call(["50000"])) == 1 with _pytest.raises(ValueError, match="finite"): call(np.asarray([np.nan])) with _pytest.raises(ValueError, match="finite"): call(np.asarray([np.inf])) - with _pytest.raises(ValueError): + with _pytest.raises(ValueError, match="Unable to parse"): call(["not-a-number"]) # Typed non-real arrays parse to finite numbers under pd.to_numeric and # must be rejected by dtype, not by parsing. @@ -484,3 +490,12 @@ def call(agi): call(np.asarray([1.0 + 2.0j])) with _pytest.raises(TypeError, match="real-valued"): call(np.asarray([True])) + # Object-wrapped and categorical payloads dodge the dtype-kind gate + # (review round 3): True parses to 1.0 and complex keeps its real + # part, so the element screen must reject them too. + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([True], dtype=object)) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([1.0 + 2.0j], dtype=object)) + with _pytest.raises(TypeError, match="real-valued"): + call(pd.Categorical([True]))