diff --git a/changelog.d/e19200-decomposition-515.fixed.md b/changelog.d/e19200-decomposition-515.fixed.md new file mode 100644 index 00000000..fbf05376 --- /dev/null +++ b/changelog.d/e19200-decomposition-515.fixed.md @@ -0,0 +1 @@ +Decompose each PUF E19200 record into mortgage and non-mortgage interest using the published 2015 SOI Table 2.1 component shares for its AGI band, restore nonzero investment-interest support, and bind the JCT mortgage reform to the mortgage-specific variable. diff --git a/experiments/build_j_recert/buildj_base.sh b/experiments/build_j_recert/buildj_base.sh index 16e5153e..de7d1389 100755 --- a/experiments/build_j_recert/buildj_base.sh +++ b/experiments/build_j_recert/buildj_base.sh @@ -35,6 +35,7 @@ required = { "pre_subsidy_rent", "self_employment_income_last_year", "previous_year_income_available", + "investment_interest_expense", "salt_refund_income", "takes_up_medicare_if_eligible", "workers_compensation", @@ -71,18 +72,19 @@ source .venv/bin/activate 2>/dev/null # ---- integrity preflight on inputs (reproduce Build F's exact base) ---- chk() { local got exp; got=$(shasum -a 256 "$1" | cut -c1-16); [ "$got" = "$2" ] || { say "FATAL sha mismatch $1: $got != $2"; echo 2 > "$LOGDIR/base.rc"; exit 2; }; } -for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do +for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/puf_2015.csv" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do [ -f "$f" ] || { say "FATAL missing input $f"; echo 2 > "$LOGDIR/base.rc"; exit 2; } done chk "$AGING_FACTS" a5d34d4aad325d8c chk "$CDX" 383a666631aafd4f chk "$BLADDER" 7ba39b959068181b chk "$USD/acs_2022.h5" 0b319b496f19a691 +chk "$USD/puf_2015.csv" 0a7fd643edb1acc5 SHORT=$(git rev-parse --short HEAD) PEUS=$(.venv/bin/python -c "from importlib.metadata import version; print(version('policyengine-us'))" 2>/dev/null) say "BUILD J BASE START commit=$SHORT pe-us=$PEUS pid=$$ pressure_log=$PLOG" -say " inputs: asec 2024/2023/2022 + puf_2024 + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32" +say " inputs: asec 2024/2023/2022 + puf_2024 + TY2015 E00100 source + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32" if [ -f "$BASE" ] && base_has_restored_signal "$BASE"; then say "BASE: already present with nondefault restored input surface at $BASE — reusing" @@ -109,6 +111,7 @@ else --asec-h5 2023="$USD/census_cps_2023.h5" \ --asec-h5 2022="$USD/census_cps_2022.h5" \ --puf-h5 "$USD/puf_2024.h5" \ + --puf-source-year-csv "$USD/puf_2015.csv" \ --acs-h5 "$USD/acs_2022.h5" \ --target-year 2024 \ --seed 0 --n-estimators 32 \ diff --git a/packages/populace-build/src/populace/build/us/country_package.json b/packages/populace-build/src/populace/build/us/country_package.json index 07708026..28148451 100644 --- a/packages/populace-build/src/populace/build/us/country_package.json +++ b/packages/populace-build/src/populace/build/us/country_package.json @@ -12,6 +12,7 @@ "release_input_coverage_manifest.json", "soca_capital_gain_distribution_shares.json", "soi_baseline_levels.json", + "soi_table_2_1_interest_components_ty2015.json", "source_stages.json", "state_program_levels.json", "state_program_reforms.json", diff --git a/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json b/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json index 5ae778c9..2e67db3d 100644 --- a/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json +++ b/packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json @@ -121,70 +121,6 @@ "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88 and 659-699 consumes only those mappings; packages/populace-build/src/populace/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person/household tables without refreshing omitted Census columns." } }, - "investment_interest_expense": { - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.", - "issue": "PolicyEngine/populace#274", - "evidence": { - "classification": "source_unavailability", - "retired_derivation": { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "utils", "mortgage_interest.py"], - "lines": "141-155,268-320,327-349" - }, - "retired_puf_source_identity": { - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "puf", "puf.py"], - "interest_mapping_lines": "636-656", - "identical_mortgage_assignment_lines": "1548-1550", - "raw_source": "E19200" - }, - "retired_qrf_and_conversion_order": { - "qrf_path_parts": ["policyengine_", "us_data", "calibration", "puf_impute.py"], - "qrf_lines": "90-198,940-1075", - "conversion_path_parts": ["policyengine_", "us_data", "datasets", "cps", "extended_cps.py"], - "conversion_lines": "1178-1190", - "finding": "The QRF predicts interest_deduction and deductible_mortgage_interest separately even though the PUF construction makes their tax-unit source amounts identical; a positive post-QRF residual is not observed microdata detail." - }, - "required_intermediates": [ - "interest_deduction", - "deductible_mortgage_interest" - ], - "processed_puf": { - "filename": "puf_2024.h5", - "release": "policyengine/irs-soi-puf/1.8.0", - "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df", - "rows": 499045, - "positive_values": 0, - "weighted_total": 0.0, - "missing_columns": [ - "interest_deduction", - "deductible_mortgage_interest", - "E19200" - ] - }, - "hermetic_asec_inputs": [ - { - "filename": "census_cps_2022.h5", - "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - }, - { - "filename": "census_cps_2023.h5", - "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - }, - { - "filename": "census_cps_2024.h5", - "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", - "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"] - } - ], - "semantic_rejection": "SCF mortgage hints and aggregate SOI totals do not identify a return-level mortgage-versus-investment interest split. Re-running two stochastic predictions solely to manufacture a residual would synthesize values rather than restore a source observation.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-72 passes only the three locked ASEC H5 files and processed PUF H5; tools/build_us_puf_support_base.py lines 240-247 reads those arrays and cannot refresh omitted restricted raw-PUF columns." - } - }, "is_unmarried_partner_of_household_head": { "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1214-1221 derives is_unmarried_partner_of_household_head exclusively from person PERRP; datasets/cps/census_cps.py lines 306-318 declares that person source. All three SHA-recorded 2022-2024 ASEC HDF inputs consumed by the hermetic build omit PERRP. The 2022 and 2023 inputs also omit PECOHAB, A_EXPRRP, and A_FAMREL; only 2024 carries those alternatives, and its A_EXPRRP code 13 conflates partner with roommate. Populace's older-year relationship fallback uses only line, spouse, sex, and parent pointers and never identifies a partner. Using the 2024 pointer for one-third of the pool while treating the other two-thirds as false, or assigning partner status from the combined partner/roommate code, would synthesize statuses the locked sources do not report.", "issue": "PolicyEngine/populace#38", diff --git a/packages/populace-build/src/populace/build/us/fiscal_target_references.json b/packages/populace-build/src/populace/build/us/fiscal_target_references.json index 116cd5dd..f9d84431 100644 --- a/packages/populace-build/src/populace/build/us/fiscal_target_references.json +++ b/packages/populace-build/src/populace/build/us/fiscal_target_references.json @@ -167,7 +167,7 @@ "metadata": { "kind": "neutralize_variable", "matrix_row": "reform_minus_baseline_income_tax", - "neutralized_variable": "interest_deduction", + "neutralized_variable": "deductible_mortgage_interest", "output_variable": "income_tax", "target_role": "jct_tax_expenditure" }, diff --git a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json index e8cefb93..336ce625 100644 --- a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json +++ b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json @@ -191,9 +191,7 @@ "status": "required" }, "investment_interest_expense": { - "issue": "PolicyEngine/populace#274", - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.", - "status": "reviewed_exclusion" + "status": "required" }, "is_blind": { "status": "required" @@ -539,8 +537,8 @@ } }, "counts": { - "required": 163, - "reviewed_exclusion": 8, + "required": 164, + "reviewed_exclusion": 7, "total": 171 }, "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, and the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/populace#462). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/populace#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", diff --git a/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json b/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json new file mode 100644 index 00000000..8affd3e1 --- /dev/null +++ b/packages/populace-build/src/populace/build/us/soi_table_2_1_interest_components_ty2015.json @@ -0,0 +1,288 @@ +{ + "source": { + "title": "Individual Income Tax Returns Complete Report, Publication 1304, Tax Year 2015, Table 2.1", + "report_url": "https://www.irs.gov/pub/irs-soi/15inalcr.pdf", + "workbook_url": "https://www.irs.gov/pub/irs-soi/15in21id.xls", + "table": "2.1", + "tax_year": 2015, + "units": "thousands_of_us_dollars", + "columns": { + "total_interest_paid_amount": { + "printed_column": 83, + "workbook_column": "CF" + }, + "home_mortgage_interest_amount": { + "printed_column": 85, + "workbook_column": "CH" + }, + "deductible_points_amount": { + "printed_column": 91, + "workbook_column": "CN" + }, + "qualified_mortgage_insurance_premiums_amount": { + "printed_column": 93, + "workbook_column": "CP" + }, + "investment_interest_amount": { + "printed_column": 95, + "workbook_column": "CR" + } + }, + "report_pages": { + "total_home_points_and_mortgage_insurance": 100, + "investment_interest": 101 + } + }, + "all_returns": { + "source_row": 10, + "total_interest_paid_amount": 304461163, + "home_mortgage_interest_amount": 283004465, + "deductible_points_amount": 1273716, + "qualified_mortgage_insurance_premiums_amount": 6287486, + "investment_interest_amount": 13895495 + }, + "agi_bands": [ + { + "source_row": 11, + "label": "Under $5,000", + "lower_bound": null, + "upper_bound": 5000, + "total_interest_paid_amount": 1231416, + "home_mortgage_interest_amount": 1174521, + "deductible_points_amount": 1459, + "qualified_mortgage_insurance_premiums_amount": 36879, + "investment_interest_amount": 18556 + }, + { + "source_row": 12, + "label": "$5,000 under $10,000", + "lower_bound": 5000, + "upper_bound": 10000, + "total_interest_paid_amount": 1284063, + "home_mortgage_interest_amount": 1237451, + "deductible_points_amount": 871, + "qualified_mortgage_insurance_premiums_amount": 35605, + "investment_interest_amount": 10135 + }, + { + "source_row": 13, + "label": "$10,000 under $15,000", + "lower_bound": 10000, + "upper_bound": 15000, + "total_interest_paid_amount": 2148799, + "home_mortgage_interest_amount": 2083827, + "deductible_points_amount": 3523, + "qualified_mortgage_insurance_premiums_amount": 56173, + "investment_interest_amount": 5276 + }, + { + "source_row": 14, + "label": "$15,000 under $20,000", + "lower_bound": 15000, + "upper_bound": 20000, + "total_interest_paid_amount": 2699978, + "home_mortgage_interest_amount": 2589295, + "deductible_points_amount": 12649, + "qualified_mortgage_insurance_premiums_amount": 93731, + "investment_interest_amount": 4303 + }, + { + "source_row": 15, + "label": "$20,000 under $25,000", + "lower_bound": 20000, + "upper_bound": 25000, + "total_interest_paid_amount": 3181401, + "home_mortgage_interest_amount": 2992991, + "deductible_points_amount": 16428, + "qualified_mortgage_insurance_premiums_amount": 148030, + "investment_interest_amount": 23951 + }, + { + "source_row": 16, + "label": "$25,000 under $30,000", + "lower_bound": 25000, + "upper_bound": 30000, + "total_interest_paid_amount": 3748228, + "home_mortgage_interest_amount": 3591380, + "deductible_points_amount": 13066, + "qualified_mortgage_insurance_premiums_amount": 136509, + "investment_interest_amount": 7273 + }, + { + "source_row": 17, + "label": "$30,000 under $35,000", + "lower_bound": 30000, + "upper_bound": 35000, + "total_interest_paid_amount": 4214763, + "home_mortgage_interest_amount": 3946351, + "deductible_points_amount": 16300, + "qualified_mortgage_insurance_premiums_amount": 246026, + "investment_interest_amount": 6085 + }, + { + "source_row": 18, + "label": "$35,000 under $40,000", + "lower_bound": 35000, + "upper_bound": 40000, + "total_interest_paid_amount": 5435778, + "home_mortgage_interest_amount": 5122692, + "deductible_points_amount": 10850, + "qualified_mortgage_insurance_premiums_amount": 289043, + "investment_interest_amount": 13193 + }, + { + "source_row": 19, + "label": "$40,000 under $45,000", + "lower_bound": 40000, + "upper_bound": 45000, + "total_interest_paid_amount": 6009927, + "home_mortgage_interest_amount": 5651139, + "deductible_points_amount": 22472, + "qualified_mortgage_insurance_premiums_amount": 322044, + "investment_interest_amount": 14272 + }, + { + "source_row": 20, + "label": "$45,000 under $50,000", + "lower_bound": 45000, + "upper_bound": 50000, + "total_interest_paid_amount": 6888540, + "home_mortgage_interest_amount": 6411648, + "deductible_points_amount": 36417, + "qualified_mortgage_insurance_premiums_amount": 430176, + "investment_interest_amount": 10299 + }, + { + "source_row": 21, + "label": "$50,000 under $55,000", + "lower_bound": 50000, + "upper_bound": 55000, + "total_interest_paid_amount": 6955789, + "home_mortgage_interest_amount": 6543517, + "deductible_points_amount": 21952, + "qualified_mortgage_insurance_premiums_amount": 378557, + "investment_interest_amount": 11762 + }, + { + "source_row": 22, + "label": "$55,000 under $60,000", + "lower_bound": 55000, + "upper_bound": 60000, + "total_interest_paid_amount": 7424513, + "home_mortgage_interest_amount": 6985554, + "deductible_points_amount": 35610, + "qualified_mortgage_insurance_premiums_amount": 387597, + "investment_interest_amount": 15752 + }, + { + "source_row": 23, + "label": "$60,000 under $75,000", + "lower_bound": 60000, + "upper_bound": 75000, + "total_interest_paid_amount": 23294604, + "home_mortgage_interest_amount": 21825640, + "deductible_points_amount": 111241, + "qualified_mortgage_insurance_premiums_amount": 1272959, + "investment_interest_amount": 84765 + }, + { + "source_row": 24, + "label": "$75,000 under $100,000", + "lower_bound": 75000, + "upper_bound": 100000, + "total_interest_paid_amount": 43398259, + "home_mortgage_interest_amount": 40897449, + "deductible_points_amount": 238507, + "qualified_mortgage_insurance_premiums_amount": 2096979, + "investment_interest_amount": 165323 + }, + { + "source_row": 25, + "label": "$100,000 under $200,000", + "lower_bound": 100000, + "upper_bound": 200000, + "total_interest_paid_amount": 104846625, + "home_mortgage_interest_amount": 103319155, + "deductible_points_amount": 466821, + "qualified_mortgage_insurance_premiums_amount": 357177, + "investment_interest_amount": 703472 + }, + { + "source_row": 26, + "label": "$200,000 under $500,000", + "lower_bound": 200000, + "upper_bound": 500000, + "total_interest_paid_amount": 53562301, + "home_mortgage_interest_amount": 51606382, + "deductible_points_amount": 206890, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1749030 + }, + { + "source_row": 27, + "label": "$500,000 under $1,000,000", + "lower_bound": 500000, + "upper_bound": 1000000, + "total_interest_paid_amount": 12836968, + "home_mortgage_interest_amount": 11335153, + "deductible_points_amount": 40960, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1460856 + }, + { + "source_row": 28, + "label": "$1,000,000 under $1,500,000", + "lower_bound": 1000000, + "upper_bound": 1500000, + "total_interest_paid_amount": 3505209, + "home_mortgage_interest_amount": 2559360, + "deductible_points_amount": 7938, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 937911 + }, + { + "source_row": 29, + "label": "$1,500,000 under $2,000,000", + "lower_bound": 1500000, + "upper_bound": 2000000, + "total_interest_paid_amount": 1666807, + "home_mortgage_interest_amount": 1043154, + "deductible_points_amount": 2635, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 621017 + }, + { + "source_row": 30, + "label": "$2,000,000 under $5,000,000", + "lower_bound": 2000000, + "upper_bound": 5000000, + "total_interest_paid_amount": 3412493, + "home_mortgage_interest_amount": 1521570, + "deductible_points_amount": 5102, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1885822 + }, + { + "source_row": 31, + "label": "$5,000,000 under $10,000,000", + "lower_bound": 5000000, + "upper_bound": 10000000, + "total_interest_paid_amount": 1535736, + "home_mortgage_interest_amount": 360807, + "deductible_points_amount": 1387, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 1173542 + }, + { + "source_row": 32, + "label": "$10,000,000 or more", + "lower_bound": 10000000, + "upper_bound": null, + "total_interest_paid_amount": 5178966, + "home_mortgage_interest_amount": 205431, + "deductible_points_amount": 637, + "qualified_mortgage_insurance_premiums_amount": 0, + "investment_interest_amount": 4972898 + } + ] +} diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index e03c8938..3d1e905b 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -13,13 +13,36 @@ "kind": "public_microdata", "format": "fixed_width_or_csv", "vintage": "2015", - "locator": "IRS SOI Public Use File, including state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" + "locator": "IRS SOI Public Use File, including source-year adjusted gross income E00100, weight S006, total interest paid deduction E19200, state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515" + }, + { + "kind": "published_aggregate_table", + "format": "xls_and_pdf", + "vintage": "tax year 2015", + "source": "https://www.irs.gov/pub/irs-soi/15in21id.xls", + "report_source": "https://www.irs.gov/pub/irs-soi/15inalcr.pdf", + "locator": "IRS SOI Table 2.1, rows 10-32; total interest CF, home mortgage interest CH, points CN, qualified mortgage insurance premiums CP, and investment interest CR; verbatim values and workbook coordinates packaged in soi_table_2_1_interest_components_ty2015.json" }, { "kind": "versioned_derived_microdata", "format": "hdf5_arrays", "vintage": "2024 (2015-based)", - "locator": "release://policyengine/irs-soi-puf/1.8.0/puf_2024.h5; qbi_simulation_version=1 materialized at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + "locator": "release://policyengine/irs-soi-puf/1.8.0/puf_2024.h5; qbi_simulation_version=1 materialized at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "lineage": { + "archived_commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "generator_path_parts": ["policyengine_", "us_data", "datasets", "puf", "puf.py"], + "generator_lines": "1378-1398", + "disaggregator_path_parts": ["policyengine_", "us_data", "datasets", "puf", "disaggregate_puf.py"], + "disaggregator_lines": "55-60,85-123", + "operation_order": [ + "uprate", + "disaggregate_aggregate_records" + ], + "aggregate_disaggregation_seed": 42, + "forbes_top_tail_enabled": true, + "forbes_aggregate_recid": 999999, + "forbes_synthetic_record_count": 3900 + } }, { "kind": "archived_derivation_evidence", @@ -96,6 +119,25 @@ "from_year": 2015, "to_year_from_build_config": true }, + { + "kind": "derive", + "method": "decompose_e19200_by_soi_agi_band", + "raw_total": "home_mortgage_interest", + "banding_variable": "E00100", + "banding_period": "source_year_2015", + "band_alignment": "direct RECID join for regular returns; archived seed-42 aggregate-record donor allocation for the three bounded synthetic buckets; source bucket floor for the Forbes-backed $100 million-plus tail", + "archived_processed_puf_disaggregation_seed": 42, + "published_source_year": 2015, + "published_total_column": "CF", + "published_home_mortgage_column": "CH", + "mortgage_output": "home_mortgage_interest", + "non_mortgage_output": "investment_interest_expense", + "allocation": "deterministic proportional allocation within each published AGI band", + "outlier_screen_before_split": { + "variable": "home_mortgage_interest", + "maximum": 10000000 + } + }, { "kind": "fit_weighted_qrf", "predictors": [ @@ -138,6 +180,7 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -202,6 +245,7 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -225,7 +269,7 @@ "unadjusted_basis_qualified_property", "w2_wages_from_qualified_business" ], - "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. IRS disclosure aggregate rows are replaced from raw PUF aggregate totals before uprating; Forbes top-tail synthesis is not enabled. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." + "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. This build consumes that versioned H5 and does not request unsupported Forbes synthesis from the source-stage runtime. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Populace places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Populace therefore aggregates each processed donor leaf to its identifiable tax-unit total, places that total on the unit's first person, and sparsifies only the PUF support channel to the donor positive rate rather than inventing an earnings-based split or reproducing the retired both-half override. This within-unit placement is policy-neutral in PolicyEngine-US 1.764.6 because the relevant formulas sum these leaves to the tax unit. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Populace deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Populace source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." }, { "stage": "education_inputs", @@ -2303,7 +2347,7 @@ { "kind": "derived_from_prior_stages", "inputs": [ - "interest_deduction", + "home_mortgage_interest", "scf_mortgage_debt", "tenure_type" ] @@ -2313,7 +2357,7 @@ { "kind": "derive_mortgage_balance_hints", "inputs": [ - "interest_deduction", + "home_mortgage_interest", "scf_mortgage_debt", "filing_status" ] @@ -2332,17 +2376,15 @@ "deductible_mortgage_interest", "home_mortgage_interest", "mortgage_interest", - "mortgage_principal", - "investment_interest_expense" + "mortgage_principal" ], "nonnegative_outputs": [ "deductible_mortgage_interest", "home_mortgage_interest", "mortgage_interest", - "mortgage_principal", - "investment_interest_expense" + "mortgage_principal" ], - "notes": "The archived conversion maps the sole interest field used by the retired derivation, E19200, to interest_deduction, assigns the identical amount as deductible mortgage interest (archived puf.py lines 654 and 1548-1550), and derives investment_interest_expense only as their nonnegative residual (utils/mortgage_interest.py lines 268-320). The retired enhanced-CPS QRF predicted those source-identical aliases separately before conversion, so positive residuals are stochastic prediction divergence rather than observed detail. The SHA-pinned PUF 1.8.0 artifact contains 499,045 all-zero investment_interest_expense values and omits both aliases and raw E19200; all three locked ASEC inputs omit them too. A nonzero hermetic restoration would synthesize an unobserved mortgage-versus-investment split, so investment_interest_expense remains a SOURCE UNAVAILABILITY WITH EVIDENCE exclusion." + "notes": "This structural stage consumes the mortgage-only lineage already produced by puf_tax_detail's source-backed Table 2.1 E19200 decomposition. It combines that lineage with SCF mortgage-debt hints to derive mortgage balances and person-carried mortgage inputs; it neither constructs nor overwrites investment_interest_expense." }, { "stage": "acs_rent", diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 90cd93be..dd9010df 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -495,13 +495,27 @@ us_prior_year_income_summary, with_us_prior_year_income_inputs, ) +from populace.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_AGI_BANDS, + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + PufE19200AgiBand, + PufE19200InterestComponents, + split_us_puf_e19200_by_agi_band, +) +from populace.build.us_runtime.puf_source_agi import ( + PUF_AGGREGATE_DISAGGREGATION_SEED, + PUF_AGGREGATE_RECIDS, + PUF_SOURCE_YEAR, + PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, + PUF_SYNTHETIC_RECID_START, + source_year_puf_adjusted_gross_income, +) from populace.build.us_runtime.puf_support import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_E19200_HOME_MORTGAGE_SHARE, US_PUF_SUPPORT_FIT_NAME, US_PUF_SUPPORT_STAGE_NAME, PufTaxDetailChainInputs, @@ -1688,8 +1702,16 @@ "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", "PUF_TAX_DETAIL_SUPPORT_CHANNEL", "PufTaxDetailChainInputs", + "PufE19200AgiBand", + "PufE19200InterestComponents", "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", - "US_PUF_E19200_HOME_MORTGAGE_SHARE", + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", "US_PUF_SUPPORT_FIT_NAME", "US_PUF_SUPPORT_STAGE_NAME", "US_STATE_INCOME_TAX_TARGET_SPECS", @@ -1721,6 +1743,8 @@ "prepare_us_puf_tax_detail_chain_inputs", "reform_validation_payload", "source_gap_family_ids", + "split_us_puf_e19200_by_agi_band", + "source_year_puf_adjusted_gross_income", "ECPS_PARITY_KNOWN_GAPS_RESOURCE", "ECPS_PARITY_REFERENCE_RESOURCE", "EcpsParityReference", @@ -2149,12 +2173,14 @@ def to_manifest(self) -> dict[str, object]: notes=( "Itemized-deduction detail, versioned processed-PUF Section 199A " "simulation leaves (carried without redrawing), partnership SE, " - "mortgage-interest split, direct E00800/E03500 alimony, direct " - "E20500 casualty loss, and the E20400 miscellaneous-expense proxy; IRS " - "disclosure aggregate rows are " - "disaggregated from raw PUF totals before uprating, with Forbes " - "top-tail synthesis disabled; support clipped to the PUF's own " - "realized ranges." + "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " + "miscellaneous-expense proxy. The pinned processed PUF uprates its " + "raw TY2015 rows before seeded disclosure-record replacement and " + "includes a Forbes-backed 3,900-record open tail; this runtime " + "reconstructs the bounded-record AGI lineage and anchors every " + "Forbes-tail record in the final published AGI band without rerunning " + "Forbes synthesis. Support is clipped to the PUF's realized ranges." ), ), US_EDUCATION_INPUTS_STAGE_NAME: DonorSpec( diff --git a/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py b/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py index 9514917f..f97ce380 100644 --- a/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py +++ b/packages/populace-build/src/populace/build/us_runtime/l0_refit_export.py @@ -187,6 +187,7 @@ *US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, *US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, *US_HOUSING_NONCONSTANT_PERSON_COLUMNS, + "investment_interest_expense", ) US_RELEASE_REQUIRED_SPM_UNIT_SOURCE_COLUMNS = ( diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py b/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py index 0a264832..9061806d 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_aggregate_records.py @@ -107,10 +107,10 @@ # Raw E19200 is Schedule A "Interest paid deduction, TOTAL" (home # mortgage + points + QMIP + investment interest), NOT mortgage-only; # the attribute name predates that distinction and stays stable because - # it keys raw-source audit payloads. The processed donor carves this - # lineage to the mortgage-only concept (populace#515, - # US_PUF_E19200_HOME_MORTGAGE_SHARE) -- do not compare this raw audit - # figure against carved donor masses without un-carving. + # it keys raw-source audit payloads. The processed donor decomposes this + # lineage using the published TY2015 Table 2.1 component shares for each + # AGI band (populace#515); compare this raw audit figure only with the sum + # of the carved mortgage and non-mortgage donor masses. "E19200": "mortgage_interest_paid", "E19800": "charitable_cash_contributions", "E20100": "charitable_noncash_contributions", diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py new file mode 100644 index 00000000..af7bc1d0 --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/puf_interest_components.py @@ -0,0 +1,220 @@ +"""Published SOI component shares for the PUF E19200 interest deduction.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib.resources import files +from typing import Any + +import numpy as np + +_SOURCE_ASSET = "soi_table_2_1_interest_components_ty2015.json" +_AMOUNT_COLUMNS = { + "total_interest_paid_amount": "CF", + "home_mortgage_interest_amount": "CH", + "deductible_points_amount": "CN", + "qualified_mortgage_insurance_premiums_amount": "CP", + "investment_interest_amount": "CR", +} + + +@dataclass(frozen=True) +class PufE19200InterestComponents: + """One published SOI Table 2.1 interest-paid component row. + + Amounts retain the source table's unit, thousands of US dollars. The + ratios used to decompose donor records are therefore unitless. + """ + + source_row: int + total_interest_paid_amount: int + home_mortgage_interest_amount: int + deductible_points_amount: int + qualified_mortgage_insurance_premiums_amount: int + investment_interest_amount: int + + @property + def non_mortgage_interest_amount(self) -> int: + """Return the conserving residual represented by the engine leaf.""" + + return self.total_interest_paid_amount - self.home_mortgage_interest_amount + + @property + def source_cells(self) -> dict[str, str]: + """Return the literal official-workbook cell for every carried amount.""" + + return { + name: f"{column}{self.source_row}" + for name, column in _AMOUNT_COLUMNS.items() + } + + +@dataclass(frozen=True) +class PufE19200AgiBand(PufE19200InterestComponents): + """One half-open adjusted-gross-income band and its component amounts.""" + + label: str + lower_bound: float | None + upper_bound: float | None + + @property + def home_mortgage_share(self) -> float: + """Return the published mortgage share of total E19200.""" + + return self.home_mortgage_interest_amount / self.total_interest_paid_amount + + +def _components(raw: dict[str, Any]) -> PufE19200InterestComponents: + return PufE19200InterestComponents( + source_row=int(raw["source_row"]), + total_interest_paid_amount=int(raw["total_interest_paid_amount"]), + home_mortgage_interest_amount=int(raw["home_mortgage_interest_amount"]), + deductible_points_amount=int(raw["deductible_points_amount"]), + qualified_mortgage_insurance_premiums_amount=int( + raw["qualified_mortgage_insurance_premiums_amount"] + ), + investment_interest_amount=int(raw["investment_interest_amount"]), + ) + + +def _band(raw: dict[str, Any]) -> PufE19200AgiBand: + components = _components(raw) + return PufE19200AgiBand( + source_row=components.source_row, + total_interest_paid_amount=components.total_interest_paid_amount, + home_mortgage_interest_amount=components.home_mortgage_interest_amount, + deductible_points_amount=components.deductible_points_amount, + qualified_mortgage_insurance_premiums_amount=( + components.qualified_mortgage_insurance_premiums_amount + ), + investment_interest_amount=components.investment_interest_amount, + label=str(raw["label"]), + lower_bound=(None if raw["lower_bound"] is None else float(raw["lower_bound"])), + upper_bound=(None if raw["upper_bound"] is None else float(raw["upper_bound"])), + ) + + +def _load_source_asset() -> tuple[ + PufE19200InterestComponents, + tuple[PufE19200AgiBand, ...], +]: + payload = json.loads(files("populace.build.us").joinpath(_SOURCE_ASSET).read_text()) + source = payload.get("source", {}) + if ( + source.get("tax_year") != 2015 + or source.get("table") != "2.1" + or source.get("units") != "thousands_of_us_dollars" + ): + raise ValueError(f"{_SOURCE_ASSET} has unexpected source metadata.") + source_columns = { + name: details.get("workbook_column") + for name, details in source.get("columns", {}).items() + } + if source_columns != _AMOUNT_COLUMNS: + raise ValueError(f"{_SOURCE_ASSET} has unexpected workbook columns.") + + all_returns = _components(payload["all_returns"]) + bands = tuple(_band(raw) for raw in payload["agi_bands"]) + if len(bands) != 22 or [band.source_row for band in bands] != list(range(11, 33)): + raise ValueError(f"{_SOURCE_ASSET} must carry Table 2.1 rows 11 through 32.") + if bands[0].lower_bound is not None or bands[-1].upper_bound is not None: + raise ValueError(f"{_SOURCE_ASSET} must cover the full real AGI line.") + for previous, following in zip(bands[:-1], bands[1:], strict=True): + if previous.upper_bound != following.lower_bound: + raise ValueError(f"{_SOURCE_ASSET} AGI bands are not contiguous.") + for row in (all_returns, *bands): + amounts = ( + row.total_interest_paid_amount, + row.home_mortgage_interest_amount, + row.deductible_points_amount, + row.qualified_mortgage_insurance_premiums_amount, + row.investment_interest_amount, + ) + if any(amount < 0 for amount in amounts) or row.total_interest_paid_amount == 0: + raise ValueError(f"{_SOURCE_ASSET} carries an invalid amount.") + component_sum = sum(amounts[1:]) + if abs(row.total_interest_paid_amount - component_sum) > 1: + raise ValueError( + f"{_SOURCE_ASSET} row {row.source_row} violates the published " + "component identity beyond $1,000 source rounding." + ) + return all_returns, bands + + +( + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + US_PUF_E19200_AGI_BANDS, +) = _load_source_asset() + +_AGI_UPPER_BOUNDS = np.asarray( + [ + band.upper_bound + for band in US_PUF_E19200_AGI_BANDS + if band.upper_bound is not None + ], + dtype=np.float64, +) +_HOME_MORTGAGE_SHARES = np.asarray( + [band.home_mortgage_share for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, +) + + +def split_us_puf_e19200_by_agi_band( + total_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray]: + """Split each E19200 record by its published TY2015 SOI AGI-band share. + + PolicyEngine has one non-mortgage interest input, + ``investment_interest_expense``. The residual routed there is broader + than the table's investment-interest column: it also carries deductible + points and qualified mortgage-insurance premiums. Computing it as + ``total - mortgage`` preserves E19200 exactly despite published component + rounding. + """ + + total = np.asarray(total_interest_paid, dtype=np.float64) + agi = np.asarray(adjusted_gross_income, dtype=np.float64) + if total.ndim != 1 or agi.ndim != 1: + raise ValueError("E19200 and adjusted_gross_income must be one-dimensional.") + if len(total) != len(agi): + raise ValueError( + "E19200 and adjusted_gross_income must have the same record count." + ) + if not np.isfinite(total).all(): + raise ValueError("E19200 must contain only finite values.") + if not np.isfinite(agi).all(): + raise ValueError("adjusted_gross_income must contain only finite values.") + if (total < 0).any(): + raise ValueError("E19200 must be nonnegative.") + + band_index = np.searchsorted(_AGI_UPPER_BOUNDS, agi, side="right") + # Reconciled split (PR #561 review, rounds 1-2): the naive pair + # (total*share, total − total*share) is algebraically conserving but not + # bit-exact in float64 (e.g. E19200=1.53 in the top band sums to + # 1.5300000000000002). Recomputing mortgage as the complement of the + # rounded residual makes conservation exact by construction, not merely + # empirical: with share in [0, 1], either the rounded mortgage or the + # rounded residual lands in [total/2, total], so one of the two + # subtractions is exact by Sterbenz's lemma and the pair sums to + # ``total`` bit-for-bit for every accepted total except signed zero + # (an accepted -0.0 reconstructs as +0.0). The mortgage component + # deviates from + # ``total*share`` by at most the two subtractions' roundings — a bound + # at the half-ULP scale of ``total`` per step, which can amount to + # several ULPs of the smaller mortgage value itself. + mortgage = total * _HOME_MORTGAGE_SHARES[band_index] + non_mortgage = total - mortgage + mortgage = total - non_mortgage + return mortgage, non_mortgage + + +__all__ = [ + "PufE19200AgiBand", + "PufE19200InterestComponents", + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "split_us_puf_e19200_by_agi_band", +] diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py b/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py index ad2a4dbb..4f2aa1d1 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_qrf_chain.py @@ -35,17 +35,21 @@ from populace.fit import QRF, QRFChainState from populace.frame import EntitySchema, Frame, WeightKind, Weights -# v2 (populace#515): the checkpointed donor frame now carries the E19200 -> -# mortgage-only concept carve (US_PUF_E19200_HOME_MORTGAGE_SHARE). Loading -# validates only schema/digest/kind/role -- not donor construction identity -- -# so a v1 checkpoint initialized pre-carve would keep fitting and drawing the -# uncarved levels under carved code. The bump rejects every pre-carve -# checkpoint and forces re-initialization through the carved constructor. +# v2 (populace#515): the checkpointed donor frame gained the interim national +# E19200 -> mortgage-only concept carve. Loading validates only +# schema/digest/kind/role -- not donor construction identity -- so a v1 +# checkpoint initialized pre-carve would keep fitting and drawing the uncarved +# levels under carved code. # v3 (populace#516): donor construction now whole-row-screens grouped raw # mortgage-interest outliers before that carve. A v2 post-carve, pre-screen # checkpoint would otherwise still fit and draw the corrupt rows under screened # code, so it too must be rejected and rebuilt. -PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 3 +# v4 (populace#515 completion): donor construction replaces the national carve +# with published per-AGI-band shares and adds investment_interest_expense as a +# learned person input. The production target-order digest also changes, but a +# schema bump is still required for standalone custom-order checkpoints whose +# manifests do not fingerprint donor-construction semantics. +PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 4 PRIMARY_QRF_MANIFEST_FILENAME = "manifest.json" PRIMARY_QRF_DONOR_FILENAME = "donor.frame.h5" PRIMARY_QRF_RECIPIENT_FILENAME = "recipient.frame.h5" @@ -55,7 +59,7 @@ *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) PRIMARY_QRF_TARGET_ORDER_SHA256 = ( - "556d713d029840848aba548d9c991842a54ea3cdbdea650c0be57d5ec5782661" + "795519d161e6b8425fc3b64de7eb435d52d25e7c8250b5861f3bb21ab48266a3" ) _ARTIFACT_KIND = "populace_primary_puf_qrf_chain" diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py b/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py new file mode 100644 index 00000000..8b0cb2aa --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/puf_source_agi.py @@ -0,0 +1,432 @@ +"""Recover TY2015 AGI-band values aligned to the processed PUF artifact. + +The processed 2024 PUF is rooted in the 2015 IRS PUF but does not export raw +``E00100``. Most processed tax units retain their IRS ``RECID`` and therefore +join directly to raw E00100. The remaining IDs replace the four IRS disclosure +aggregate rows. This module reproduces the archived, seeded donor allocation +for the three bounded aggregate buckets; the Forbes-backed $100 million-plus +tail receives that bucket's lower bound because every such record maps to the +same final SOI Table 2.1 band ($10 million or more). +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +PUF_SOURCE_YEAR = 2015 +PUF_AGGREGATE_RECIDS = (999_996, 999_997, 999_998, 999_999) +PUF_SYNTHETIC_RECID_START = 1_000_000 +PUF_AGGREGATE_DISAGGREGATION_SEED = 42 +PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS = ( + "RECID", + "MARS", + "S006", + "E00100", + "E00200", + "P23250", + "P22250", + "E00650", + "E00300", + "E26270", + "E00900", + "E02100", + "E00400", + "E00600", +) + +_SCREENED_FIELDS = PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS[4:] +_SELECTION_POWER = 24 +_MAX_AGI_DOMINANCE = 0.20 +_AGI_CAP_100M_PLUS = 1_250_000_000.0 +_NUMERIC_TOLERANCE = 1e-9 +_AGGREGATE_BUCKET_BOUNDS = { + 999_996: (None, 0.0), + 999_997: (0.0, 10_000_000.0), + 999_998: (10_000_000.0, 100_000_000.0), + 999_999: (100_000_000.0, None), +} + + +def _source_frame( + source: str | Path | pd.DataFrame | Mapping[str, Sequence[Any]], +) -> pd.DataFrame: + if isinstance(source, (str, Path)): + source_path = Path(source) + frame = pd.read_csv( + source_path, + usecols=lambda name: name in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, + ) + else: + frame = pd.DataFrame(source).copy() + missing = [ + column for column in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS if column not in frame + ] + if missing: + raise ValueError(f"TY2015 PUF source is missing column(s): {missing}.") + frame = frame.loc[:, PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS].copy() + for column in PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + if frame[list(PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS[:4])].isna().any().any(): + raise ValueError("TY2015 PUF RECID, MARS, S006, and E00100 must be numeric.") + frame[list(_SCREENED_FIELDS)] = frame[list(_SCREENED_FIELDS)].fillna(0.0) + return frame + + +def _integral_ids(values: Sequence[Any], *, label: str) -> np.ndarray: + numeric = np.asarray(values, dtype=np.float64) + if numeric.ndim != 1 or not np.isfinite(numeric).all(): + raise ValueError(f"{label} must be a finite one-dimensional array.") + integral = numeric.astype(np.int64) + if not np.array_equal(numeric, integral.astype(np.float64)): + raise ValueError(f"{label} must contain integral values.") + if len(np.unique(integral)) != len(integral): + raise ValueError(f"{label} must be unique.") + return integral + + +def _choose_n_synthetic(population_weight: float) -> int: + return int(min(40, max(20, round(population_weight / 10)))) + + +def _assign_weights( + population_weight: float, + count: int, + rng: np.random.Generator, +) -> np.ndarray: + total_weight = int(round(population_weight)) + base = max(total_weight // count, 3) + weights = np.full(count, base, dtype=int) + remainder = total_weight - int(weights.sum()) + if remainder > 0: + weights[rng.choice(count, size=remainder, replace=False)] += 1 + elif remainder < 0: + reducible = np.where(weights > 1)[0] + reduce_count = min(-remainder, len(reducible)) + weights[rng.choice(reducible, size=reduce_count, replace=False)] -= 1 + gap = total_weight - int(weights.sum()) + if gap: + weights[0] += gap + return weights.astype(np.float64) + + +def _bucket_mask(frame: pd.DataFrame, recid: int) -> pd.Series: + agi = frame["E00100"] + if recid == 999_996: + return agi < 0 + if recid == 999_997: + return (agi >= 0) & (agi < 10_000_000) + if recid == 999_998: + return (agi >= 10_000_000) & (agi < 100_000_000) + if recid == 999_999: + return agi >= 100_000_000 + raise ValueError(f"Unknown aggregate RECID {recid}.") + + +def _eligibility_scores(frame: pd.DataFrame) -> pd.Series: + maximum = np.zeros(len(frame), dtype=np.float64) + for field in _SCREENED_FIELDS: + values = frame[field].fillna(0.0).to_numpy(dtype=np.float64) + scores = np.zeros(len(frame), dtype=np.float64) + + positive = values > 0 + reference_positive = np.sort(values[positive]) + if positive.any(): + scores[positive] = np.searchsorted( + reference_positive, values[positive], side="right" + ) / len(reference_positive) + + negative = values < 0 + reference_negative = np.sort(-values[negative]) + if negative.any(): + scores[negative] = np.maximum( + scores[negative], + np.searchsorted( + reference_negative, + -values[negative], + side="right", + ) + / len(reference_negative), + ) + maximum = np.maximum(maximum, scores) + return pd.Series(maximum, index=frame.index, dtype=np.float64) + + +def _sample_bucket_donors( + regular: pd.DataFrame, + scores: pd.Series, + *, + recid: int, + target_mean_agi: float, + count: int, + rng: np.random.Generator, +) -> pd.DataFrame: + donor_bucket = regular.loc[_bucket_mask(regular, recid)] + if donor_bucket.empty: + donor_bucket = regular + bucket_scores = scores.loc[donor_bucket.index].to_numpy(dtype=np.float64) + score_mass = np.clip(bucket_scores, 1e-6, None) ** _SELECTION_POWER + donor_abs_agi = np.abs(donor_bucket["E00100"].to_numpy(dtype=np.float64)) + target_abs_agi = max(abs(float(target_mean_agi)), 1.0) + distance = np.abs(np.log1p(donor_abs_agi) - np.log1p(target_abs_agi)) + probabilities = score_mass * np.sqrt(1.0 / (1.0 + distance)) + if not np.isfinite(probabilities).all() or probabilities.sum() <= 0: + probabilities = np.ones(len(donor_bucket), dtype=np.float64) + probabilities /= probabilities.sum() + selected_index = rng.choice( + donor_bucket.index.to_numpy(), + size=count, + replace=len(donor_bucket) < count, + p=probabilities, + ) + return donor_bucket.loc[selected_index].reset_index(drop=True) + + +def _project_weighted_sum_to_bounds( + values: np.ndarray, + weights: np.ndarray, + target_total: float, + lower: np.ndarray, + upper: np.ndarray, +) -> np.ndarray: + projected = np.clip(values.astype(np.float64), lower, upper) + for _ in range(50): + residual = float(target_total - np.dot(projected, weights)) + if abs(residual) <= 1e-6: + return projected + slack = upper - projected if residual > 0 else projected - lower + free = slack > _NUMERIC_TOLERANCE + if not free.any(): + break + basis = np.abs(projected[free]) + if basis.sum() <= _NUMERIC_TOLERANCE: + basis = np.ones(free.sum(), dtype=np.float64) + denominator = float(np.dot(weights[free], basis)) + if denominator <= _NUMERIC_TOLERANCE: + basis = np.ones(free.sum(), dtype=np.float64) + denominator = float(weights[free].sum()) + delta = residual * basis / denominator + if residual > 0: + delta = np.minimum(delta, slack[free]) + else: + delta = -np.minimum(-delta, slack[free]) + projected[free] += delta + projected = np.clip(projected, lower, upper) + + residual = float(target_total - np.dot(projected, weights)) + if abs(residual) > 1e-6: + slack = upper - projected if residual > 0 else projected - lower + free = np.where(slack > _NUMERIC_TOLERANCE)[0] + if len(free): + best = free[np.argmax(slack[free] * weights[free])] + projected[best] = np.clip( + projected[best] + residual / weights[best], + lower[best], + upper[best], + ) + return projected + + +def _allocate_agi( + donor_agi: np.ndarray, + weights: np.ndarray, + *, + recid: int, + target_total: float, +) -> np.ndarray: + dominance_cap = _MAX_AGI_DOMINANCE * abs(target_total) / weights + lower_bound, upper_bound = _AGGREGATE_BUCKET_BOUNDS[recid] + if recid == 999_996: + lower = -dominance_cap + upper = np.zeros(len(weights), dtype=np.float64) + else: + lower = np.full(len(weights), float(lower_bound), dtype=np.float64) + numeric_upper = ( + _AGI_CAP_100M_PLUS if upper_bound is None else float(upper_bound) + ) + upper = np.minimum( + np.full(len(weights), numeric_upper, dtype=np.float64), + dominance_cap, + ) + + base = np.abs(np.asarray(donor_agi, dtype=np.float64)) + active = base > _NUMERIC_TOLERANCE + if not active.any(): + active = np.ones(len(base), dtype=bool) + allocated = np.zeros(len(base), dtype=np.float64) + magnitudes = base[active] + if magnitudes.sum() <= _NUMERIC_TOLERANCE: + magnitudes = np.ones(active.sum(), dtype=np.float64) + denominator = float(np.dot(weights[active], magnitudes)) + if denominator <= _NUMERIC_TOLERANCE: + magnitudes = np.ones(active.sum(), dtype=np.float64) + denominator = float(weights[active].sum()) + allocated[active] = ( + np.sign(target_total) * magnitudes * abs(target_total) / denominator + ) + return _project_weighted_sum_to_bounds( + allocated, + weights, + target_total, + lower, + upper, + ) + + +def source_year_puf_adjusted_gross_income( + source: str | Path | pd.DataFrame | Mapping[str, Sequence[Any]], + *, + processed_tax_unit_ids: Sequence[Any], + processed_tax_unit_weights: Sequence[Any], +) -> np.ndarray: + """Return TY2015 AGI banding values in processed-PUF tax-unit order. + + Regular tax units carry literal IRS PUF E00100. The first three synthetic + disclosure buckets reproduce the archived seeded donor allocation. The + open Forbes-backed tail receives $100 million, its source bucket floor; + this is an AGI-band anchor rather than a fabricated point estimate, and all + such rows belong to Table 2.1's final "$10 million or more" band. + """ + + frame = _source_frame(source) + source_ids = _integral_ids(frame["RECID"], label="TY2015 PUF RECID") + frame["RECID"] = source_ids + if set(frame.loc[frame["MARS"] == 0, "RECID"]) != set(PUF_AGGREGATE_RECIDS): + raise ValueError("TY2015 PUF must contain exactly the four aggregate RECIDs.") + + processed_ids = _integral_ids( + processed_tax_unit_ids, + label="Processed PUF tax_unit_id", + ) + processed_weights = np.asarray(processed_tax_unit_weights, dtype=np.float64) + if ( + processed_weights.ndim != 1 + or len(processed_weights) != len(processed_ids) + or not np.isfinite(processed_weights).all() + or (processed_weights <= 0).any() + ): + raise ValueError( + "Processed PUF tax-unit weights must be finite, positive, and " + "aligned one-for-one with tax_unit_id." + ) + + aggregate_mask = frame["RECID"].isin(PUF_AGGREGATE_RECIDS) + regular = frame.loc[~aggregate_mask].copy() + regular_ids = regular["RECID"].to_numpy(dtype=np.int64) + if len(processed_ids) <= len(regular_ids) or not np.array_equal( + processed_ids[: len(regular_ids)], + regular_ids, + ): + raise ValueError( + "Processed PUF regular RECID order does not match the TY2015 source." + ) + + synthetic_ids = processed_ids[len(regular_ids) :] + expected_synthetic_ids = np.arange( + PUF_SYNTHETIC_RECID_START, + PUF_SYNTHETIC_RECID_START + len(synthetic_ids), + dtype=np.int64, + ) + if not np.array_equal(synthetic_ids, expected_synthetic_ids): + raise ValueError( + "Processed PUF synthetic tax_unit_id values are not the archived " + "contiguous aggregate-record replacement IDs." + ) + + source_regular_weights = regular["S006"].to_numpy(dtype=np.float64) / 100.0 + if (source_regular_weights <= 0).any(): + raise ValueError("TY2015 regular PUF S006 weights must be positive.") + weight_ratios = processed_weights[: len(regular_ids)] / source_regular_weights + population_scale = float(np.median(weight_ratios)) + if not np.allclose( + weight_ratios, + population_scale, + rtol=1e-12, + atol=1e-12, + ): + raise ValueError( + "Processed PUF regular weights do not share one source uprating factor." + ) + + aggregate = frame.loc[aggregate_mask].set_index("RECID") + aggregate_population = { + recid: float(aggregate.loc[recid, "S006"]) / 100.0 * population_scale + for recid in PUF_AGGREGATE_RECIDS + } + bounded_counts = { + recid: _choose_n_synthetic(aggregate_population[recid]) + for recid in PUF_AGGREGATE_RECIDS[:3] + } + top_count = len(synthetic_ids) - sum(bounded_counts.values()) + if top_count <= 0: + raise ValueError( + "Processed PUF does not contain the archived open-tail replacement rows." + ) + counts = { + **bounded_counts, + PUF_AGGREGATE_RECIDS[-1]: top_count, + } + + output = np.empty(len(processed_ids), dtype=np.float64) + output[: len(regular_ids)] = regular["E00100"].to_numpy(dtype=np.float64) + if not np.isfinite(output[: len(regular_ids)]).all(): + raise ValueError("TY2015 regular PUF E00100 must be finite.") + + scores = _eligibility_scores(regular) + rng = np.random.default_rng(PUF_AGGREGATE_DISAGGREGATION_SEED) + cursor = len(regular_ids) + for recid in PUF_AGGREGATE_RECIDS[:3]: + count = counts[recid] + assigned_weights = _assign_weights( + aggregate_population[recid], + count, + rng, + ) + processed_bucket_weights = processed_weights[cursor : cursor + count] + if not np.array_equal(assigned_weights, processed_bucket_weights): + raise ValueError( + f"Processed PUF synthetic weights for aggregate RECID {recid} " + "do not match the archived seeded assignment." + ) + target_mean_agi = float(aggregate.loc[recid, "E00100"]) + selected = _sample_bucket_donors( + regular, + scores, + recid=recid, + target_mean_agi=target_mean_agi, + count=count, + rng=rng, + ) + output[cursor : cursor + count] = _allocate_agi( + selected["E00100"].to_numpy(dtype=np.float64), + assigned_weights, + recid=recid, + target_total=aggregate_population[recid] * target_mean_agi, + ) + cursor += count + + top_weights = processed_weights[cursor:] + expected_top_weight = int(round(aggregate_population[999_999])) + if not np.isclose(top_weights.sum(), expected_top_weight, rtol=0.0, atol=1e-9): + raise ValueError( + "Processed PUF open-tail weights do not preserve the archived " + "aggregate-row population." + ) + output[cursor:] = _AGGREGATE_BUCKET_BOUNDS[999_999][0] + if not np.isfinite(output).all(): + raise ValueError("Recovered TY2015 PUF AGI banding values must be finite.") + return output + + +__all__ = [ + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", + "source_year_puf_adjusted_gross_income", +] diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index 4e40d049..b5a4ae30 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -17,6 +17,9 @@ import pandas as pd from populace.build.gates import FitWeightRecord +from populace.build.us_runtime.puf_interest_components import ( + split_us_puf_e19200_by_agi_band, +) from populace.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, @@ -33,7 +36,6 @@ "PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS", "PUF_TAX_DETAIL_SUPPORT_CHANNEL", "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", - "US_PUF_E19200_HOME_MORTGAGE_SHARE", "US_PUF_SUPPORT_FIT_NAME", "US_PUF_SUPPORT_STAGE_NAME", "assert_formula_owned_blocklist_current", @@ -57,16 +59,6 @@ #: this fit by name. US_PUF_SUPPORT_FIT_NAME = "us_puf_tax_detail_support" -# Interim populace#515 carve: SOI Pub 1304 TY2015 Table 2.1 reports -# $283,004,465 thousand of home-mortgage interest within $304,461,163 thousand -# of total interest paid. Apply that concept share per donor record before the -# QRF learns levels and realized support. The #486 ``support_value_repairs`` -# surface is instead a release-time total pin, so it cannot express this -# donor-column concept correction (#492). A uniform carve cannot identify the -# records carrying the removed points, qualified mortgage-insurance premiums, -# or investment interest, so ``investment_interest_expense`` remains all-zero. -US_PUF_E19200_HOME_MORTGAGE_SHARE = 283_004_465 / 304_461_163 - # populace#516 donor outlier screen: $10M of annual home-mortgage interest # implies roughly a $250M mortgage at 4%, not a genuine Schedule A return; the # pinned artifact's maximum REAL-scale unit values are only low single-digit @@ -83,6 +75,7 @@ # Reserved internal column used to thread the grouped RAW mortgage value to # the outlier screen; never a legal requested output (populace#516). _MORTGAGE_OUTLIER_SCREEN_COLUMN = "_raw_home_mortgage_interest_for_outlier_screen" +_E19200_AGI_BAND_COLUMN = "_adjusted_gross_income_for_e19200_band" _DEFAULT_SUPPORT_CHANNELS = ( BASE_ASEC_SUPPORT_CHANNEL, @@ -151,6 +144,7 @@ def target_order(self) -> tuple[str, ...]: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -305,6 +299,7 @@ def target_order(self) -> tuple[str, ...]: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -509,6 +504,7 @@ def clone_us_frame_for_puf_support( def puf_tax_unit_donor_from_arrays( arrays: Mapping[str, Sequence[Any]], *, + adjusted_gross_income: Sequence[Any] | None = None, person_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, tax_unit_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) -> pd.DataFrame: @@ -521,6 +517,9 @@ def puf_tax_unit_donor_from_arrays( Args: arrays: Mapping from column name to array-like values. + adjusted_gross_income: Explicit tax-unit-grain AGI used to select the + published TY2015 SOI Table 2.1 component share for each record. + Required whenever the processed PUF carries nonzero E19200. person_outputs: Person-grain PE input variables to aggregate by tax unit. tax_unit_outputs: Tax-unit-grain PE input variables to carry or derive. @@ -545,11 +544,62 @@ def puf_tax_unit_donor_from_arrays( "filing_status_code": _filing_status_codes(arrays["filing_status"]), } ) + if adjusted_gross_income is not None: + # Strict conversion (PR #561 review findings, rounds 1-2): + # _numeric_array coerces nonnumeric/NaN to 0.0, and errors="raise" + # alone is parse-strict but not real-number-strict — datetime and + # timedelta arrays (including NaT) convert to finite epoch/sentinel + # integers, complex drops its imaginary part, and booleans pass. + # Any of those would route records to the wrong AGI band with the + # finiteness check below none the wiser, so non-real dtypes are + # rejected up front. NaN/inf survive strict conversion and fail the + # explicit check. + agi_series = pd.Series(adjusted_gross_income) + if agi_series.dtype.kind not in "iufO": + raise TypeError( + "adjusted_gross_income must be real-valued; got dtype " + f"{agi_series.dtype}." + ) + if agi_series.dtype.kind == "O": + # Element screen (PR #561 review, round 3): object and + # categorical arrays reach the strict parse below with kind + # "O", where object-wrapped booleans become 1.0/0.0 and + # object-wrapped complex keeps its real part with only a + # warning. Allow only element types whose float conversion is + # faithful: real numbers, Decimal, strings (strict-parsed), + # and missing values (which fail the finiteness check). + inferred = pd.api.types.infer_dtype(agi_series, skipna=True) + if inferred not in { + "integer", + "floating", + "mixed-integer-float", + "decimal", + "string", + "empty", + }: + raise TypeError( + f"adjusted_gross_income must be real-valued; got {inferred} values." + ) + agi = pd.to_numeric(agi_series, errors="raise").to_numpy(dtype=np.float64) + if len(agi) != len(tax_unit_id): + raise ValueError( + "adjusted_gross_income must align one-for-one with tax_unit_id." + ) + if not np.isfinite(agi).all(): + raise ValueError("adjusted_gross_income must contain only finite values.") + tax_unit[_E19200_AGI_BAND_COLUMN] = agi person = pd.DataFrame({"tax_unit_id": person_tax_unit_id}) - if _MORTGAGE_OUTLIER_SCREEN_COLUMN in (*person_outputs, *tax_unit_outputs): + reserved_outputs = { + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + } + requested_reserved = reserved_outputs.intersection( + (*person_outputs, *tax_unit_outputs) + ) + if requested_reserved: raise ValueError( - f"{_MORTGAGE_OUTLIER_SCREEN_COLUMN!r} is reserved for the donor " - "mortgage outlier screen and cannot be a requested output." + f"{sorted(requested_reserved)!r} are reserved for donor E19200 " + "processing and cannot be requested outputs." ) raw_home_mortgage_interest = _person_source_values( arrays, @@ -611,29 +661,80 @@ def puf_tax_unit_donor_from_arrays( tax_unit[_MORTGAGE_OUTLIER_SCREEN_COLUMN] < US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING ) - tax_unit = ( - tax_unit.loc[retained] - .drop(columns=[_MORTGAGE_OUTLIER_SCREEN_COLUMN]) - .reset_index(drop=True) - ) - # Keep the carve BEFORE _add_predictor_aliases: no mortgage predictor - # alias exists today, but if one is ever added it must derive from the - # carved column (aliases skip already-present columns, so a post-alias - # carve would leave a stale uncarved predictor copy). - _carve_us_puf_e19200_home_mortgage_share(tax_unit) + tax_unit = tax_unit.loc[retained].reset_index(drop=True) + # Keep the split BEFORE _add_predictor_aliases: no mortgage predictor alias + # exists today, but if one is ever added it must derive from the decomposed + # column (aliases skip already-present columns, so a post-alias split would + # leave a stale total-interest predictor copy). + _split_us_puf_e19200_components(tax_unit) _add_predictor_aliases(tax_unit, PUF_TAX_DETAIL_DEFAULT_PREDICTORS) return tax_unit -def _carve_us_puf_e19200_home_mortgage_share(donor: pd.DataFrame) -> None: - """Carve E19200-lineage donor columns to the mortgage-only concept.""" +def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: + """Split raw E19200 into mortgage and modeled non-mortgage components.""" + + if _MORTGAGE_OUTLIER_SCREEN_COLUMN not in donor: + donor.drop(columns=[_E19200_AGI_BAND_COLUMN], errors="ignore", inplace=True) + return + raw_total = donor[_MORTGAGE_OUTLIER_SCREEN_COLUMN].to_numpy( + dtype=np.float64, + copy=False, + ) + has_nonzero_e19200 = bool((raw_total != 0).any()) + if has_nonzero_e19200 and _E19200_AGI_BAND_COLUMN not in donor: + raise ValueError( + "adjusted_gross_income is required to split nonzero PUF E19200 " + "records by the published SOI AGI bands." + ) + if "investment_interest_expense" in donor: + existing = donor["investment_interest_expense"].to_numpy( + dtype=np.float64, + copy=False, + ) + if (existing != 0).any(): + raise ValueError( + "Processed PUF already carries nonzero investment_interest_expense; " + "refusing to overwrite independently sourced values with the " + "E19200 residual." + ) + if not has_nonzero_e19200: + if "investment_interest_expense" in donor: + donor["investment_interest_expense"] = np.zeros_like(raw_total) + donor.drop( + columns=[ + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + ], + errors="ignore", + inplace=True, + ) + return + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + raw_total, + donor[_E19200_AGI_BAND_COLUMN].to_numpy(dtype=np.float64, copy=False), + ) + band_share = np.divide( + mortgage, + raw_total, + out=np.ones_like(mortgage), + where=raw_total != 0, + ) for column in _US_PUF_E19200_LINEAGE_DONOR_COLUMNS: if column in donor: donor[column] = ( - donor[column].to_numpy(dtype=np.float64, copy=False) - * US_PUF_E19200_HOME_MORTGAGE_SHARE + donor[column].to_numpy(dtype=np.float64, copy=False) * band_share ) + if "investment_interest_expense" in donor: + donor["investment_interest_expense"] = non_mortgage + donor.drop( + columns=[ + _MORTGAGE_OUTLIER_SCREEN_COLUMN, + _E19200_AGI_BAND_COLUMN, + ], + inplace=True, + ) def impute_us_puf_tax_detail_support( diff --git a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py index b5a38dd1..3e7ad855 100644 --- a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py +++ b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py @@ -182,6 +182,7 @@ *US_HOUSING_INPUTS_OUTPUT_COLUMNS, *US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS, *US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, + "investment_interest_expense", "spm_unit_pre_subsidy_childcare_expenses", "household_weight", "unreimbursed_business_employee_expenses", diff --git a/packages/populace-build/tests/test_puf_qrf_chain.py b/packages/populace-build/tests/test_puf_qrf_chain.py index b0692097..70f2b02e 100644 --- a/packages/populace-build/tests/test_puf_qrf_chain.py +++ b/packages/populace-build/tests/test_puf_qrf_chain.py @@ -152,7 +152,8 @@ def test_primary_qrf_production_target_order_is_locked() -> None: *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) assert PRIMARY_QRF_TARGET_ORDER == expected - assert len(PRIMARY_QRF_TARGET_ORDER) == 64 + assert len(PRIMARY_QRF_TARGET_ORDER) == 65 + assert "investment_interest_expense" in PRIMARY_QRF_TARGET_ORDER digest = hashlib.sha256( json.dumps(list(PRIMARY_QRF_TARGET_ORDER), separators=(",", ":")).encode() ).hexdigest() @@ -244,7 +245,7 @@ def test_target_subprocess_chain_matches_monolith_raw_bits_and_final_frame( run_primary_puf_qrf_chain(checkpoint_dir) -def test_primary_qrf_rejects_pre_carve_schema_versions( +def test_primary_qrf_rejects_pre_decomposition_schema_versions( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -277,10 +278,10 @@ def test_primary_qrf_rejects_pre_carve_schema_versions( manifest_path = checkpoint_dir / "manifest.json" original_manifest = json.loads(manifest_path.read_text()) assert original_manifest["schema_version"] == PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION - # Every stale version must reject -- v1 (pre-carve) AND v2 (post-carve, - # pre-screen): a loader relaxed to accept {2, 3} would pass a v1-only pin - # while resurrecting the exact checkpoint populace#516 invalidates. - for stale_version in (1, 2): + # Every stale version must reject: pre-carve v1, pre-screen v2, and + # national-carve/all-zero-investment v3. This protects custom target-order + # standalone checkpoints whose manifests do not hash donor construction. + for stale_version in (1, 2, 3): stale_manifest = dict(original_manifest) stale_manifest["schema_version"] = stale_version manifest_path.write_text(json.dumps(stale_manifest)) @@ -294,7 +295,7 @@ def test_primary_qrf_rejects_pre_carve_schema_versions( with h5py.File(target_path, mode="r") as h5: pristine_metadata = json.loads(bytes(h5["metadata_json"][...]).decode()) assert pristine_metadata["schema_version"] == PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION - for stale_version in (1, 2): + for stale_version in (1, 2, 3): metadata = dict(pristine_metadata) metadata["schema_version"] = stale_version with h5py.File(target_path, mode="r+") as h5: diff --git a/packages/populace-build/tests/test_release_input_coverage.py b/packages/populace-build/tests/test_release_input_coverage.py index 2f226f64..07db4cf2 100644 --- a/packages/populace-build/tests/test_release_input_coverage.py +++ b/packages/populace-build/tests/test_release_input_coverage.py @@ -44,6 +44,9 @@ us_release_input_coverage_gate, us_release_reform_coverage_probes, ) +from populace.build.us_runtime.release_input_coverage import ( + RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS, +) from populace.frame import EntitySchema, Frame, WeightKind, Weights _REPO_ROOT = Path(__file__).resolve().parents[3] @@ -564,6 +567,13 @@ def test_ssi_assets_are_required_without_exclusion(self) -> None: assert asset in manifest.required_columns assert asset not in manifest.reviewed_exclusions + def test_restored_investment_interest_is_required_without_exclusion(self) -> None: + manifest = load_release_input_coverage_manifest() + column = "investment_interest_expense" + assert column in RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS + assert column in manifest.required_columns + assert column not in manifest.reviewed_exclusions + def test_post_reference_ssi_disability_criterion_has_unique_probe(self) -> None: manifest = load_release_input_coverage_manifest() column = "meets_ssi_disability_criteria" @@ -1220,8 +1230,7 @@ def test_shipped_overtime_probes_cite_the_jct_ledger_anchor(self) -> None: reason = by_id[probe_id].reason assert "JCX-35-25" in reason assert ( - "jct.obbba_title_vii.fy2026.no_tax_on_overtime.revenue_effect" - in reason + "jct.obbba_title_vii.fy2026.no_tax_on_overtime.revenue_effect" in reason ) assert "-$32.806 billion" in reason neutralization = by_id["fsla_overtime_premium_neutralization"].reason @@ -1238,9 +1247,7 @@ def test_shipped_tips_probes_cite_the_jct_ledger_anchor(self) -> None: for probe_id in ("obbba_no_tax_on_tips", "tip_income_neutralization"): reason = by_id[probe_id].reason assert "JCX-35-25" in reason - assert ( - "jct.obbba_title_vii.fy2026.no_tax_on_tips.revenue_effect" in reason - ) + assert "jct.obbba_title_vii.fy2026.no_tax_on_tips.revenue_effect" in reason assert "-$10.121 billion" in reason neutralization = by_id["tip_income_neutralization"].reason assert "certified Build N" in neutralization diff --git a/packages/populace-build/tests/test_us_buildj_base_script.py b/packages/populace-build/tests/test_us_buildj_base_script.py index 111891c7..8ef78884 100644 --- a/packages/populace-build/tests/test_us_buildj_base_script.py +++ b/packages/populace-build/tests/test_us_buildj_base_script.py @@ -34,6 +34,7 @@ def test_cache_rejects_bases_predating_salt_refund_and_energy_subsidy() -> None: def test_cache_rejects_bases_predating_measured_medicare_take_up() -> None: required = _cache_required_columns() + assert "investment_interest_expense" in required["person"] assert "takes_up_medicare_if_eligible" in required["person"] assert "workers_compensation" in required["person"] assert "would_claim_wic" in required["person"] diff --git a/packages/populace-build/tests/test_us_capital_gain_details.py b/packages/populace-build/tests/test_us_capital_gain_details.py index cd8638ee..1b766129 100644 --- a/packages/populace-build/tests/test_us_capital_gain_details.py +++ b/packages/populace-build/tests/test_us_capital_gain_details.py @@ -3,12 +3,7 @@ from __future__ import annotations import importlib.util -import json -from hashlib import sha256 -from importlib.resources import files -from pathlib import Path -import h5py import numpy as np import pandas as pd import pytest @@ -48,7 +43,6 @@ not policyengine_us_installed, reason="requires the policyengine-us [us] extra (build environment)", ) -ROOT = Path(__file__).resolve().parents[3] class _ResolvedWeights: @@ -273,88 +267,9 @@ def test_release_wiring_promotes_only_source_backed_detail_leaves() -> None: RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS ) assert set(US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS) <= manifest.required_columns - assert "investment_interest_expense" in manifest.reviewed_exclusions - assert manifest.reviewed_exclusions["investment_interest_expense"].startswith( - "SOURCE UNAVAILABILITY WITH EVIDENCE:" - ) - - -def test_investment_interest_exclusion_has_machine_reviewable_source_evidence() -> None: - path = files("populace.build.us").joinpath("ecps_parity_known_gaps.json") - entry = json.loads(path.read_text(encoding="utf-8"))["known_gaps"][ - "investment_interest_expense" - ] - - assert entry["reason"].startswith("SOURCE UNAVAILABILITY WITH EVIDENCE:") - evidence = entry["evidence"] - assert evidence["classification"] == "source_unavailability" - assert evidence["retired_derivation"]["lines"] == "141-155,268-320,327-349" - assert evidence["required_intermediates"] == [ - "interest_deduction", - "deductible_mortgage_interest", - ] - assert evidence["processed_puf"]["positive_values"] == 0 - assert evidence["processed_puf"]["missing_columns"] == [ - "interest_deduction", - "deductible_mortgage_interest", - "E19200", - ] - assert all( - item["missing_columns"] - == ["interest_deduction", "deductible_mortgage_interest", "E19200"] - for item in evidence["hermetic_asec_inputs"] - ) - - -def test_investment_interest_evidence_matches_build_j_artifact_pins() -> None: - entry = json.loads( - files("populace.build.us") - .joinpath("ecps_parity_known_gaps.json") - .read_text(encoding="utf-8") - )["known_gaps"]["investment_interest_expense"] - evidence = entry["evidence"] - summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - - assert evidence["processed_puf"]["sha256"] == summary["puf_sha256"] - assert { - item["filename"]: item["sha256"] for item in evidence["hermetic_asec_inputs"] - } == { - Path(item["path"]).name: item["sha256"] - for item in summary["base_source"]["sources"] - } - - -def _sha256(path: Path) -> str: - digest = sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def test_mounted_puf_artifact_confirms_zero_and_missing_source() -> None: - entry = json.loads( - files("populace.build.us") - .joinpath("ecps_parity_known_gaps.json") - .read_text(encoding="utf-8") - )["known_gaps"]["investment_interest_expense"] - evidence = entry["evidence"]["processed_puf"] - summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - path = Path(summary["puf_h5"]) - if not path.is_file(): - pytest.skip("SHA-locked PUF artifact is not mounted in this environment") - - assert _sha256(path) == evidence["sha256"] - with h5py.File(path) as h5: - assert set(evidence["missing_columns"]).isdisjoint(h5.keys()) - values = h5["investment_interest_expense"][...] - assert len(values) == evidence["rows"] - assert int(np.count_nonzero(values > 0.0)) == evidence["positive_values"] - assert float(values.sum()) == evidence["weighted_total"] + assert "investment_interest_expense" in RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS + assert "investment_interest_expense" in manifest.required_columns + assert "investment_interest_expense" not in manifest.reviewed_exclusions def test_shipped_neutralization_probes_bind_each_restored_leaf() -> None: diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 76be49f9..9591c302 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -6111,10 +6111,21 @@ def test_target_materialization_cache_rejects_pre_557_identities(tmp_path) -> No ) -def test_soi_eitc_child_targets_materialize_distinct_child_slices( +def test_soi_filtered_targets_keep_mortgage_and_broad_interest_distinct( monkeypatch, ) -> None: + from populace.build.us_runtime import split_us_puf_e19200_by_agi_band + builder = _load_builder_module() + e19200_total = np.asarray([100.0, 200.0, 300.0, 400.0]) + source_year_agi = np.asarray([-5_000.0, 20_000.0, 100_000.0, 10_000_000.0]) + mortgage_interest, non_mortgage_interest = split_us_puf_e19200_by_agi_band( + e19200_total, source_year_agi + ) + assert np.all(non_mortgage_interest > 0) + broader_interest = mortgage_interest + non_mortgage_interest + np.testing.assert_array_equal(broader_interest, e19200_total) + frame = Frame( { "person": pd.DataFrame( @@ -6442,6 +6453,22 @@ def eitc_spec(name, measure, child_filter, *, count=False, variable="eitc"): "itemized_only": "true", }, ), + TargetSpec( + name="home_mortgage_interest_amount", + entity="household", + measure="home_mortgage_interest_amount", + value=1.0, + source="fixture", + family="irs_soi", + metadata={ + "variable": "deductible_mortgage_interest", + "agi_lower_bound": "-inf", + "agi_upper_bound": "inf", + "filing_status": "All", + "source_measure_id": "home_mortgage_interest_amount", + "itemized_only": "true", + }, + ), ) class FakeVariable: @@ -6460,6 +6487,7 @@ class FakeSystem: "eitc_child_count", "itemized_taxable_income_deductions", "charitable_deduction", + "deductible_mortgage_interest", "interest_deduction", "medical_expense_deduction", "real_estate_taxes", @@ -6498,7 +6526,8 @@ def calculate(self, variable, *, period, **kwargs): [1_000.0, 2_000.0, 3_000.0, 4_000.0] ), "charitable_deduction": np.asarray([10.0, 20.0, 30.0, 40.0]), - "interest_deduction": np.asarray([1.0, 2.0, 3.0, 4.0]), + "deductible_mortgage_interest": mortgage_interest, + "interest_deduction": broader_interest, "medical_expense_deduction": np.asarray([100.0, 200.0, 300.0, 400.0]), "real_estate_taxes": np.asarray([5_000.0, 6_000.0, 7_000.0, 8_000.0]), "salt_deduction": np.asarray([500.0, 600.0, 700.0, 800.0]), @@ -6530,6 +6559,7 @@ def _invalidate_all_caches(self): "itemized_taxable_income_deductions" ), "charitable_deduction": "charitable_deduction", + "deductible_mortgage_interest": "deductible_mortgage_interest", "interest_deduction": "interest_deduction", "medical_expense_deduction": "medical_expense_deduction", "real_estate_taxes": "real_estate_taxes", @@ -6585,9 +6615,23 @@ def _invalidate_all_caches(self): ) assert np.array_equal(household["charitable_amount"], np.asarray([20.0, 0.0])) assert np.array_equal( - household["interest_paid_deduction_amount"], np.asarray([2.0, 0.0]) + household["interest_paid_deduction_amount"], + np.asarray([broader_interest[1], 0.0]), + ) + assert np.array_equal( + household["home_mortgage_interest_amount"], + np.asarray([mortgage_interest[1], 0.0]), + ) + np.testing.assert_allclose( + household["interest_paid_deduction_amount"] + - household["home_mortgage_interest_amount"], + np.asarray([non_mortgage_interest[1], 0.0]), + ) + assert not np.array_equal( + household["home_mortgage_interest_amount"], + household["interest_paid_deduction_amount"], ) - assert len(registry) == 20 + assert len(registry) == 21 assert compilation["dropped_target_names"] == [] diff --git a/packages/populace-build/tests/test_us_fiscal_targets.py b/packages/populace-build/tests/test_us_fiscal_targets.py index 0fa469b4..c72e940f 100644 --- a/packages/populace-build/tests/test_us_fiscal_targets.py +++ b/packages/populace-build/tests/test_us_fiscal_targets.py @@ -31,7 +31,7 @@ "charitable_deduction": ( "jct.tax_expenditures.cy2024.charitable_deduction.revenue_loss" ), - "interest_deduction": ( + "deductible_mortgage_interest": ( "jct.tax_expenditures.cy2024.deductible_mortgage_interest.revenue_loss" ), "qualified_business_income_deduction": ( @@ -807,6 +807,20 @@ def test_jct_tax_expenditure_references_are_simple_income_tax_reforms() -> None: assert spec.neutralized_variable +def test_jct_mortgage_reform_neutralizes_only_the_mortgage_component() -> None: + mortgage_target = ( + "jct.tax_expenditures.cy2024.deductible_mortgage_interest.revenue_loss" + ) + reform = next( + spec + for spec in US_JCT_TAX_EXPENDITURE_REFORMS + if spec.target_name == mortgage_target + ) + + assert reform.neutralized_variable == "deductible_mortgage_interest" + assert reform.neutralized_variable != "interest_deduction" + + def test_jct_reform_objects_satisfy_their_own_coverage_requirement() -> None: for spec in US_JCT_TAX_EXPENDITURE_REFORMS: result = target_profile_coverage_gate([spec], [spec.coverage_requirement()]) diff --git a/packages/populace-build/tests/test_us_l0_refit_export.py b/packages/populace-build/tests/test_us_l0_refit_export.py index 9ba5d4cb..2dfa1f0e 100644 --- a/packages/populace-build/tests/test_us_l0_refit_export.py +++ b/packages/populace-build/tests/test_us_l0_refit_export.py @@ -69,6 +69,7 @@ def _us_frame(**person_extra: object) -> Frame: "casualty_loss": [0.0, 2_500.0, 0.0], "unreimbursed_business_employee_expenses": [1_200.0, 0.0, 800.0], "investment_income_elected_form_4952": [0.0, 500.0, 250.0], + "investment_interest_expense": [0.0, 800.0, 200.0], "salt_refund_income": [0.0, 1_200.0, 400.0], "long_term_capital_gains_on_collectibles": [0.0, 2_500.0, 1_000.0], "qualified_tuition_expenses": [1_000.0, 0.0, 2_500.0], @@ -643,6 +644,28 @@ def test_required_us_release_source_columns_enforces_form_4952_signal() -> None: assert_required_us_release_source_columns(raw_frame) +def test_required_us_release_source_columns_enforces_investment_interest_signal() -> ( + None +): + frame = _us_frame() + raw_people = frame.table("person").copy() + raw_people["investment_interest_expense"] = 0.0 + raw_frame = Frame( + { + **{entity: frame.table(entity).copy() for entity in frame.schema.entities}, + "person": raw_people, + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + + with pytest.raises( + ValueError, + match="person.investment_interest_expense: not nonconstant", + ): + assert_required_us_release_source_columns(raw_frame) + + @pytest.mark.parametrize( "column", [ @@ -1066,6 +1089,7 @@ def write_dataset(self, bundle, path, period): "investment_income_elected_form_4952" in summary["required_person_source_columns"] ) + assert "investment_interest_expense" in summary["required_person_source_columns"] assert ( "other_health_insurance_premiums" in summary["required_person_source_columns"] ) diff --git a/packages/populace-build/tests/test_us_plan.py b/packages/populace-build/tests/test_us_plan.py index 453ad6cd..c5613679 100644 --- a/packages/populace-build/tests/test_us_plan.py +++ b/packages/populace-build/tests/test_us_plan.py @@ -846,8 +846,9 @@ def test_puf_stage_outputs_match_runtime_defaults(self) -> None: assert "educator_expense" in stage.nonnegative_outputs assert "educator_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS - def test_puf_stage_disaggregates_aggregate_records_before_uprating(self) -> None: - operations = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"].operations + def test_puf_stage_distinguishes_runtime_prefix_from_artifact_lineage(self) -> None: + stage = US_SOURCE_MANIFEST.stage_map()["puf_tax_detail"] + operations = stage.operations kinds = [operation.kind for operation in operations] assert ( kinds.index("read_table") @@ -904,7 +905,45 @@ def test_puf_stage_disaggregates_aggregate_records_before_uprating(self) -> None "amount_columns": "irs_puf_amount_columns", "seed_from_build_config": True, } - assert "forbes" not in str(operation.parameters).lower() + assert "use_forbes_top_tail" not in operation.parameters + artifact = next( + artifact + for artifact in stage.artifacts + if artifact["kind"] == "versioned_derived_microdata" + ) + assert artifact["lineage"] == { + "archived_commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "generator_path_parts": [ + "policyengine_", + "us_data", + "datasets", + "puf", + "puf.py", + ], + "generator_lines": "1378-1398", + "disaggregator_path_parts": [ + "policyengine_", + "us_data", + "datasets", + "puf", + "disaggregate_puf.py", + ], + "disaggregator_lines": "55-60,85-123", + "operation_order": ["uprate", "disaggregate_aggregate_records"], + "aggregate_disaggregation_seed": 42, + "forbes_top_tail_enabled": True, + "forbes_aggregate_recid": 999999, + "forbes_synthetic_record_count": 3900, + } + notes = stage.notes + assert "uprates the raw TY2015 rows before replacing" in notes + assert "Forbes backbone for aggregate RECID 999999" in notes + assert "3,900-record open tail" in notes + assert "does not request unsupported Forbes synthesis" in notes + donor_notes = US_DONORS["puf_tax_detail"].notes + assert "uprates its raw TY2015 rows before" in donor_notes + assert "Forbes-backed 3,900-record open tail" in donor_notes + assert "without rerunning Forbes synthesis" in donor_notes def test_aca_stage_declares_marketplace_input_surface(self) -> None: stage = US_SOURCE_MANIFEST.stage_map()["aca_marketplace_inputs"] diff --git a/packages/populace-build/tests/test_us_puf_e19200_decomposition.py b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py new file mode 100644 index 00000000..43407510 --- /dev/null +++ b/packages/populace-build/tests/test_us_puf_e19200_decomposition.py @@ -0,0 +1,501 @@ +from __future__ import annotations + +from hashlib import sha256 +from pathlib import Path + +import numpy as np +import pytest + +from populace.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_AGI_BANDS, + US_PUF_E19200_ALL_RETURNS_COMPONENTS, + split_us_puf_e19200_by_agi_band, +) +from populace.build.us_runtime.puf_source_agi import ( + source_year_puf_adjusted_gross_income, +) +from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays + +_REAL_PUF_STORAGE = ( + Path.home() + / "PolicyEngine" + / ("policyengine" + "-us-data") + / ("policyengine" + "_us_data") + / "storage" +) +_REAL_PROCESSED_PUF = _REAL_PUF_STORAGE / "puf_2024.h5" +_REAL_SOURCE_PUF = _REAL_PUF_STORAGE / "puf_2015.csv" +_REAL_PROCESSED_PUF_SHA256 = ( + "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df" +) +_REAL_SOURCE_PUF_SHA256 = ( + "0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df" +) +_REAL_SOURCE_AGI_VECTOR_SHA256 = ( + "8b2b6c206e8e9f7b80dcfd86554962b0966b25ad8c47eab7d9db4f27f69ebc0d" +) + + +def _representative_agi(lower: float | None, upper: float | None) -> float: + if lower is None: + return -10_000.0 + if upper is None: + return lower + 1_000_000.0 + return (lower + upper) / 2 + + +def _one_record_per_band_arrays() -> tuple[dict[str, list[object]], np.ndarray]: + tax_unit_ids = np.arange(1, len(US_PUF_E19200_AGI_BANDS) + 1, dtype=np.int64) + total_interest = np.full(len(tax_unit_ids), 1_000.0, dtype=np.float64) + adjusted_gross_income = np.asarray( + [ + _representative_agi(band.lower_bound, band.upper_bound) + for band in US_PUF_E19200_AGI_BANDS + ], + dtype=np.float64, + ) + arrays: dict[str, list[object]] = { + "tax_unit_id": tax_unit_ids.tolist(), + "household_weight": np.ones(len(tax_unit_ids)).tolist(), + "filing_status": [b"SINGLE"] * len(tax_unit_ids), + "person_tax_unit_id": tax_unit_ids.tolist(), + "home_mortgage_interest": total_interest.tolist(), + # The processed PUF leaf is all-zero before this decomposition. The + # split must replace it from E19200, not preserve or add this sentinel. + "investment_interest_expense": np.zeros(len(tax_unit_ids)).tolist(), + } + return arrays, adjusted_gross_income + + +def _raw_puf_source_fixture() -> tuple[dict[str, list[float]], np.ndarray, np.ndarray]: + regular_ids = np.arange(1, 31, dtype=np.int64) + regular_agi = np.asarray( + [ + 4_000.0, + *np.linspace(-500_000.0, -10_000.0, 9), + *np.linspace(10_000.0, 9_000_000.0, 10), + *np.linspace(10_000_000.0, 200_000_000.0, 10), + ], + dtype=np.float64, + ) + aggregate_ids = np.asarray([999_996, 999_997, 999_998, 999_999]) + aggregate_agi = np.asarray([-100_000.0, 2_000_000.0, 20_000_000.0, 200_000_000.0]) + recid = np.concatenate((regular_ids, aggregate_ids)) + agi = np.concatenate((regular_agi, aggregate_agi)) + source: dict[str, list[float]] = { + "RECID": recid.tolist(), + "MARS": np.concatenate((np.ones(len(regular_ids)), np.zeros(4))).tolist(), + "S006": np.concatenate( + ( + np.full(len(regular_ids), 100.0), + np.asarray([14_000.0, 23_000.0, 39_000.0, 10_000.0]), + ) + ).tolist(), + "E00100": agi.tolist(), + } + screened_fields = ( + "E00200", + "P23250", + "P22250", + "E00650", + "E00300", + "E26270", + "E00900", + "E02100", + "E00400", + "E00600", + ) + for offset, field in enumerate(screened_fields, start=1): + source[field] = (np.abs(agi) / offset + offset).tolist() + + synthetic_ids = np.arange(1_000_000, 1_000_102, dtype=np.int64) + processed_ids = np.concatenate((regular_ids, synthetic_ids)) + processed_weights = np.concatenate( + ( + np.ones(len(regular_ids)), + np.full(20, 7.0), + np.full(23, 10.0), + np.full(39, 10.0), + np.full(20, 5.0), + ) + ) + return source, processed_ids, processed_weights + + +def _file_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def test_ty2015_e19200_component_rows_are_complete_cited_and_conservative() -> None: + bands = US_PUF_E19200_AGI_BANDS + + assert len(bands) == 22 + assert [band.source_row for band in bands] == list(range(11, 33)) + assert bands[0].label == "Under $5,000" + assert bands[0].lower_bound is None + assert bands[0].upper_bound == 5_000 + assert bands[-1].lower_bound == 10_000_000 + assert bands[-1].upper_bound is None + for previous, following in zip(bands[:-1], bands[1:], strict=True): + assert previous.upper_bound == following.lower_bound + for band in bands: + assert band.source_cells == { + "total_interest_paid_amount": f"CF{band.source_row}", + "home_mortgage_interest_amount": f"CH{band.source_row}", + "deductible_points_amount": f"CN{band.source_row}", + "qualified_mortgage_insurance_premiums_amount": (f"CP{band.source_row}"), + "investment_interest_amount": f"CR{band.source_row}", + } + published_components = ( + band.home_mortgage_interest_amount + + band.deductible_points_amount + + band.qualified_mortgage_insurance_premiums_amount + + band.investment_interest_amount + ) + assert abs(band.total_interest_paid_amount - published_components) <= 1 + + assert sum(band.total_interest_paid_amount for band in bands) == 304_461_163 + assert sum(band.home_mortgage_interest_amount for band in bands) == 283_004_467 + assert sum(band.non_mortgage_interest_amount for band in bands) == 21_456_696 + assert sum(band.investment_interest_amount for band in bands) == 13_895_493 + + all_returns = US_PUF_E19200_ALL_RETURNS_COMPONENTS + assert all_returns.source_row == 10 + assert all_returns.total_interest_paid_amount == 304_461_163 + assert all_returns.home_mortgage_interest_amount == 283_004_465 + assert all_returns.deductible_points_amount == 1_273_716 + assert all_returns.qualified_mortgage_insurance_premiums_amount == 6_287_486 + assert all_returns.investment_interest_amount == 13_895_495 + + +def test_e19200_first_band_includes_negative_zero_and_below_5000_agi() -> None: + values = np.ones(4, dtype=np.float64) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + values, + np.asarray([-100_000.0, 0.0, 4_999.99, 5_000.0]), + ) + first_share = ( + US_PUF_E19200_AGI_BANDS[0].home_mortgage_interest_amount + / US_PUF_E19200_AGI_BANDS[0].total_interest_paid_amount + ) + second_share = ( + US_PUF_E19200_AGI_BANDS[1].home_mortgage_interest_amount + / US_PUF_E19200_AGI_BANDS[1].total_interest_paid_amount + ) + + np.testing.assert_array_equal( + mortgage[:3].view(np.uint64), + np.full(3, first_share, dtype=np.float64).view(np.uint64), + ) + assert mortgage[3] == second_share + np.testing.assert_array_equal(mortgage + non_mortgage, values) + + +def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: + arrays, adjusted_gross_income = _one_record_per_band_arrays() + + donor = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + total = np.asarray( + [band.total_interest_paid_amount for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ) + expected_mortgage = np.asarray( + [1_000.0 * band.home_mortgage_share for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ) + expected_non_mortgage = 1_000.0 - expected_mortgage + np.testing.assert_allclose(donor["home_mortgage_interest"], expected_mortgage) + np.testing.assert_allclose( + donor["investment_interest_expense"], + expected_non_mortgage, + ) + np.testing.assert_array_equal( + donor["home_mortgage_interest"].to_numpy() + + donor["investment_interest_expense"].to_numpy(), + np.full(len(total), 1_000.0), + ) + assert (donor["investment_interest_expense"] > 0).all() + + # The same proportional rule applied to the literal source rows recovers + # the published mortgage amounts and the full conserving residual mass. + source_mortgage, source_non_mortgage = split_us_puf_e19200_by_agi_band( + total, + adjusted_gross_income, + ) + np.testing.assert_allclose( + source_mortgage, + np.asarray( + [band.home_mortgage_interest_amount for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, + ), + ) + assert source_non_mortgage.sum() == 21_456_696 + + +def test_e19200_donor_split_requires_explicit_adjusted_gross_income() -> None: + arrays = { + "tax_unit_id": [1], + "household_weight": [1.0], + "filing_status": [b"SINGLE"], + "person_tax_unit_id": [1], + "home_mortgage_interest": [100.0], + "investment_interest_expense": [0.0], + # This predictor is deliberately tempting but is not AGI and cannot + # silently select a source-table band. + "employment_income": [250_000.0], + } + + with pytest.raises(ValueError, match="adjusted_gross_income"): + puf_tax_unit_donor_from_arrays( + arrays, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + +def test_e19200_donor_split_is_bit_deterministic_and_order_invariant() -> None: + arrays, adjusted_gross_income = _one_record_per_band_arrays() + first = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + second = puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ) + + permutation = np.arange(len(adjusted_gross_income) - 1, -1, -1) + shuffled_arrays = { + name: np.asarray(values)[permutation].tolist() + for name, values in arrays.items() + } + shuffled = puf_tax_unit_donor_from_arrays( + shuffled_arrays, + adjusted_gross_income=adjusted_gross_income[permutation], + person_outputs=( + "home_mortgage_interest", + "investment_interest_expense", + ), + tax_unit_outputs=(), + ).sort_values("tax_unit_id") + + columns = ("home_mortgage_interest", "investment_interest_expense") + for column in columns: + np.testing.assert_array_equal( + first[column].to_numpy().view(np.uint64), + second[column].to_numpy().view(np.uint64), + ) + np.testing.assert_array_equal( + first[column].to_numpy().view(np.uint64), + shuffled[column].to_numpy().view(np.uint64), + ) + + +def test_source_year_puf_agi_aligns_regular_and_synthetic_records() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + + adjusted_gross_income = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + + # The source record remains in its literal TY2015 band. A target-year + # formula or nominal uprating could move this $4,000 return into a later + # band, which is precisely what this source-aligned seam prevents. + assert adjusted_gross_income[0] == 4_000.0 + np.testing.assert_array_equal( + adjusted_gross_income[:30], + np.asarray(source["E00100"][:30]), + ) + assert (adjusted_gross_income[30:50] < 5_000.0).all() + assert ( + (adjusted_gross_income[50:73] >= 0.0) + & (adjusted_gross_income[50:73] < 10_000_000.0) + ).all() + assert (adjusted_gross_income[73:] >= 10_000_000.0).all() + + +def test_source_year_puf_agi_is_bit_deterministic() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + + first = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + second = source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + + np.testing.assert_array_equal(first.view(np.uint64), second.view(np.uint64)) + + +def test_source_year_puf_agi_fails_closed_on_processed_id_drift() -> None: + source, processed_ids, processed_weights = _raw_puf_source_fixture() + processed_ids = processed_ids.copy() + processed_ids[3] = 123_456 + + with pytest.raises(ValueError, match="regular RECID order"): + source_year_puf_adjusted_gross_income( + source, + processed_tax_unit_ids=processed_ids, + processed_tax_unit_weights=processed_weights, + ) + + +def test_source_year_puf_agi_matches_pinned_flat_array_artifact() -> None: + if not _REAL_PROCESSED_PUF.exists() or not _REAL_SOURCE_PUF.exists(): + pytest.skip("Pinned restricted PUF artifacts are not available.") + assert _file_sha256(_REAL_PROCESSED_PUF) == _REAL_PROCESSED_PUF_SHA256 + assert _file_sha256(_REAL_SOURCE_PUF) == _REAL_SOURCE_PUF_SHA256 + + import h5py + + with h5py.File(_REAL_PROCESSED_PUF, "r") as h5: + # This is deliberately the production artifact's flat root-array + # layout, not a USSingleYearDataset entity-table fixture. + assert "person" not in h5 + adjusted_gross_income = source_year_puf_adjusted_gross_income( + _REAL_SOURCE_PUF, + processed_tax_unit_ids=np.asarray(h5["tax_unit_id"]), + processed_tax_unit_weights=np.asarray(h5["household_weight"]), + ) + + assert len(adjusted_gross_income) == 211_677 + assert ( + sha256(adjusted_gross_income.astype(" None: + """PR #561 review finding 1: the naive (total*share, total-mortgage) + pair fails bit-exact conservation (E19200=1.53 in the top band summed to + 1.5300000000000002). The reconciled split must reproduce the total + bit-for-bit on the reviewer's reproducer and on an adversarial sweep.""" + import numpy as np + + from populace.build.us_runtime.puf_interest_components import ( + split_us_puf_e19200_by_agi_band, + ) + + total = np.asarray([1.53], dtype=np.float64) + agi = np.asarray([10_000_000.0], dtype=np.float64) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band(total, agi) + assert float(mortgage[0] + non_mortgage[0]) == 1.53 + + rng = np.random.default_rng(561) + totals = np.round(rng.uniform(0.01, 5_000_000.0, size=20_000), 2) + agis = rng.uniform(-100_000.0, 20_000_000.0, size=20_000) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band(totals, agis) + recon = mortgage + non_mortgage + exact = recon == totals + assert exact.all(), ( + f"{(~exact).sum()} of 20,000 adversarial records failed bit-exact " + "conservation after reconciliation" + ) + assert (mortgage >= 0.0).all() and (non_mortgage >= 0.0).all() + + +def test_donor_api_rejects_non_real_agi_through_a_valid_fixture() -> None: + """PR #561 review findings 2 (rounds 1-2): the donor API must fail + closed on AGI that is not a finite real number. Round 1's + ``_numeric_array`` coerced nonnumeric/NaN to 0.0; round 2's + ``errors="raise"`` was parse-strict but not real-number-strict — + datetime/timedelta arrays (including NaT) convert to finite epoch + sentinels, complex drops its imaginary part, and booleans pass, all + silently routing records to the wrong AGI band. The fixture below is + proven valid by a positive control, so each rejection can only come + from the AGI validation itself.""" + import numpy as np + import pandas as pd + import pytest as _pytest + + from populace.build.us_runtime.puf_support import puf_tax_unit_donor_from_arrays + + arrays = { + "tax_unit_id": np.asarray([1], dtype="int64"), + "person_tax_unit_id": np.asarray([1], dtype="int64"), + "household_weight": np.asarray([1.0]), + "filing_status": np.asarray(["SINGLE"]), + } + + def call(agi): + return puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=agi, + person_outputs=(), + tax_unit_outputs=(), + ) + + # Positive control: the fixture succeeds with valid AGI (the band + # column is reserved for internal processing and is consumed before + # return), so the rejections below cannot be masked by an unrelated + # fixture defect. + donor = call(np.asarray([50_000.0])) + assert len(donor) == 1 + assert donor["tax_unit_id"].tolist() == [1] + # Faithful float conversions are accepted (review round 3): Decimal + # and numeric strings parse strictly to the same real value. + from decimal import Decimal + + assert len(call([Decimal("50000")])) == 1 + assert len(call(["50000"])) == 1 + + with _pytest.raises(ValueError, match="finite"): + call(np.asarray([np.nan])) + with _pytest.raises(ValueError, match="finite"): + call(np.asarray([np.inf])) + with _pytest.raises(ValueError, match="Unable to parse"): + call(["not-a-number"]) + # Typed non-real arrays parse to finite numbers under pd.to_numeric and + # must be rejected by dtype, not by parsing. + with _pytest.raises(TypeError, match="real-valued"): + call(pd.Series([pd.NaT])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray(["NaT"], dtype="datetime64[ns]")) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([np.timedelta64(1, "D")])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([1.0 + 2.0j])) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([True])) + # Object-wrapped and categorical payloads dodge the dtype-kind gate + # (review round 3): True parses to 1.0 and complex keeps its real + # part, so the element screen must reject them too. + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([True], dtype=object)) + with _pytest.raises(TypeError, match="real-valued"): + call(np.asarray([1.0 + 2.0j], dtype=object)) + with _pytest.raises(TypeError, match="real-valued"): + call(pd.Categorical([True])) diff --git a/packages/populace-build/tests/test_us_puf_support.py b/packages/populace-build/tests/test_us_puf_support.py index 1a145592..fa3b07b0 100644 --- a/packages/populace-build/tests/test_us_puf_support.py +++ b/packages/populace-build/tests/test_us_puf_support.py @@ -13,11 +13,11 @@ CPS_CARRIED_FORMULA_OWNED_COLUMNS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_E19200_HOME_MORTGAGE_SHARE, clone_us_frame_for_puf_support, derive_us_cps_carried_inputs, impute_us_puf_tax_detail_support, puf_tax_unit_donor_from_arrays, + split_us_puf_e19200_by_agi_band, support_channel_column, support_clone_index_column, support_source_id_column, @@ -307,6 +307,7 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: "taxable_unemployment_compensation": [13.0, 17.0, 19.0], "state_and_local_sales_or_income_tax": [40.0, 50.0], }, + adjusted_gross_income=[0.0, 0.0], person_outputs=( "employment_income_before_lsr", "qualified_dividend_income", @@ -335,7 +336,10 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: assert donor["unemployment_compensation"].tolist() == [30.0, 19.0] np.testing.assert_allclose( donor["home_mortgage_interest"].to_numpy(), - np.asarray([30.0, 30.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + split_us_puf_e19200_by_agi_band( + np.asarray([30.0, 30.0]), + np.asarray([0.0, 0.0]), + )[0], ) assert donor["educator_expense"].tolist() == [300.0, 300.0] assert "interest_deduction" not in donor @@ -346,7 +350,10 @@ def test_puf_tax_unit_donor_from_arrays_aggregates_person_values() -> None: def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: assert US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING == 10_000_000.0 - carved_below_ceiling = 10_500_000.0 * US_PUF_E19200_HOME_MORTGAGE_SHARE + carved_below_ceiling = split_us_puf_e19200_by_agi_band( + np.asarray([10_500_000.0]), + np.asarray([10_000_000.0]), + )[0][0] assert carved_below_ceiling < US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING donor = puf_tax_unit_donor_from_arrays( @@ -374,6 +381,7 @@ def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: ], "domestic_production_ald": [900_000_000.0, 700.0, 800_000_000.0], }, + adjusted_gross_income=[0.0, 0.0, 10_000_000.0], person_outputs=( "home_mortgage_interest", "employment_income_before_lsr", @@ -389,7 +397,10 @@ def test_puf_tax_unit_donor_drops_grouped_raw_mortgage_outlier_rows() -> None: assert donor["domestic_production_ald"].tolist() == [700.0] np.testing.assert_allclose( donor["home_mortgage_interest"].to_numpy(), - np.asarray([5_000_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + split_us_puf_e19200_by_agi_band( + np.asarray([5_000_000.0]), + np.asarray([0.0]), + )[0], ) @@ -397,7 +408,7 @@ def test_puf_tax_unit_donor_rejects_reserved_screen_column_output() -> None: # populace#516: the raw-mortgage helper name is reserved -- requesting it # as an output would let the screen threshold and then delete a caller's # column, silently violating the requested-output contract. - with pytest.raises(ValueError, match="reserved for the donor"): + with pytest.raises(ValueError, match="reserved for donor"): puf_tax_unit_donor_from_arrays( { "tax_unit_id": [10], @@ -411,8 +422,7 @@ def test_puf_tax_unit_donor_rejects_reserved_screen_column_output() -> None: ) -def test_puf_e19200_home_mortgage_carve_scales_only_lineage_columns() -> None: - assert US_PUF_E19200_HOME_MORTGAGE_SHARE == 283_004_465 / 304_461_163 +def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: # Pin the lineage tuple by exact membership: an accidental addition (the # sol round-1 failure mode was appending investment_interest_expense, # invisible behind a zero sentinel) must fail here, not silently carve a @@ -451,31 +461,36 @@ def test_puf_e19200_home_mortgage_carve_scales_only_lineage_columns() -> None: "second_home_mortgage_balance": [0.0, 125_000.0], "first_home_mortgage_origination_year": [2018.0, 2016.0], "second_home_mortgage_origination_year": [0.0, 2020.0], - # Nonzero sentinel: the artifact carries this column all-zero, - # but a zero fixture cannot distinguish "not scaled" from - # "scaled" (0 x share == 0). The root #515 ETL carve will make - # it nonzero, and it must stay uncarved then. - "investment_interest_expense": [12.0, 34.0], + "investment_interest_expense": [0.0, 0.0], + puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN: [100.0, 200.0], + puf_support_module._E19200_AGI_BAND_COLUMN: [0.0, 10_000_000.0], } ) original = donor.copy(deep=True) - puf_support_module._carve_us_puf_e19200_home_mortgage_share(donor) + puf_support_module._split_us_puf_e19200_components(donor) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( + np.asarray([100.0, 200.0]), + np.asarray([0.0, 10_000_000.0]), + ) + shares = mortgage / np.asarray([100.0, 200.0]) for column in puf_support_module._US_PUF_E19200_LINEAGE_DONOR_COLUMNS: np.testing.assert_allclose( donor[column].to_numpy(), - original[column].to_numpy() * US_PUF_E19200_HOME_MORTGAGE_SHARE, + original[column].to_numpy() * shares, ) + np.testing.assert_allclose(donor["investment_interest_expense"], non_mortgage) for column in ( "real_estate_taxes", "first_home_mortgage_balance", "second_home_mortgage_balance", "first_home_mortgage_origination_year", "second_home_mortgage_origination_year", - "investment_interest_expense", ): np.testing.assert_array_equal(donor[column], original[column]) + assert puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN not in donor + assert puf_support_module._E19200_AGI_BAND_COLUMN not in donor def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: @@ -493,6 +508,13 @@ def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: assert "self_employment_income_before_lsr" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "self_employment_income" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "tax_exempt_interest_income" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS + assert "investment_interest_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS + assert ( + "investment_interest_expense" + in puf_support_module._PUF_TAX_DETAIL_NONNEGATIVE_OUTPUTS + ) + assert "investment_interest_expense" not in PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS + assert "investment_interest_expense" not in PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS assert "unemployment_compensation" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert ( "long_term_capital_gains_before_response" @@ -1256,6 +1278,7 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: "household_weight": [100.0, 200.0], "filing_status": [b"SINGLE", b"JOINT"], "person_tax_unit_id": [10, 20], + "home_mortgage_interest": [10_000.0, 25_000.0], "first_home_mortgage_balance": [250_000.0, 500_000.0], "second_home_mortgage_balance": [0.0, 125_000.0], "first_home_mortgage_interest": [10_000.0, 20_000.0], @@ -1266,6 +1289,7 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: "domestic_production_ald": [7_500.0, 0.0], "unrecaptured_section_1250_gain": [500.0, 0.0], }, + adjusted_gross_income=[0.0, 10_000_000.0], person_outputs=(), tax_unit_outputs=PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) @@ -1277,11 +1301,25 @@ def test_puf_tax_unit_donor_carries_structural_mortgage_leaves() -> None: assert donor["second_home_mortgage_balance"].tolist() == [0.0, 125_000.0] np.testing.assert_allclose( donor["first_home_mortgage_interest"].to_numpy(), - np.asarray([10_000.0, 20_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + np.asarray([10_000.0, 20_000.0]) + * ( + split_us_puf_e19200_by_agi_band( + np.asarray([10_000.0, 25_000.0]), + np.asarray([0.0, 10_000_000.0]), + )[0] + / np.asarray([10_000.0, 25_000.0]) + ), ) np.testing.assert_allclose( donor["second_home_mortgage_interest"].to_numpy(), - np.asarray([0.0, 5_000.0]) * US_PUF_E19200_HOME_MORTGAGE_SHARE, + np.asarray([0.0, 5_000.0]) + * ( + split_us_puf_e19200_by_agi_band( + np.asarray([10_000.0, 25_000.0]), + np.asarray([0.0, 10_000_000.0]), + )[0] + / np.asarray([10_000.0, 25_000.0]) + ), ) assert donor["first_home_mortgage_origination_year"].tolist() == [ 2018.0, diff --git a/packages/populace-build/tests/test_us_puf_support_base_builder.py b/packages/populace-build/tests/test_us_puf_support_base_builder.py index 27cd6c59..8dc0f06b 100644 --- a/packages/populace-build/tests/test_us_puf_support_base_builder.py +++ b/packages/populace-build/tests/test_us_puf_support_base_builder.py @@ -485,6 +485,88 @@ def test_outer_stage_resume_rejects_changed_builder_code( ) +def test_puf_donor_builder_threads_explicit_source_year_agi( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + builder = _load_support_builder_module() + path = tmp_path / "puf.h5" + source_path = tmp_path / "puf_2015.csv" + arrays = { + "tax_unit_id": np.asarray([10.0, 20.0]), + "household_weight": np.asarray([1.0, 2.0]), + } + adjusted_gross_income = np.asarray([-5_000.0, 250_000.0]) + captured: dict[str, object] = {} + + monkeypatch.setattr(builder, "_read_h5_arrays", lambda actual: arrays) + + def fake_agi( + actual_source_path: Path, + *, + processed_tax_unit_ids, + processed_tax_unit_weights, + ) -> np.ndarray: + captured["source_path"] = actual_source_path + captured["processed_tax_unit_ids"] = processed_tax_unit_ids + captured["processed_tax_unit_weights"] = processed_tax_unit_weights + return adjusted_gross_income + + def fake_donor(actual_arrays, *, adjusted_gross_income): + captured["arrays"] = actual_arrays + captured["adjusted_gross_income"] = adjusted_gross_income + return "donor" + + monkeypatch.setattr(builder, "_source_year_puf_adjusted_gross_income", fake_agi) + monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", fake_donor) + + assert ( + builder._puf_tax_unit_donor_from_h5( + path, + source_puf_csv=source_path, + ) + == "donor" + ) + assert captured["source_path"] == source_path + assert captured["processed_tax_unit_ids"] is arrays["tax_unit_id"] + assert captured["processed_tax_unit_weights"] is arrays["household_weight"] + assert captured["arrays"] is arrays + assert captured["adjusted_gross_income"] is adjusted_gross_income + + +def test_source_year_puf_input_content_is_checkpoint_identity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + builder = _load_support_builder_module() + source = tmp_path / "puf_2015.csv" + source.write_bytes(b"first") + args = builder._parse_args( + [ + "--base-h5", + str(tmp_path / "base.h5"), + "--puf-h5", + str(tmp_path / "puf.h5"), + "--puf-source-year-csv", + str(source), + "--out", + str(tmp_path / "out"), + "--without-block-ladder", + ] + ) + monkeypatch.setattr( + builder, + "_builder_code_identity", + lambda: {"source_sha256": "builder"}, + ) + first = builder._stage_run_config(args) + source.write_bytes(b"second") + second = builder._stage_run_config(args) + + assert first["puf_source_year_csv"] == str(source.resolve()) + assert first["puf_source_year_csv_sha256"] != second["puf_source_year_csv_sha256"] + + def test_monolith_equivalence_observer_writes_all_boundaries_and_raw_bits( tmp_path: Path, ) -> None: @@ -1284,8 +1366,11 @@ def fake_retirement_distributions( "clone_us_frame_for_puf_support", lambda frame: "expanded", ) - monkeypatch.setattr(builder, "_read_h5_arrays", lambda path: {}) - monkeypatch.setattr(builder, "puf_tax_unit_donor_from_arrays", lambda arrays: None) + monkeypatch.setattr( + builder, + "_puf_tax_unit_donor_from_h5", + lambda path, *, source_puf_csv: None, + ) monkeypatch.setattr( builder, "impute_and_audit_us_puf_support", @@ -1759,6 +1844,8 @@ def test_stage_cli_round_trips_the_locked_run_config(tmp_path: Path) -> None: builder = _load_support_builder_module() puf = tmp_path / "puf.h5" puf.write_bytes(b"puf") + source_puf = tmp_path / "puf_2015.csv" + source_puf.write_bytes(b"source puf") sidecar = tmp_path / "asecpub24csv.zip" sidecar.write_bytes(b"zip") argv = [ @@ -1768,6 +1855,8 @@ def test_stage_cli_round_trips_the_locked_run_config(tmp_path: Path) -> None: f"2024={tmp_path / 'census_cps_2024.h5'}", "--puf-h5", str(puf), + "--puf-source-year-csv", + str(source_puf), "--asec-education-source", f"2023={sidecar}", "--target-year", diff --git a/packages/populace-build/tests/test_us_puf_tail_bound.py b/packages/populace-build/tests/test_us_puf_tail_bound.py index 687fd2b4..d4a38dc5 100644 --- a/packages/populace-build/tests/test_us_puf_tail_bound.py +++ b/packages/populace-build/tests/test_us_puf_tail_bound.py @@ -323,6 +323,7 @@ def test_real_puf_weighted_p999_is_below_build_m_ceiling_draw() -> None: arrays = {column: h5[column][...] for column in required} donor = puf_tax_unit_donor_from_arrays( arrays, + adjusted_gross_income=np.zeros(len(arrays["tax_unit_id"])), person_outputs=(_TARGET,), tax_unit_outputs=(), ) diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index 1ac65361..d7992939 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -65,6 +65,7 @@ load_congressional_district_vintage_crosswalk, load_us_block_ladder, puf_tax_unit_donor_from_arrays, + source_year_puf_adjusted_gross_income, support_channel_column, translate_congressional_district_facts_to_current_vintage, us_adult_care_signal_gate, @@ -266,6 +267,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ), ) parser.add_argument("--puf-h5", required=True, type=Path) + parser.add_argument( + "--puf-source-year-csv", + type=Path, + help=( + "Restricted raw TY2015 IRS PUF CSV carrying RECID, E00100, S006, " + "and the archived aggregate-record donor fields. Required when " + "building the E19200 decomposition from a nonzero processed PUF." + ), + ) parser.add_argument( "--asec-2023-weeks-unemployed-source", type=Path, @@ -518,6 +528,11 @@ def _stage_cli_args(args: argparse.Namespace, stage: str) -> list[str]: command.extend(("--asec-max-households", str(args.asec_max_households))) _append_path_argument(command, "--support-spine-spec", args.support_spine_spec) command.extend(("--puf-h5", str(args.puf_h5))) + _append_path_argument( + command, + "--puf-source-year-csv", + args.puf_source_year_csv, + ) _append_path_argument( command, "--asec-2023-weeks-unemployed-source", @@ -664,6 +679,12 @@ def path(value: Path | None) -> str | None: "n_estimators": args.n_estimators, "out": path(args.out), "puf_h5": path(args.puf_h5), + "puf_source_year_csv": path(args.puf_source_year_csv), + "puf_source_year_csv_sha256": ( + _sha256(args.puf_source_year_csv) + if args.puf_source_year_csv is not None + else None + ), "seed": args.seed, "support_spine_spec": path(args.support_spine_spec), "target_year": args.target_year, @@ -992,8 +1013,10 @@ def _run_all( ) _observe_frame_boundary(boundary_observer, "pre_clone_enrichment", base) expanded = clone_us_frame_for_puf_support(base) - arrays = _read_h5_arrays(args.puf_h5) - donor = puf_tax_unit_donor_from_arrays(arrays) + donor = _puf_tax_unit_donor_from_h5( + args.puf_h5, + source_puf_csv=getattr(args, "puf_source_year_csv", None), + ) _observe_frame_boundary(boundary_observer, "clone_feature_extraction", expanded) tail_bound_diagnostics: list[dict[str, object]] = [] if boundary_observer is None: @@ -1376,6 +1399,16 @@ def _run_all( "base_sha256": _sha256(args.base_h5) if args.base_h5 is not None else None, "puf_h5": str(args.puf_h5.resolve()), "puf_sha256": _sha256(args.puf_h5), + "puf_source_year_csv": ( + str(args.puf_source_year_csv.resolve()) + if args.puf_source_year_csv is not None + else None + ), + "puf_source_year_csv_sha256": ( + _sha256(args.puf_source_year_csv) + if args.puf_source_year_csv is not None + else None + ), "acs_h5": str(args.acs_h5.resolve()) if args.acs_h5 is not None else None, "acs_sha256": _sha256(args.acs_h5) if args.acs_h5 is not None else None, "acs_rent_donor_rows": ( @@ -1860,7 +1893,10 @@ def _clone_feature_extraction_stage( base: Frame, ) -> tuple[Frame, dict[str, object]]: expanded = clone_us_frame_for_puf_support(base) - donor = puf_tax_unit_donor_from_arrays(_read_h5_arrays(args.puf_h5)) + donor = _puf_tax_unit_donor_from_h5( + args.puf_h5, + source_puf_csv=args.puf_source_year_csv, + ) qrf_dir = args.checkpoint_dir / "primary_qrf" if qrf_dir.exists(): # The outer context marks clone_feature_extraction only after both its @@ -1878,8 +1914,12 @@ def _clone_feature_extraction_stage( return expanded, { "puf_h5": str(args.puf_h5.resolve()), "puf_sha256": _sha256(args.puf_h5), + "puf_source_year_csv": str(args.puf_source_year_csv.resolve()), + "puf_source_year_csv_sha256": _sha256(args.puf_source_year_csv), "puf_donor_rows": int(len(donor)), "puf_donor_columns": sorted(donor.columns.tolist()), + "puf_e19200_agi_variable": "E00100", + "puf_e19200_agi_period": 2015, "primary_qrf_checkpoint_dir": str(qrf_dir.resolve()), } @@ -2398,6 +2438,8 @@ def _export_staged_result( ), "puf_h5": clone["puf_h5"], "puf_sha256": clone["puf_sha256"], + "puf_source_year_csv": clone["puf_source_year_csv"], + "puf_source_year_csv_sha256": clone["puf_source_year_csv_sha256"], "acs_h5": pre_clone["acs_h5"], "acs_sha256": pre_clone["acs_sha256"], "acs_rent_donor_rows": pre_clone["acs_rent_donor_rows"], @@ -2708,6 +2750,45 @@ def _read_h5_arrays(path: Path) -> dict[str, np.ndarray]: return {name: np.asarray(dataset) for name, dataset in h5.items()} +def _puf_tax_unit_donor_from_h5( + path: Path, + *, + source_puf_csv: Path | None, +) -> pd.DataFrame: + """Build the PUF donor with source-year E00100-aligned banding values.""" + + arrays = _read_h5_arrays(path) + if source_puf_csv is None: + raise ValueError( + "--puf-source-year-csv is required to align nonzero E19200 records " + "to the published TY2015 SOI AGI bands." + ) + adjusted_gross_income = _source_year_puf_adjusted_gross_income( + source_puf_csv, + processed_tax_unit_ids=arrays["tax_unit_id"], + processed_tax_unit_weights=arrays["household_weight"], + ) + return puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + ) + + +def _source_year_puf_adjusted_gross_income( + source_puf_csv: Path, + *, + processed_tax_unit_ids: Sequence[object], + processed_tax_unit_weights: Sequence[object], +) -> np.ndarray: + """Load the restricted source only through its narrow alignment seam.""" + + return source_year_puf_adjusted_gross_income( + source_puf_csv, + processed_tax_unit_ids=processed_tax_unit_ids, + processed_tax_unit_weights=processed_tax_unit_weights, + ) + + def _row_counts(frame: Frame) -> dict[str, int]: return {entity: frame.n(entity) for entity in frame.entities} diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 6e76efcf..d7499fc7 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -256,6 +256,7 @@ *RETIREMENT_DISTRIBUTION_INPUTS, "domestic_production_ald", "household_weight", + "investment_interest_expense", "spm_unit_pre_subsidy_childcare_expenses", "unreimbursed_business_employee_expenses", *QBI_INPUTS,