Finish #515: per-record E19200 decomposition by published SOI 2.1 AGI-band components

changelog.d/e19200-decomposition-515.fixed.md

@@ -0,0 +1 @@
+Decompose each PUF E19200 record into mortgage and non-mortgage interest using the published 2015 SOI Table 2.1 component shares for its AGI band, restore nonzero investment-interest support, and bind the JCT mortgage reform to the mortgage-specific variable.

experiments/build_j_recert/buildj_base.sh

@@ -35,6 +35,7 @@ required = {
         "pre_subsidy_rent",
         "self_employment_income_last_year",
         "previous_year_income_available",
+        "investment_interest_expense",
         "salt_refund_income",
         "takes_up_medicare_if_eligible",
         "workers_compensation",
@@ -71,18 +72,19 @@ source .venv/bin/activate 2>/dev/null
 
 # ---- integrity preflight on inputs (reproduce Build F's exact base) ----
 chk() { local got exp; got=$(shasum -a 256 "$1" | cut -c1-16); [ "$got" = "$2" ] || { say "FATAL sha mismatch $1: $got != $2"; echo 2 > "$LOGDIR/base.rc"; exit 2; }; }
-for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do
+for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/puf_2015.csv" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do
   [ -f "$f" ] || { say "FATAL missing input $f"; echo 2 > "$LOGDIR/base.rc"; exit 2; }
 done
 chk "$AGING_FACTS" a5d34d4aad325d8c
 chk "$CDX" 383a666631aafd4f
 chk "$BLADDER" 7ba39b959068181b
 chk "$USD/acs_2022.h5" 0b319b496f19a691
+chk "$USD/puf_2015.csv" 0a7fd643edb1acc5
 
 SHORT=$(git rev-parse --short HEAD)
 PEUS=$(.venv/bin/python -c "from importlib.metadata import version; print(version('policyengine-us'))" 2>/dev/null)
 say "BUILD J BASE START commit=$SHORT pe-us=$PEUS pid=$$ pressure_log=$PLOG"
-say "  inputs: asec 2024/2023/2022 + puf_2024 + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32"
+say "  inputs: asec 2024/2023/2022 + puf_2024 + TY2015 E00100 source + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32"
 
 if [ -f "$BASE" ] && base_has_restored_signal "$BASE"; then
   say "BASE: already present with nondefault restored input surface at $BASE — reusing"
@@ -109,6 +111,7 @@ else
     --asec-h5 2023="$USD/census_cps_2023.h5" \
     --asec-h5 2022="$USD/census_cps_2022.h5" \
     --puf-h5 "$USD/puf_2024.h5" \
+    --puf-source-year-csv "$USD/puf_2015.csv" \
     --acs-h5 "$USD/acs_2022.h5" \
     --target-year 2024 \
     --seed 0 --n-estimators 32 \

packages/populace-build/src/populace/build/us/country_package.json

@@ -12,6 +12,7 @@
   "release_input_coverage_manifest.json",
   "soca_capital_gain_distribution_shares.json",
   "soi_baseline_levels.json",
+  "soi_table_2_1_interest_components_ty2015.json",
   "source_stages.json",
   "state_program_levels.json",
   "state_program_reforms.json",

packages/populace-build/src/populace/build/us/ecps_parity_known_gaps.json

@@ -121,70 +121,6 @@
         "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88 and 659-699 consumes only those mappings; packages/populace-build/src/populace/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person/household tables without refreshing omitted Census columns."
       }
     },
-    "investment_interest_expense": {
-      "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.",
-      "issue": "PolicyEngine/populace#274",
-      "evidence": {
-        "classification": "source_unavailability",
-        "retired_derivation": {
-          "repository_owner": "PolicyEngine",
-          "repository_name_parts": ["policyengine-", "us-data"],
-          "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe",
-          "path_parts": ["policyengine_", "us_data", "utils", "mortgage_interest.py"],
-          "lines": "141-155,268-320,327-349"
-        },
-        "retired_puf_source_identity": {
-          "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe",
-          "path_parts": ["policyengine_", "us_data", "datasets", "puf", "puf.py"],
-          "interest_mapping_lines": "636-656",
-          "identical_mortgage_assignment_lines": "1548-1550",
-          "raw_source": "E19200"
-        },
-        "retired_qrf_and_conversion_order": {
-          "qrf_path_parts": ["policyengine_", "us_data", "calibration", "puf_impute.py"],
-          "qrf_lines": "90-198,940-1075",
-          "conversion_path_parts": ["policyengine_", "us_data", "datasets", "cps", "extended_cps.py"],
-          "conversion_lines": "1178-1190",
-          "finding": "The QRF predicts interest_deduction and deductible_mortgage_interest separately even though the PUF construction makes their tax-unit source amounts identical; a positive post-QRF residual is not observed microdata detail."
-        },
-        "required_intermediates": [
-          "interest_deduction",
-          "deductible_mortgage_interest"
-        ],
-        "processed_puf": {
-          "filename": "puf_2024.h5",
-          "release": "policyengine/irs-soi-puf/1.8.0",
-          "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df",
-          "rows": 499045,
-          "positive_values": 0,
-          "weighted_total": 0.0,
-          "missing_columns": [
-            "interest_deduction",
-            "deductible_mortgage_interest",
-            "E19200"
-          ]
-        },
-        "hermetic_asec_inputs": [
-          {
-            "filename": "census_cps_2022.h5",
-            "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e",
-            "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"]
-          },
-          {
-            "filename": "census_cps_2023.h5",
-            "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88",
-            "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"]
-          },
-          {
-            "filename": "census_cps_2024.h5",
-            "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d",
-            "missing_columns": ["interest_deduction", "deductible_mortgage_interest", "E19200"]
-          }
-        ],
-        "semantic_rejection": "SCF mortgage hints and aggregate SOI totals do not identify a return-level mortgage-versus-investment interest split. Re-running two stochastic predictions solely to manufacture a residual would synthesize values rather than restore a source observation.",
-        "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-72 passes only the three locked ASEC H5 files and processed PUF H5; tools/build_us_puf_support_base.py lines 240-247 reads those arrays and cannot refresh omitted restricted raw-PUF columns."
-      }
-    },
     "is_unmarried_partner_of_household_head": {
       "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1214-1221 derives is_unmarried_partner_of_household_head exclusively from person PERRP; datasets/cps/census_cps.py lines 306-318 declares that person source. All three SHA-recorded 2022-2024 ASEC HDF inputs consumed by the hermetic build omit PERRP. The 2022 and 2023 inputs also omit PECOHAB, A_EXPRRP, and A_FAMREL; only 2024 carries those alternatives, and its A_EXPRRP code 13 conflates partner with roommate. Populace's older-year relationship fallback uses only line, spouse, sex, and parent pointers and never identifies a partner. Using the 2024 pointer for one-third of the pool while treating the other two-thirds as false, or assigning partner status from the combined partner/roommate code, would synthesize statuses the locked sources do not report.",
       "issue": "PolicyEngine/populace#38",

packages/populace-build/src/populace/build/us/fiscal_target_references.json

@@ -167,7 +167,7 @@
       "metadata": {
         "kind": "neutralize_variable",
         "matrix_row": "reform_minus_baseline_income_tax",
-        "neutralized_variable": "interest_deduction",
+        "neutralized_variable": "deductible_mortgage_interest",
         "output_variable": "income_tax",
         "target_role": "jct_tax_expenditure"
       },

packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json

@@ -191,9 +191,7 @@
    "status": "required"
   },
   "investment_interest_expense": {
-   "issue": "PolicyEngine/populace#274",
-   "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe maps the sole interest field used by the retired derivation, E19200, to interest_deduction at datasets/puf/puf.py line 654 and explicitly assigns that identical amount as deductible mortgage interest at lines 1548-1550. utils/mortgage_interest.py lines 141-155 and 268-320 derives investment_interest_expense only as max(total_interest_deduction - tax_unit_deductible, 0); the retired QRF separately predicted the two source-identical aliases before conversion, so any positive residual is stochastic prediction divergence rather than observed source detail. The SHA-pinned PUF 1.8.0 artifact consumed by the hermetic build has 499,045 all-zero investment_interest_expense values and omits interest_deduction, deductible_mortgage_interest, and raw E19200; all three locked ASEC inputs omit them too. A nonzero restoration would invent an unobserved mortgage-versus-investment split.",
-   "status": "reviewed_exclusion"
+   "status": "required"
   },
   "is_blind": {
    "status": "required"
@@ -539,8 +537,8 @@
   }
  },
  "counts": {
-  "required": 163,
-  "reviewed_exclusion": 8,
+  "required": 164,
+  "reviewed_exclusion": 7,
   "total": 171
  },
  "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, and the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/populace#462). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/populace#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.",