diff --git a/changelog.d/stacked-spine-pilot-578.added.md b/changelog.d/stacked-spine-pilot-578.added.md new file mode 100644 index 00000000..b42b8078 --- /dev/null +++ b/changelog.d/stacked-spine-pilot-578.added.md @@ -0,0 +1 @@ +Stacked-spine pilot for the #578 increment-2 revision: assemble ASEC plus a seeded ACS household sample into one origin-labeled spine, gap-fill survey-specific fields cross-origin with native predictors under the null-vs-zero doctrine, run a single PUF pass with a seeded clone attachment, and gate the result with a pre-simulation completeness proof and a by-origin battery bound to one immutable, live-digested 90-target authority bundle. diff --git a/packages/populace-build/src/populace/build/gates.py b/packages/populace-build/src/populace/build/gates.py index 0f8bc9fb..70f9474f 100644 --- a/packages/populace-build/src/populace/build/gates.py +++ b/packages/populace-build/src/populace/build/gates.py @@ -39,7 +39,9 @@ from __future__ import annotations import math +import pickle from collections.abc import Iterable, Mapping +from copy import deepcopy from dataclasses import dataclass, field import numpy as np @@ -100,6 +102,10 @@ {kind.value for kind in WeightKind} | {UNWEIGHTED_KIND, EXPLICIT_KIND} ) +_STACKED_AUTHORITY_GATE_NAMES = frozenset( + {"us_stacked_completeness", "us_by_origin_battery"} +) + @dataclass(frozen=True) class GateResult: @@ -120,6 +126,12 @@ class GateResult: passed: bool failures: tuple[str, ...] = () details: Mapping[str, object] = field(default_factory=dict) + _details_protocol_5_snapshot: bytes | None = field( + init=False, + repr=False, + compare=False, + default=None, + ) def __post_init__(self) -> None: if self.passed and self.failures: @@ -128,6 +140,12 @@ def __post_init__(self) -> None: raise ValueError( f"Gate {self.name!r} cannot fail without naming a failure." ) + if self.name in _STACKED_AUTHORITY_GATE_NAMES: + object.__setattr__( + self, + "_details_protocol_5_snapshot", + pickle.dumps(dict(self.details), protocol=5), + ) @dataclass(frozen=True) @@ -156,13 +174,89 @@ def failures(self) -> tuple[str, ...]: def to_manifest(self) -> dict[str, object]: """A JSON-ready summary for the release manifest.""" + for result in self.results: + authority = result.details.get("authority") + components = ( + authority.get("components") if isinstance(authority, Mapping) else None + ) + targets = result.details.get("targets") + recognizable_stacked_receipt = ( + isinstance(authority, Mapping) + and ( + str(authority.get("authority_id", "")).startswith( + "us_stacked_spine_authority" + ) + or bool( + { + "authority_form", + "declared_authority_form", + "canonical", + "canonical_identity", + "canonical_content", + "integrity_valid", + "digest_matches_declared", + "production_manifest_permitted", + } + & set(authority) + ) + or ( + isinstance(components, Mapping) + and set(components) + == { + "gap_fill_plan", + "declared_surface", + "metric_registry", + "support_profile", + } + ) + ) + ) or ( + isinstance(targets, Mapping) + and any( + isinstance(receipt, Mapping) + and ( + "authority_form" in receipt + or { + "authority_sha256", + "plan_sha256", + "surface_sha256", + }.issubset(receipt) + ) + for receipt in targets.values() + ) + ) + if result.name not in _STACKED_AUTHORITY_GATE_NAMES: + if recognizable_stacked_receipt: + raise ValueError( + f"Gate {result.name!r} carries a stacked authority receipt " + "under an unrecognized gate name; production manifest " + "emission is forbidden." + ) + continue + snapshot = result._details_protocol_5_snapshot + if ( + snapshot is None + or pickle.dumps(dict(result.details), protocol=5) != snapshot + ): + raise ValueError( + f"Gate {result.name!r} details changed after evaluation; " + "production manifest emission is forbidden." + ) + # Import lazily to keep the generic gates shard independent during + # module initialization. The stacked doctrine owns the trusted + # canonical surface and digests; emission must not trust receipts. + from populace.build.us_runtime.stacked_spine import ( + _validate_stacked_gate_manifest_details, + ) + + _validate_stacked_gate_manifest_details(result.name, result.details) return { "passed": self.passed, "gates": { result.name: { "passed": result.passed, "failures": list(result.failures), - "details": dict(result.details), + "details": deepcopy(dict(result.details)), } for result in self.results }, diff --git a/packages/populace-build/src/populace/build/us_runtime/puf_support.py b/packages/populace-build/src/populace/build/us_runtime/puf_support.py index 8c09caaf..a33a5eba 100644 --- a/packages/populace-build/src/populace/build/us_runtime/puf_support.py +++ b/packages/populace-build/src/populace/build/us_runtime/puf_support.py @@ -9,6 +9,8 @@ from __future__ import annotations +import hashlib +import json from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from typing import Any @@ -51,6 +53,9 @@ __all__ = [ "BASE_ASEC_SUPPORT_CHANNEL", "PufTaxDetailChainInputs", + "PUF_ABSENT_CELLS_LEGACY_ZERO_FILL", + "PUF_ABSENT_CELLS_PRESERVE_NULLS", + "PUF_CLONE_ATTACHMENT_MANIFEST_KEY", "PUF_TAX_DETAIL_CLONE_INDEX", "PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS", "PUF_TAX_DETAIL_SUPPORT_CHANNEL", @@ -73,10 +78,32 @@ "support_clone_index_column", "support_role_series", "support_source_id_column", + "validate_puf_clone_attachment", ] US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" +#: Frozen receipt binding a seeded clone attachment (populace#578 revision +#: item 3) to the live rows: fraction, seed, the floor-rule counts, and the +#: digest of the attached households' assembly-unique source IDs. +PUF_CLONE_ATTACHMENT_MANIFEST_KEY = "us_puf_clone_attachment_manifest" +_PUF_CLONE_ATTACHMENT_MANIFEST_VERSION = 1 + +#: Finalization policies for cells the PUF pass does not own (populace#578 +#: revision, audit item 1). The legacy policy reproduces the historical +#: two-arm behavior byte for byte: every requested output column is coerced +#: with a global ``fillna(0.0)``, so cells that were never imputed read as +#: observed zeros. The preserve-nulls policy is the stacked-spine doctrine: +#: absence stays null until an authorized stage fills it — finalization only +#: writes the PUF clone arm's cells, creates missing columns as null, and +#: never converts absence into an observed ``0.0``. +PUF_ABSENT_CELLS_LEGACY_ZERO_FILL = "legacy_zero_fill" +PUF_ABSENT_CELLS_PRESERVE_NULLS = "preserve_nulls" +_PUF_ABSENT_CELLS_POLICIES = ( + PUF_ABSENT_CELLS_LEGACY_ZERO_FILL, + PUF_ABSENT_CELLS_PRESERVE_NULLS, +) + #: The name the PUF tax-detail support fit records in the build weights audit #: (populace #300). Stable so a release manifest and its allowlist can refer to #: this fit by name. @@ -470,6 +497,8 @@ def clone_us_frame_for_puf_support( frame: Frame, *, channels: Sequence[str] = _DEFAULT_SUPPORT_CHANNELS, + clone_attachment_fraction: float | None = None, + clone_attachment_seed: int | None = None, ) -> Frame: """Clone a US frame into support channels for PUF detail imputation. @@ -481,6 +510,18 @@ def clone_us_frame_for_puf_support( channels: The canonical ``("asec", "puf_tax_detail")`` operator-role pair. Custom roles are rejected at this boundary because downstream operators accept only these two roles. + clone_attachment_fraction: Optional seeded-attachment fraction in + ``(0, 1]`` (populace#578 revision item 3). When set — assembled + frames only — the PUF clone arm attaches to a seeded whole- + household sample of the spine: ``floor(fraction * households)`` + households keep their clone pair at half weight each, every other + household keeps a single full-weight native lineage, and partial + attachments carry a manifest binding fraction, seed, and the + realized selection digest to the live rows. ``None`` and exact + ``1.0`` both return the ordinary full two-arm clone unchanged. + clone_attachment_seed: Non-negative selection seed; required exactly + when a fraction is given so the attachment identity is always + explicit. Returns: A new frame with every entity table cloned once per support role, all @@ -489,14 +530,44 @@ def clone_us_frame_for_puf_support( Raises: ValueError: If the frame is not US-schema, channel names are invalid, - metadata is partial or already operated, or an ID remapping would - collide. + metadata is partial or already operated, an ID remapping would + collide, or the attachment configuration is invalid. """ if frame.schema != US_SCHEMA: raise ValueError("PUF support expansion currently requires the US schema.") support_channels = _validate_channels(channels) has_assembly_provenance = _has_native_assembly_provenance(frame) + if (clone_attachment_fraction is None) != (clone_attachment_seed is None): + raise ValueError( + "clone_attachment_fraction and clone_attachment_seed must be " + "provided together; a seeded attachment identity is never implicit." + ) + if clone_attachment_fraction is not None: + if not has_assembly_provenance: + raise ValueError( + "Seeded clone attachment requires an assembled frame with " + "native support provenance." + ) + if ( + isinstance(clone_attachment_fraction, bool) + or not isinstance(clone_attachment_fraction, (int, float)) + or not np.isfinite(clone_attachment_fraction) + or not 0.0 < float(clone_attachment_fraction) <= 1.0 + ): + raise ValueError( + "clone_attachment_fraction must be a finite number in (0, 1]; " + f"got {clone_attachment_fraction!r}." + ) + if ( + isinstance(clone_attachment_seed, bool) + or not isinstance(clone_attachment_seed, int) + or clone_attachment_seed < 0 + ): + raise ValueError( + "clone_attachment_seed must be a non-negative integer; got " + f"{clone_attachment_seed!r}." + ) if has_assembly_provenance: validate_assembly_provenance( frame, @@ -564,9 +635,354 @@ def clone_us_frame_for_puf_support( result, boundary="PUF support clone output", ) + if clone_attachment_fraction is not None: + assert clone_attachment_seed is not None # validated at entry + if float(clone_attachment_fraction) == 1.0: + validate_puf_clone_attachment( + result, + boundary="PUF support full-clone identity output", + expected_fraction=float(clone_attachment_fraction), + expected_seed=clone_attachment_seed, + ) + return result + result = _attach_clone_arm_to_seeded_sample( + result, + fraction=clone_attachment_fraction, + seed=clone_attachment_seed, + ) + validate_assembly_provenance( + result, + boundary="PUF support clone attachment output", + ) + validate_puf_clone_attachment( + result, + boundary="PUF support clone attachment output", + expected_fraction=float(clone_attachment_fraction), + expected_seed=clone_attachment_seed, + ) return result +def _attach_clone_arm_to_seeded_sample( + cloned: Frame, + *, + fraction: float, + seed: int, +) -> Frame: + """Keep the PUF clone pair on a seeded household sample only. + + Operates on the full two-arm clone so the selected pairs are byte- + identical to the full expansion: the detail lineages of unselected + households are dropped whole (via :meth:`Frame.select`) and those + households' native weights are restored to their pre-clone values. + Mass is conserved: selected households carry half weight on each arm, + unselected households carry full weight on their native lineage. + """ + + household = cloned.table("household") + household_clone = household[support_clone_index_column("household")] + household_source = household[support_source_id_column("household")] + native_source_ids = np.sort( + household_source.loc[household_clone.eq(0)].to_numpy(dtype=np.int64) + ) + eligible = int(len(native_source_ids)) + requested = int(np.floor(fraction * eligible)) + if requested < 1: + raise ValueError( + f"clone_attachment_fraction {fraction!r} floors to zero households " + f"(floor(fraction * eligible) with eligible={eligible}); the PUF " + "pass requires at least one attached household." + ) + rng = np.random.default_rng(seed) + selected = np.sort(rng.choice(native_source_ids, size=requested, replace=False)) + selected_set = frozenset(int(value) for value in selected) + + person = cloned.table("person") + person_clone = person[support_clone_index_column("person")] + person_household_source = _person_household_source_ids(cloned) + keep_person = person_clone.eq(0).to_numpy() | np.isin( + person_household_source, selected + ) + trimmed = cloned.select(keep_person) + + trimmed_household = trimmed.table("household") + trimmed_clone = trimmed_household[support_clone_index_column("household")] + trimmed_source = trimmed_household[support_source_id_column("household")] + restored = np.array( + trimmed.weights_for("household").values, + dtype=np.float64, + copy=True, + ) + unattached_native = ( + trimmed_clone.eq(0) & ~trimmed_source.isin(list(selected_set)) + ).to_numpy() + restored[unattached_native] *= 2.0 + weights = { + entity: trimmed.weights_for(entity) for entity in trimmed.weighted_entities + } + weights["household"] = Weights(restored, trimmed.weights_for("household").kind) + + manifest = { + PUF_CLONE_ATTACHMENT_MANIFEST_KEY: { + "version": _PUF_CLONE_ATTACHMENT_MANIFEST_VERSION, + "clone_attachment_fraction": float(fraction), + "clone_attachment_seed": int(seed), + "eligible_household_count": eligible, + "requested_household_count": requested, + "realized_household_count": requested, + "exact_count_rule": "floor(fraction * eligible)", + "selected_household_source_ids_sha256": _source_ids_sha256(selected), + } + } + trimmed_metadata = {**trimmed.metadata, **manifest} + trimmed_mass_log = trimmed.mass_log + return Frame( + {entity: trimmed.table(entity) for entity in trimmed.entities}, + trimmed.schema, + weights, + trimmed.strata, + mass_log=trimmed_mass_log, + metadata=trimmed_metadata, + ) + + +def _person_household_source_ids(frame: Frame) -> np.ndarray: + """Map each person row to its household's assembly-unique source ID.""" + + household = frame.table("household") + lookup = pd.Series( + household[support_source_id_column("household")].to_numpy(), + index=household["household_id"].to_numpy(), + ) + mapped = frame.table("person")["person_household_id"].map(lookup) + if mapped.isna().any(): + raise ValueError( + "Clone attachment cannot resolve household source IDs for " + f"{int(mapped.isna().sum())} person row(s)." + ) + return mapped.to_numpy(dtype=np.int64) + + +def validate_puf_clone_attachment( + frame: Frame, + *, + boundary: str, + expected_fraction: float | None = None, + expected_seed: int | None = None, +) -> Mapping[str, Any]: + """Validate the clone-attachment manifest against live clone lineages. + + The realized clone-pair count, the selection digest over live detail-arm + household source IDs, the floor rule, and per-pair weight symmetry must + all match the manifest; any mutation of the sample, the counts, or the + manifest fails closed with a named error. An explicitly expected full + attachment uses the metadata-symmetric ordinary clone instead: exact + native/detail household lineage and pair-weight identity are validated + and returned as an out-of-frame authority receipt. + """ + + if (expected_fraction is None) != (expected_seed is None): + raise ValueError( + f"{boundary}: expected clone attachment fraction and seed must be " + "provided together." + ) + if expected_fraction is not None: + if ( + isinstance(expected_fraction, bool) + or not isinstance(expected_fraction, (int, float)) + or not np.isfinite(expected_fraction) + or not 0.0 < float(expected_fraction) <= 1.0 + ): + raise ValueError( + f"{boundary}: expected clone attachment fraction must be a " + f"finite number in (0, 1], got {expected_fraction!r}." + ) + if ( + isinstance(expected_seed, bool) + or not isinstance(expected_seed, int) + or expected_seed < 0 + ): + raise ValueError( + f"{boundary}: expected clone attachment seed must be a " + f"non-negative integer, got {expected_seed!r}." + ) + + manifest = frame.metadata.get(PUF_CLONE_ATTACHMENT_MANIFEST_KEY) + if expected_fraction is not None and float(expected_fraction) == 1.0: + if manifest is not None: + raise ValueError( + f"{boundary}: full-clone metadata symmetry failed: attachment " + "manifest must be absent from both full-clone paths." + ) + return _validate_full_clone_identity( + frame, + boundary=boundary, + expected_seed=expected_seed, + ) + if manifest is None: + raise ValueError( + f"{boundary}: clone attachment manifest " + f"{PUF_CLONE_ATTACHMENT_MANIFEST_KEY!r} is absent." + ) + if ( + not isinstance(manifest, Mapping) + or manifest.get("version") != _PUF_CLONE_ATTACHMENT_MANIFEST_VERSION + ): + raise ValueError(f"{boundary}: clone attachment manifest is malformed.") + fraction = manifest.get("clone_attachment_fraction") + if not isinstance(fraction, float) or isinstance(fraction, bool): + raise ValueError( + f"{boundary}: clone attachment fraction must be a float, got {fraction!r}." + ) + seed = manifest.get("clone_attachment_seed") + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError( + f"{boundary}: clone attachment seed must be a non-negative " + f"integer, got {seed!r}." + ) + if expected_fraction is not None and fraction != float(expected_fraction): + raise ValueError( + f"{boundary}: clone attachment fraction {fraction!r} differs from " + f"the expected fraction {float(expected_fraction)!r}." + ) + if expected_seed is not None and seed != expected_seed: + raise ValueError( + f"{boundary}: clone attachment seed {seed!r} differs from the " + f"expected seed {expected_seed!r}." + ) + + household = frame.table("household") + clone_index = household[support_clone_index_column("household")] + source_ids = household[support_source_id_column("household")] + native = clone_index.eq(0) + detail = clone_index.eq(PUF_TAX_DETAIL_CLONE_INDEX) + eligible = int(native.sum()) + if int(manifest.get("eligible_household_count", -1)) != eligible: + raise ValueError( + f"{boundary}: live eligible household count {eligible} differs " + "from the clone attachment manifest " + f"{manifest.get('eligible_household_count')!r}." + ) + requested = int(manifest.get("requested_household_count", -1)) + if requested != int(np.floor(float(fraction) * eligible)): + raise ValueError( + f"{boundary}: clone attachment requested count {requested} " + "violates floor(fraction * eligible) for " + f"fraction={fraction!r}, eligible={eligible}." + ) + realized = int(manifest.get("realized_household_count", -1)) + live_detail_ids = np.sort(source_ids.loc[detail].to_numpy(dtype=np.int64)) + if realized != requested or int(detail.sum()) != realized: + raise ValueError( + f"{boundary}: live attached clone-pair count {int(detail.sum())} " + f"differs from the manifest's realized attachment count " + f"{realized} (requested {requested})." + ) + live_sha = _source_ids_sha256(live_detail_ids) + if live_sha != manifest.get("selected_household_source_ids_sha256"): + raise ValueError( + f"{boundary}: live attached-household selection digest {live_sha} " + "differs from the clone attachment manifest digest " + f"{manifest.get('selected_household_source_ids_sha256')!r}." + ) + native_ids = set(source_ids.loc[native].to_numpy(dtype=np.int64).tolist()) + orphaned = [int(value) for value in live_detail_ids if int(value) not in native_ids] + if orphaned: + raise ValueError( + f"{boundary}: {len(orphaned)} attached clone lineage(s) have no " + f"native partner (first: {orphaned[:5]})." + ) + + weights = np.asarray(frame.weights_for("household").values, dtype=np.float64) + pair_frame = pd.DataFrame( + { + "source_id": source_ids.to_numpy(dtype=np.int64), + "clone_index": clone_index.to_numpy(), + "weight": weights, + } + ) + attached = pair_frame[pair_frame["source_id"].isin(live_detail_ids)] + by_pair = attached.pivot_table( + index="source_id", + columns="clone_index", + values="weight", + aggfunc="sum", + ) + if not np.allclose( + by_pair[0].to_numpy(dtype=np.float64), + by_pair[PUF_TAX_DETAIL_CLONE_INDEX].to_numpy(dtype=np.float64), + rtol=1e-12, + atol=0.0, + ): + raise ValueError( + f"{boundary}: attached clone pairs must split household mass " + "evenly between the native and detail arms." + ) + return manifest + + +def _validate_full_clone_identity( + frame: Frame, + *, + boundary: str, + expected_seed: int, +) -> Mapping[str, Any]: + """Receipt exact full-clone coverage without mutating frame metadata.""" + + household = frame.table("household") + clone_index = household[support_clone_index_column("household")] + source_ids = household[support_source_id_column("household")] + native = clone_index.eq(0) + detail = clone_index.eq(PUF_TAX_DETAIL_CLONE_INDEX) + weights = frame.weights_for("household").values + + native_ids = source_ids.loc[native].to_numpy(dtype=np.int64) + detail_ids = source_ids.loc[detail].to_numpy(dtype=np.int64) + native_order = np.argsort(native_ids, kind="stable") + detail_order = np.argsort(detail_ids, kind="stable") + ordered_native_ids = native_ids[native_order] + ordered_detail_ids = detail_ids[detail_order] + native_weights = np.ascontiguousarray(weights[native.to_numpy()][native_order]) + detail_weights = np.ascontiguousarray(weights[detail.to_numpy()][detail_order]) + + lineages_exact = ( + bool((native | detail).all()) + and len(ordered_native_ids) == len(ordered_detail_ids) + and len(np.unique(ordered_native_ids)) == len(ordered_native_ids) + and len(np.unique(ordered_detail_ids)) == len(ordered_detail_ids) + and np.array_equal(ordered_native_ids, ordered_detail_ids) + ) + weights_exact = ( + native_weights.dtype == detail_weights.dtype + and native_weights.shape == detail_weights.shape + and native_weights.tobytes(order="C") == detail_weights.tobytes(order="C") + ) + if not lineages_exact or not weights_exact: + raise ValueError( + f"{boundary}: full-clone identity failed: native/detail household " + "lineages or pair weights are not exact." + ) + + return { + "authority_form": "full_clone_identity_no_manifest", + "clone_attachment_fraction": 1.0, + "clone_attachment_seed": expected_seed, + "eligible_household_count": len(ordered_native_ids), + "requested_household_count": len(ordered_native_ids), + "realized_household_count": len(ordered_detail_ids), + "exact_count_rule": "full native/detail identity", + "selected_household_source_ids_sha256": _source_ids_sha256(ordered_detail_ids), + } + + +def _source_ids_sha256(ids: np.ndarray) -> str: + payload = json.dumps( + [int(value) for value in np.asarray(ids).tolist()], + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode()).hexdigest() + + def puf_tax_unit_donor_from_arrays( arrays: Mapping[str, Sequence[Any]], *, @@ -912,6 +1328,8 @@ def impute_us_puf_tax_detail_support( fit_records: list[FitWeightRecord] | None = None, raw_predictions_callback: Callable[[pd.DataFrame], None] | None = None, tail_bound_diagnostics: list[dict[str, object]] | None = None, + require_complete_recipient_predictors: bool = False, + absent_cells: str = PUF_ABSENT_CELLS_LEGACY_ZERO_FILL, ) -> Frame: """Impute PUF-observed inputs onto the PUF support channel. @@ -940,6 +1358,12 @@ def impute_us_puf_tax_detail_support( tail_bound_diagnostics: Optional output sink for the per-target tail-bound records produced during finalization. Build callers publish these records with the QRF-finalization telemetry. + require_complete_recipient_predictors: Stacked-spine doctrine switch + (populace#578): build recipient features null-preserving and fail + closed by name when any recipient row is missing a predictor + value, instead of the legacy silent zero-fill. + absent_cells: Finalization policy for cells outside the PUF clone arm + (see :func:`finalize_us_puf_tax_detail_predictions`). """ if frame.schema != US_SCHEMA: @@ -958,6 +1382,14 @@ def impute_us_puf_tax_detail_support( person_outputs = tuple(person_outputs) tax_unit_outputs = tuple(tax_unit_outputs) outputs = (*person_outputs, *tax_unit_outputs) + puf_mask = puf_tax_detail_clone_mask( + frame.table("tax_unit"), + entity="tax_unit", + ) + if not puf_mask.any(): + raise ValueError("PUF detail clone has no tax-unit rows.") + if require_complete_recipient_predictors: + _require_complete_recipient_predictor_sources(frame, puf_mask, predictors) donor_tax_units = donor_tax_units.copy() _add_predictor_aliases(donor_tax_units, predictors) missing_donor = [ @@ -992,13 +1424,13 @@ def impute_us_puf_tax_detail_support( # fit did not silently resolve unweighted (populace #300). fit_records.append(FitWeightRecord(US_PUF_SUPPORT_FIT_NAME, fitted.weight_kind)) - features = _tax_unit_feature_frame(frame, predictors) - puf_mask = puf_tax_detail_clone_mask( - frame.table("tax_unit"), - entity="tax_unit", + features = _tax_unit_feature_frame( + frame, + predictors, + preserve_nulls=require_complete_recipient_predictors, ) - if not puf_mask.any(): - raise ValueError("PUF detail clone has no tax-unit rows.") + if require_complete_recipient_predictors: + _require_complete_recipient_predictors(features, puf_mask, predictors) predictions = fitted.predict( features.loc[puf_mask, list(predictors)], release_models=True ) @@ -1011,6 +1443,7 @@ def impute_us_puf_tax_detail_support( person_outputs=person_outputs, tax_unit_outputs=tax_unit_outputs, tail_bound_diagnostics=tail_bound_diagnostics, + absent_cells=absent_cells, ) @@ -1021,8 +1454,16 @@ def prepare_us_puf_tax_detail_chain_inputs( predictors: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_PREDICTORS, person_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, tax_unit_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, + require_complete_recipient_predictors: bool = False, ) -> PufTaxDetailChainInputs: - """Prepare lossless donor and recipient inputs for targetwise QRF workers.""" + """Prepare lossless donor and recipient inputs for targetwise QRF workers. + + ``require_complete_recipient_predictors`` is the stacked-spine doctrine + switch (populace#578): recipient features are built null-preserving and a + missing predictor value on any recipient row is a named terminal failure + instead of a silent zero-fill. The legacy default keeps the historical + zero-filled feature surface byte for byte. + """ if frame.schema != US_SCHEMA: raise ValueError("PUF tax-detail support imputation requires the US schema.") @@ -1040,6 +1481,14 @@ def prepare_us_puf_tax_detail_chain_inputs( person_outputs = tuple(person_outputs) tax_unit_outputs = tuple(tax_unit_outputs) outputs = (*person_outputs, *tax_unit_outputs) + puf_mask = puf_tax_detail_clone_mask( + frame.table("tax_unit"), + entity="tax_unit", + ) + if not puf_mask.any(): + raise ValueError("PUF detail clone has no tax-unit rows.") + if require_complete_recipient_predictors: + _require_complete_recipient_predictor_sources(frame, puf_mask, predictors) donor_tax_units = donor_tax_units.copy() _add_predictor_aliases(donor_tax_units, predictors) missing_donor = [ @@ -1056,13 +1505,13 @@ def prepare_us_puf_tax_detail_chain_inputs( for column in donor.columns: donor[column] = pd.to_numeric(donor[column], errors="coerce").fillna(0.0) donor_frame = _tax_unit_model_frame(donor) - features = _tax_unit_feature_frame(frame, predictors) - puf_mask = puf_tax_detail_clone_mask( - frame.table("tax_unit"), - entity="tax_unit", + features = _tax_unit_feature_frame( + frame, + predictors, + preserve_nulls=require_complete_recipient_predictors, ) - if not puf_mask.any(): - raise ValueError("PUF detail clone has no tax-unit rows.") + if require_complete_recipient_predictors: + _require_complete_recipient_predictors(features, puf_mask, predictors) recipient_features = features.loc[puf_mask, list(predictors)].copy() recipient_tax_unit_ids = ( frame.table("tax_unit").loc[puf_mask, "tax_unit_id"].to_numpy(copy=True) @@ -1087,6 +1536,7 @@ def finalize_us_puf_tax_detail_predictions( tax_unit_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, tail_bound_diagnostics: list[dict[str, object]] | None = None, tail_bound_quantiles: Mapping[str, float] | None = None, + absent_cells: str = PUF_ABSENT_CELLS_LEGACY_ZERO_FILL, ) -> Frame: """Finalize a complete raw PUF QRF chain onto its support channel. @@ -1102,8 +1552,22 @@ def finalize_us_puf_tax_detail_predictions( invocation's exact output surface. Diagnostics report recipient-design- weighted mass over the affected raw tax-unit draws before and after clipping; build callers must provide the sink and publish every active cap. + + ``absent_cells`` selects the finalization policy for cells outside the PUF + clone arm (populace#578 audit item 1). The legacy default reproduces the + historical global zero-fill byte for byte. Under + :data:`PUF_ABSENT_CELLS_PRESERVE_NULLS` — the stacked-spine doctrine — + absence stays null on every row this pass does not own, and the + donor-rate sparsification and placement rewrites are scoped to the PUF + clone arm so no boundary converts absence into an observed zero. """ + if absent_cells not in _PUF_ABSENT_CELLS_POLICIES: + raise ValueError( + f"absent_cells must be one of {list(_PUF_ABSENT_CELLS_POLICIES)}; " + f"got {absent_cells!r}." + ) + preserve_nulls = absent_cells == PUF_ABSENT_CELLS_PRESERVE_NULLS person_outputs = tuple(person_outputs) tax_unit_outputs = tuple(tax_unit_outputs) outputs = (*person_outputs, *tax_unit_outputs) @@ -1233,7 +1697,11 @@ def finalize_us_puf_tax_detail_predictions( requested_components=person_outputs, ) for column in tax_unit_outputs: - _ensure_float_output_column(tables["tax_unit"], column) + _ensure_float_output_column( + tables["tax_unit"], + column, + preserve_nulls=preserve_nulls, + ) tables["tax_unit"].loc[puf_mask, column] = predictions[column].to_numpy() for column in tax_unit_outputs: if column in _PUF_TAX_DETAIL_SPARSE_TAX_UNIT_OUTPUTS: @@ -1246,6 +1714,7 @@ def finalize_us_puf_tax_detail_predictions( ), household_weights=frame.weights_for("household").values, tax_unit_clone_index=tax_unit_clone_index, + puf_role_only=preserve_nulls, ) person_puf_mask = puf_tax_detail_clone_mask( @@ -1253,7 +1722,11 @@ def finalize_us_puf_tax_detail_predictions( entity="person", ) for column in person_outputs: - _ensure_float_output_column(tables["person"], column) + _ensure_float_output_column( + tables["person"], + column, + preserve_nulls=preserve_nulls, + ) totals = pd.Series(predictions[column].to_numpy(), index=tax_unit_ids) if column in _PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: _write_person_tax_unit_boolean_counts( @@ -1288,6 +1761,7 @@ def finalize_us_puf_tax_detail_predictions( household_weights=frame.weights_for("household").values, person_clone_index=person_clone_index, tax_unit_clone_index=tax_unit_clone_index, + puf_role_only=preserve_nulls, ) for column in person_outputs: if column in _PUF_TAX_DETAIL_SIGNED_MASS_CALIBRATED_PERSON_OUTPUTS: @@ -1897,7 +2371,23 @@ def _reject_formula_owned_outputs( ) -def _tax_unit_feature_frame(frame: Frame, columns: Sequence[str]) -> pd.DataFrame: +def _tax_unit_feature_frame( + frame: Frame, + columns: Sequence[str], + *, + preserve_nulls: bool = False, +) -> pd.DataFrame: + """Build the tax-unit predictor surface under the active absence policy. + + The legacy policy zero-fills every missing predictor cell — the exact + boundary the populace#578 audit identified as collapsing recipient draws + to zero-conditioned degenerates. Under ``preserve_nulls`` absence + propagates as null (a leaf-alias sum is null wherever any component is + null, and an entirely absent component column is null everywhere) so the + strict recipient check can fail closed by name instead of a silent fill. + Structural person counts are not absence and stay zero-filled. + """ + tax_unit = frame.table("tax_unit") person = frame.table("person") result = pd.DataFrame(index=tax_unit.index) @@ -1909,7 +2399,10 @@ def _tax_unit_feature_frame(frame: Frame, columns: Sequence[str]) -> pd.DataFram source = tax_unit.get("filing_status") if source is None: raise ValueError("tax_unit table lacks filing-status input.") - result[column] = _filing_status_codes(source) + result[column] = _filing_status_codes( + source, + preserve_nulls=preserve_nulls, + ) elif source_column == "tax_unit_person_count": result[column] = ( person.groupby("person_tax_unit_id", sort=False) @@ -1919,54 +2412,210 @@ def _tax_unit_feature_frame(frame: Frame, columns: Sequence[str]) -> pd.DataFram .to_numpy(dtype=np.float64) ) elif source_column in tax_unit.columns: - result[column] = pd.to_numeric( - tax_unit[source_column], errors="coerce" - ).fillna(0.0) + numeric = pd.to_numeric( + tax_unit[source_column], + errors="raise" if preserve_nulls else "coerce", + ) + result[column] = numeric if preserve_nulls else numeric.fillna(0.0) else: - result[column] = _person_tax_unit_sum(frame, source_column) + result[column] = _person_tax_unit_sum( + frame, + source_column, + preserve_nulls=preserve_nulls, + ) return result -def _person_tax_unit_sum(frame: Frame, column: str) -> np.ndarray: +def _person_tax_unit_sum( + frame: Frame, + column: str, + *, + preserve_nulls: bool = False, +) -> np.ndarray: person = frame.table("person") tax_unit = frame.table("tax_unit") if column == "dividend_income" and column not in person.columns: values = _optional_person( - person, "non_qualified_dividend_income" - ) + _optional_person(person, "qualified_dividend_income") + person, + "non_qualified_dividend_income", + preserve_nulls=preserve_nulls, + ) + _optional_person( + person, + "qualified_dividend_income", + preserve_nulls=preserve_nulls, + ) elif column not in person.columns and column in _PREDICTOR_LEAF_ALIASES: values = np.zeros(len(person), dtype=np.float64) for leaf in _PREDICTOR_LEAF_ALIASES[column]: - values += _optional_person(person, leaf) + values += _optional_person(person, leaf, preserve_nulls=preserve_nulls) else: if column not in person.columns: raise ValueError( f"Cannot build tax-unit predictor {column!r}; no matching " "tax_unit column or person column exists." ) - values = pd.to_numeric(person[column], errors="coerce").fillna(0.0) - grouped = ( - pd.DataFrame( - { - "person_tax_unit_id": person["person_tax_unit_id"], - column: np.asarray(values, dtype=np.float64), - } + numeric = pd.to_numeric( + person[column], + errors="raise" if preserve_nulls else "coerce", ) - .groupby("person_tax_unit_id", sort=False)[column] - .sum() - ) + if not preserve_nulls: + numeric = numeric.fillna(0.0) + values = numeric + grouped_frame = pd.DataFrame( + { + "person_tax_unit_id": person["person_tax_unit_id"], + column: np.asarray(values, dtype=np.float64), + } + ).groupby("person_tax_unit_id", sort=False)[column] + if preserve_nulls: + # ``sum`` normally skips partial nulls, which would turn a missing + # member-level source into an observed unit total. Poison any such + # group so strict recipient validation sees the absence by name. + grouped = grouped_frame.sum(min_count=1) + grouped[grouped_frame.count() != grouped_frame.size()] = np.nan + return grouped.reindex(tax_unit["tax_unit_id"]).to_numpy() + grouped = grouped_frame.sum() return grouped.reindex(tax_unit["tax_unit_id"]).fillna(0.0).to_numpy() -def _optional_person(person: pd.DataFrame, column: str) -> np.ndarray: +def _optional_person( + person: pd.DataFrame, + column: str, + *, + preserve_nulls: bool = False, +) -> np.ndarray: if column not in person.columns: + if preserve_nulls: + return np.full(len(person), np.nan, dtype=np.float64) return np.zeros(len(person), dtype=np.float64) - return pd.to_numeric(person[column], errors="coerce").fillna(0.0).to_numpy() + numeric = pd.to_numeric( + person[column], + errors="raise" if preserve_nulls else "coerce", + ) + if not preserve_nulls: + numeric = numeric.fillna(0.0) + return numeric.to_numpy(dtype=np.float64) + + +def _require_complete_recipient_predictor_sources( + frame: Frame, + recipient_mask: np.ndarray, + predictors: Sequence[str], +) -> None: + """Reject raw recipient absence before feature conversion or coercion.""" + + tax_unit = frame.table("tax_unit") + person = frame.table("person") + recipient_ids = tax_unit.loc[recipient_mask, "tax_unit_id"] + offenders: dict[str, int] = {} + for predictor in predictors: + source = _predictor_source_column(predictor) + if source == "filing_status_code": + values = tax_unit.get("filing_status_input") + if values is None: + values = tax_unit.get("filing_status") + missing = ( + np.ones(int(recipient_mask.sum()), dtype=bool) + if values is None + else values.loc[recipient_mask].isna().to_numpy() + ) + elif source == "tax_unit_person_count": + missing = np.zeros(int(recipient_mask.sum()), dtype=bool) + elif source in tax_unit.columns: + missing = tax_unit.loc[recipient_mask, source].isna().to_numpy() + else: + if source == "dividend_income" and source not in person.columns: + source_columns = ( + "non_qualified_dividend_income", + "qualified_dividend_income", + ) + elif source not in person.columns and source in _PREDICTOR_LEAF_ALIASES: + source_columns = _PREDICTOR_LEAF_ALIASES[source] + else: + source_columns = (source,) + absent_columns = [ + column for column in source_columns if column not in person.columns + ] + if absent_columns: + missing = np.ones(int(recipient_mask.sum()), dtype=bool) + else: + link = person["person_tax_unit_id"] + relevant = link.isin(recipient_ids) + observed_ids = set(link.loc[relevant].tolist()) + null_source = ( + person.loc[relevant, list(source_columns)].isna().any(axis=1) + ) + null_ids = set(link.loc[relevant].loc[null_source].tolist()) + missing = np.asarray( + [ + tax_unit_id not in observed_ids or tax_unit_id in null_ids + for tax_unit_id in recipient_ids + ], + dtype=bool, + ) + count = int(missing.sum()) + if count: + offenders[str(predictor)] = count + if offenders: + raise ValueError( + "PUF recipient predictor source(s) have missing values before " + f"coercion: {offenders} (of {int(len(recipient_ids))} recipient rows). " + "This is a terminal stacked-spine failure; gap-fill the source " + "before the PUF pass." + ) -def _ensure_float_output_column(table: pd.DataFrame, column: str) -> None: +def _require_complete_recipient_predictors( + features: pd.DataFrame, + recipient_mask: np.ndarray, + predictors: Sequence[str], +) -> None: + """Fail closed when any recipient row is missing a predictor value. + + The populace#578 audit found the primary QRF silently zero-filling + target-like predictors on recipient rows whose source never measured + them, collapsing those draws to degenerate near-zero values. Under the + stacked-spine doctrine that absence is a named terminal failure: the + PUF pass runs only after gap-fill has made every predictor observable on + every origin. + """ + + recipient = features.loc[recipient_mask, list(predictors)] + null_counts = recipient.isna().sum() + offenders = { + str(name): int(count) for name, count in null_counts.items() if int(count) + } + if offenders: + raise ValueError( + "PUF recipient predictor(s) have missing values on recipient " + f"rows: {offenders} (of {int(len(recipient))} recipient rows). " + "The primary QRF must not zero-fill absence (populace#578); " + "gap-fill the stacked spine before the PUF pass." + ) + + +def _ensure_float_output_column( + table: pd.DataFrame, + column: str, + *, + preserve_nulls: bool = False, +) -> None: + """Coerce one requested output column to float64 under the active policy. + + The legacy policy globally converts missing cells to ``0.0`` (the + historical two-arm behavior). Under the preserve-nulls doctrine a missing + column materializes as null and existing nulls survive the coercion: + absence must stay null until the stage that owns those cells fills them + (populace#578 audit item 1). Preserve-nulls coercion is also + parse-strict — a non-numeric observed value fails closed instead of + silently becoming absence. + """ + if column not in table.columns: - table[column] = 0.0 + table[column] = np.nan if preserve_nulls else 0.0 + return + if preserve_nulls: + table[column] = pd.to_numeric(table[column], errors="raise").astype("float64") return table[column] = ( pd.to_numeric(table[column], errors="coerce").fillna(0.0).astype("float64") @@ -2154,8 +2803,14 @@ def _sparsify_tax_unit_output_to_donor_positive_rate( household_weights: np.ndarray, tax_unit_clone_index: str | None = None, tax_unit_channel: str | None = None, + puf_role_only: bool = False, ) -> None: - """Prune a sparse tax-unit amount to the donor's weighted positive rate.""" + """Prune a sparse tax-unit amount to the donor's weighted positive rate. + + ``puf_role_only`` scopes the pruning to the PUF clone arm: under the + preserve-nulls doctrine other arms' cells are source- or gap-fill-owned + (observed values or authorized nulls) and must never be rewritten here. + """ if (tax_unit_clone_index is None) == (tax_unit_channel is None): raise ValueError("Provide exactly one tax-unit support role column.") @@ -2163,6 +2818,11 @@ def _sparsify_tax_unit_output_to_donor_positive_rate( tax_unit_clone_index if tax_unit_clone_index is not None else tax_unit_channel ) assert tax_unit_role_column is not None + puf_role: int | str = ( + PUF_TAX_DETAIL_CLONE_INDEX + if tax_unit_clone_index is not None + else PUF_TAX_DETAIL_SUPPORT_CHANNEL + ) positive_rate = float(np.clip(donor_positive_rate, 0.0, 1.0)) person = tables["person"] household = tables["household"] @@ -2183,7 +2843,12 @@ def _sparsify_tax_unit_output_to_donor_positive_rate( ) tax_unit_weight = tax_unit_household_id.map(household_weight).fillna(0.0) - for clone_index in tax_unit[tax_unit_role_column].dropna().unique(): + role_values = tax_unit[tax_unit_role_column].dropna().unique() + if puf_role_only: + role_values = np.asarray( + [role_value for role_value in role_values if role_value == puf_role] + ) + for clone_index in role_values: clone_mask = tax_unit[tax_unit_role_column] == clone_index channel_rows = tax_unit.loc[clone_mask] amounts = pd.Series( @@ -2229,6 +2894,7 @@ def _sparsify_tax_unit_person_output_to_donor_positive_rate( tax_unit_clone_index: str | None = None, person_channel: str | None = None, tax_unit_channel: str | None = None, + puf_role_only: bool = False, ) -> None: if (person_clone_index is None) == (person_channel is None) or ( (tax_unit_clone_index is None) == (tax_unit_channel is None) @@ -2274,7 +2940,7 @@ def _sparsify_tax_unit_person_output_to_donor_positive_rate( ) clone_indices = tax_unit[tax_unit_role_column].dropna().unique() - if column in _PUF_TAX_DETAIL_PRESERVE_BASE_ASEC_OUTPUTS: + if puf_role_only or column in _PUF_TAX_DETAIL_PRESERVE_BASE_ASEC_OUTPUTS: clone_indices = np.asarray( [clone_index for clone_index in clone_indices if clone_index == puf_role] ) @@ -2657,9 +3323,16 @@ def _tax_unit_source_values( return None -def _filing_status_codes(values: Sequence[Any]) -> np.ndarray: +def _filing_status_codes( + values: Sequence[Any], + *, + preserve_nulls: bool = False, +) -> np.ndarray: decoded = pd.Series(values).map(_decode_status).str.upper() - return decoded.map(_FILING_STATUS_CODES).fillna(0.0).to_numpy(dtype=np.float64) + codes = decoded.map(_FILING_STATUS_CODES) + if not preserve_nulls: + codes = codes.fillna(0.0) + return codes.to_numpy(dtype=np.float64) def _decode_status(value: Any) -> str: diff --git a/packages/populace-build/src/populace/build/us_runtime/stacked_spine.py b/packages/populace-build/src/populace/build/us_runtime/stacked_spine.py new file mode 100644 index 00000000..16366ee7 --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/stacked_spine.py @@ -0,0 +1,2979 @@ +"""US stacked-spine pilot: one origin-labeled spine (populace#578 revision). + +The ratified #578 increment-2 revision removes the two-spine / two-pipeline +agreement seam instead of patching it: ASEC and a seeded ACS household sample +are assembled into ONE spine whose rows carry their origin in the +receipt-validated support-channel columns. Survey-specific fields are then +gap-filled cross-origin with native predictors, a single PUF pass runs after +gap-fill, and by-origin statistics replace spine-vs-spine agreement. + +This module is a source-spine provenance OWNER (see the reviewed allowlist in +``test_us_spine_blindness.py``): stacking, gap-fill donor routing, activation +authority, the pre-simulation completeness gate, and the by-origin battery are +exactly the surfaces that must read origin labels. Population operators stay +spine-blind; this module selects donors and verifies activation authority so +they never have to. + +Weight harmonization (the two-arm P-lineage precedent) +------------------------------------------------------ +Both origins jointly represent the same population once, exactly like the +ASEC/PUF support channels in :mod:`populace.build.us_runtime.puf_support` +(each channel receives a declared share of the incoming mass so the population +does not double). :func:`~populace.build.us_runtime.spine_assembly.assemble_spines` +implements the allocation: for a household ``i`` of arm ``s`` with incoming +weight ``w_i``, + + ``w_i' = w_i * share_s * M_anchor / M_s`` + +where ``M_s`` is arm ``s``'s incoming household mass and ``M_anchor`` is the +mass-anchor arm's incoming mass. For the seeded ACS sample, +``M_acs_sample ~= fraction * M_acs_full``, so the allocation factor contains +the inverse-sampling upweighting ``1 / fraction`` automatically; the realized +per-arm scale factors are receipted rather than assumed. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import pickle +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from populace.build.gates import FitWeightRecord, GateResult +from populace.build.serialization_dtypes import canonicalize_table_string_dtypes +from populace.build.us_runtime.acs_transfer import ( + DEFAULT_ACS_TRANSFER_MAX_TARGETS_PER_FIT, + AcsTransferResult, + AcsTransferTargetBank, + TargetFamilies, + declared_acs_transfer_target_families, + transfer_acs_inputs, +) +from populace.build.us_runtime.puf_support import ( + PUF_ABSENT_CELLS_PRESERVE_NULLS, + clone_us_frame_for_puf_support, + impute_us_puf_tax_detail_support, + validate_puf_clone_attachment, +) +from populace.build.us_runtime.spine_assembly import assemble_spines +from populace.build.us_runtime.support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + SPINE_ASSEMBLY_MANIFEST_KEY, + spine_source_id_column, + support_channel_column, + support_clone_index_column, + validate_assembly_provenance, +) +from populace.frame import US_SCHEMA, Frame + +__all__ = [ + "ACS_STACKED_SUPPORT_CHANNEL", + "CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY", + "CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE", + "CANONICAL_STACKED_DECLARED_SURFACE", + "CANONICAL_STACKED_GAP_FILL_PLAN", + "DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES", + "ORIGIN_BATTERY_METRIC_KINDS", + "STACKED_PILOT_ACS_SAMPLE_FRACTION", + "STACKED_PILOT_ACS_SAMPLE_SEED", + "STACKED_SPINE_MANIFEST_KEY", + "AbsenceProof", + "GapFillDirection", + "GapFillResult", + "OriginBatterySpec", + "StackedPufPassResult", + "StackedSpineResult", + "assemble_stacked_spine", + "by_origin_battery", + "gap_fill_stacked_spine", + "run_stacked_puf_pass", + "sample_acs_households", + "stacked_completeness_gate", + "stacked_gap_fill_plan", + "validate_stacked_spine_frame", +] + +ACS_STACKED_SUPPORT_CHANNEL = "acs" + +#: Fixed arm shares for the pilot stack, matching the two-arm P-lineage +#: precedent (the ASEC/PUF support expansion splits incoming mass in half so +#: two arms jointly represent the population once). Calibration remains +#: downstream. +DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES: Mapping[str, float] = { + BASE_ASEC_SUPPORT_CHANNEL: 0.5, + ACS_STACKED_SUPPORT_CHANNEL: 0.5, +} + +STACKED_SPINE_MANIFEST_KEY = "us_stacked_spine_manifest" +_STACKED_SPINE_MANIFEST_VERSION = 1 +_EXACT_COUNT_RULE = "floor(fraction * eligible)" +_MASS_RTOL = 1e-9 + +#: The ratified pilot stack configuration (#578 revision): a seeded 10% ACS +#: household sample enters the spine. Scale-up beyond the pilot changes this +#: declared fraction, never an implicit default. +STACKED_PILOT_ACS_SAMPLE_FRACTION = 0.10 +STACKED_PILOT_ACS_SAMPLE_SEED = 578 + + +@dataclass(frozen=True) +class StackedSpineResult: + """One stacked spine plus its manifest-ready stack receipt.""" + + frame: Frame + receipt: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame): + raise TypeError( + "StackedSpineResult.frame must be a Frame, got " + f"{type(self.frame).__name__}." + ) + if not isinstance(self.receipt, Mapping): + raise TypeError("StackedSpineResult.receipt must be a mapping.") + + +def sample_acs_households( + acs: Frame, + *, + fraction: float, + seed: int, +) -> tuple[Frame, dict[str, object]]: + """Draw a seeded, whole-household ACS sample with an exact-count receipt. + + The realized count follows the deterministic exact-count rule + ``floor(fraction * eligible)``. Selection operates on the sorted + household-ID inventory so equal frames produce equal samples regardless of + incidental row order, and whole lineages (every entity row of a selected + household) enter the sample together via :meth:`Frame.select`. + + Args: + acs: The full pre-assembly ACS source frame (US schema, household + weights, no support provenance). + fraction: Household sampling fraction in ``(0, 1]``. + seed: Non-negative integer seed for the selection RNG. + + Returns: + The sampled frame and a JSON-ready receipt with eligible, requested, + and realized counts, the selection digest, and mass bookkeeping. + + Raises: + TypeError: If ``acs`` is not a Frame. + ValueError: If the schema, configuration, or realized selection + violates the sampling contract (including a floor of zero + households, which fails closed). + """ + + if not isinstance(acs, Frame): + raise TypeError(f"acs must be a Frame, got {type(acs).__name__}.") + if acs.schema != US_SCHEMA: + raise ValueError("ACS household sampling requires the US entity schema.") + _validate_fraction(fraction) + _validate_seed(seed) + household_channel = support_channel_column("household") + if household_channel in acs.table("household").columns: + raise ValueError( + "ACS household sampling runs before assembly; the source frame " + f"already carries support provenance ({household_channel!r})." + ) + + household_ids = acs.table("household")["household_id"].to_numpy() + eligible = int(len(household_ids)) + incoming_mass = float(acs.weights_for("household").total) + requested = int(math.floor(fraction * eligible)) + if requested < 1: + raise ValueError( + f"ACS sample fraction {fraction!r} floors to zero households " + f"({_EXACT_COUNT_RULE} with eligible={eligible}); the stacked " + "spine requires at least one sampled household." + ) + + ordered_ids = np.sort(np.asarray(household_ids, copy=True)) + if requested == eligible: + selected_ids = ordered_ids + sampled = acs + else: + rng = np.random.default_rng(seed) + selected_ids = np.sort(rng.choice(ordered_ids, size=requested, replace=False)) + person_mask = ( + acs.table("person")["person_household_id"].isin(selected_ids).to_numpy() + ) + sampled = acs.select(person_mask) + + realized_ids = np.sort(sampled.table("household")["household_id"].to_numpy()) + if not np.array_equal(realized_ids, selected_ids): + raise ValueError( + "ACS household sampling realized a different household set than " + "it selected; whole-household selection failed." + ) + receipt: dict[str, object] = { + "fraction": float(fraction), + "seed": int(seed), + "eligible_household_count": eligible, + "requested_household_count": requested, + "realized_household_count": int(len(realized_ids)), + "exact_count_rule": _EXACT_COUNT_RULE, + "selected_household_ids_sha256": _ids_sha256(selected_ids), + "incoming_household_mass": incoming_mass, + "sampled_household_mass": float(sampled.weights_for("household").total), + } + return sampled, receipt + + +def assemble_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float, + acs_sample_seed: int, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpineResult: + """Assemble ASEC plus a seeded ACS household sample into one spine. + + The sample is drawn by :func:`sample_acs_households`, the combination + reuses the reviewed :func:`assemble_spines` seam unchanged, and the + resulting frame carries a stacked-spine manifest binding the sampling + configuration (fraction, seed), the realized selection digest, and the + per-arm weight-harmonization receipts to the live rows. Origin labels + survive as the ordinary support-channel columns. + + Returns: + A validated :class:`StackedSpineResult` whose receipt mirrors the + frozen manifest as a JSON-ready mapping. + """ + + shares = ( + dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES) + if household_mass_shares is None + else dict(household_mass_shares) + ) + sampled, sample_receipt = sample_acs_households( + acs, + fraction=acs_sample_fraction, + seed=acs_sample_seed, + ) + asec_incoming_mass = float(asec.weights_for("household").total) + incoming_masses = { + BASE_ASEC_SUPPORT_CHANNEL: asec_incoming_mass, + ACS_STACKED_SUPPORT_CHANNEL: float(sample_receipt["sampled_household_mass"]), + } + assembled = assemble_spines( + { + BASE_ASEC_SUPPORT_CHANNEL: asec, + ACS_STACKED_SUPPORT_CHANNEL: sampled, + }, + household_mass_shares=shares, + mass_anchor_channel=mass_anchor_channel, + ) + + harmonization = _harmonization_receipt( + assembled, + shares=shares, + anchor_mass=incoming_masses[mass_anchor_channel], + incoming_masses=incoming_masses, + ) + manifest: dict[str, object] = { + "version": _STACKED_SPINE_MANIFEST_VERSION, + "acs_sample_fraction": float(acs_sample_fraction), + "acs_sample_seed": int(acs_sample_seed), + "acs_sample": sample_receipt, + "household_mass_shares": { + channel: float(share) for channel, share in shares.items() + }, + "mass_anchor_channel": mass_anchor_channel, + "weight_harmonization": harmonization, + } + # The assembly metadata is preserved in full and augmented with the stack + # manifest; the mass history is carried unchanged from the same source. + stacked_metadata = {**assembled.metadata, STACKED_SPINE_MANIFEST_KEY: manifest} + stacked_mass_log = assembled.mass_log + stacked = Frame( + {entity: assembled.table(entity) for entity in assembled.entities}, + assembled.schema, + { + entity: assembled.weights_for(entity) + for entity in assembled.weighted_entities + }, + assembled.strata, + mass_log=stacked_mass_log, + metadata=stacked_metadata, + ) + validated = validate_stacked_spine_frame( + stacked, + boundary="stacked spine assembly output", + ) + return StackedSpineResult(frame=stacked, receipt=_json_ready(validated)) + + +def validate_stacked_spine_frame( + frame: Frame, + *, + boundary: str, +) -> Mapping[str, object]: + """Validate the stacked-spine manifest against the live origin labels. + + Layered on :func:`validate_assembly_provenance` (which already proves the + live channel counts against the frozen assembly manifest), this validator + binds the sampling identity: the manifest's fraction and seed must be + present and typed, the realized count must satisfy the exact-count rule, + the live native ACS household lineage must hash to the manifest's + selection digest, and the live per-arm household masses must match the + declared share allocation. Any mutation of the sample, the counts, or the + manifest fails closed with a named error. + """ + + validate_assembly_provenance(frame, boundary=boundary) + manifest = frame.metadata.get(STACKED_SPINE_MANIFEST_KEY) + if manifest is None: + raise ValueError( + f"{boundary}: stacked spine manifest {STACKED_SPINE_MANIFEST_KEY!r} " + "is absent." + ) + if not isinstance(manifest, Mapping): + raise ValueError(f"{boundary}: stacked spine manifest is malformed.") + if manifest.get("version") != _STACKED_SPINE_MANIFEST_VERSION: + raise ValueError( + f"{boundary}: stacked spine manifest has unsupported version " + f"{manifest.get('version')!r}." + ) + assembly = frame.metadata[SPINE_ASSEMBLY_MANIFEST_KEY] + channels = tuple(assembly["channels"]) + expected_channels = (BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL) + if set(channels) != set(expected_channels): + raise ValueError( + f"{boundary}: stacked spine requires exactly the channels " + f"{sorted(expected_channels)}; assembly declares {sorted(channels)}." + ) + + fraction = manifest.get("acs_sample_fraction") + seed = manifest.get("acs_sample_seed") + if not isinstance(fraction, float) or isinstance(fraction, bool): + raise ValueError( + f"{boundary}: stacked spine manifest acs_sample_fraction must be " + f"a float, got {fraction!r}." + ) + _validate_fraction(fraction, boundary=boundary) + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError( + f"{boundary}: stacked spine manifest acs_sample_seed must be a " + f"non-negative integer, got {seed!r}." + ) + + sample = manifest.get("acs_sample") + if not isinstance(sample, Mapping): + raise ValueError(f"{boundary}: stacked spine sample receipt is absent.") + required_keys = ( + "eligible_household_count", + "requested_household_count", + "realized_household_count", + "exact_count_rule", + "selected_household_ids_sha256", + ) + missing = [key for key in required_keys if key not in sample] + if missing: + raise ValueError( + f"{boundary}: stacked spine sample receipt is missing {missing}." + ) + if sample["exact_count_rule"] != _EXACT_COUNT_RULE: + raise ValueError( + f"{boundary}: stacked spine sample declares exact-count rule " + f"{sample['exact_count_rule']!r}; expected {_EXACT_COUNT_RULE!r}." + ) + eligible = int(sample["eligible_household_count"]) + requested = int(sample["requested_household_count"]) + realized = int(sample["realized_household_count"]) + if requested != int(math.floor(fraction * eligible)): + raise ValueError( + f"{boundary}: stacked spine requested household count {requested} " + f"violates {_EXACT_COUNT_RULE} for fraction={fraction!r}, " + f"eligible={eligible}." + ) + if realized != requested: + raise ValueError( + f"{boundary}: stacked spine realized household count {realized} " + f"differs from the requested count {requested}." + ) + + household = frame.table("household") + channel_values = household[support_channel_column("household")].astype(str) + clone_index = household[support_clone_index_column("household")] + native_acs = channel_values.eq(ACS_STACKED_SUPPORT_CHANNEL) & clone_index.eq(0) + live_count = int(native_acs.sum()) + if live_count != realized: + raise ValueError( + f"{boundary}: live native ACS household count {live_count} differs " + f"from the stacked spine manifest's realized count {realized}." + ) + live_ids = np.sort( + household.loc[native_acs, spine_source_id_column("household")].to_numpy() + ) + live_sha = _ids_sha256(live_ids) + if live_sha != sample["selected_household_ids_sha256"]: + raise ValueError( + f"{boundary}: live native ACS household lineage digest {live_sha} " + "differs from the stacked spine manifest's selection digest " + f"{sample['selected_household_ids_sha256']}." + ) + + shares = manifest.get("household_mass_shares") + if not isinstance(shares, Mapping) or set(shares) != set(expected_channels): + raise ValueError( + f"{boundary}: stacked spine manifest household_mass_shares must " + f"exactly cover {sorted(expected_channels)}." + ) + total_share = float(sum(float(value) for value in shares.values())) + if not np.isclose(total_share, 1.0, rtol=_MASS_RTOL, atol=_MASS_RTOL): + raise ValueError( + f"{boundary}: stacked spine household_mass_shares sum to " + f"{total_share!r}; expected 1.0." + ) + harmonization = manifest.get("weight_harmonization") + if not isinstance(harmonization, Mapping): + raise ValueError( + f"{boundary}: stacked spine weight-harmonization receipt is absent." + ) + mass_anchor_channel = manifest.get("mass_anchor_channel") + if mass_anchor_channel != channels[0]: + raise ValueError( + f"{boundary}: stacked spine mass_anchor_channel " + f"{mass_anchor_channel!r} differs from the assembly anchor " + f"channel {channels[0]!r}." + ) + live_anchor_mass = float(frame.weights_for("household").total) + anchor_arm = harmonization.get(mass_anchor_channel) + if not isinstance(anchor_arm, Mapping) or "incoming_mass" not in anchor_arm: + raise ValueError( + f"{boundary}: stacked spine weight-harmonization receipt for " + f"anchor {mass_anchor_channel!r} is malformed." + ) + anchor_incoming = float(anchor_arm["incoming_mass"]) + if not np.isclose( + anchor_incoming, + live_anchor_mass, + rtol=_MASS_RTOL, + atol=0.0, + ): + raise ValueError( + f"{boundary}: selected anchor {mass_anchor_channel!r} incoming " + f"mass {anchor_incoming!r} differs from live anchor mass " + f"{live_anchor_mass!r}." + ) + + weights = np.asarray(frame.weights_for("household").values, dtype=np.float64) + for channel in expected_channels: + arm = harmonization.get(channel) + if not isinstance(arm, Mapping) or not { + "allocated_mass", + "declared_allocation", + }.issubset(arm): + raise ValueError( + f"{boundary}: stacked spine weight-harmonization receipt for " + f"{channel!r} is malformed." + ) + live_mass = float(weights[channel_values.eq(channel).to_numpy()].sum()) + allocated = float(arm["allocated_mass"]) + declared_allocation = float(arm["declared_allocation"]) + expected_allocation = float(shares[channel]) * live_anchor_mass + if not np.isclose( + declared_allocation, + expected_allocation, + rtol=_MASS_RTOL, + atol=0.0, + ): + raise ValueError( + f"{boundary}: declared {channel!r} allocation " + f"{declared_allocation!r} differs from share " + f"{float(shares[channel])!r} times live anchor mass " + f"{live_anchor_mass!r}." + ) + if not np.isclose(live_mass, allocated, rtol=_MASS_RTOL, atol=0.0): + raise ValueError( + f"{boundary}: live {channel!r} household mass {live_mass!r} " + f"drifted from the allocated arm mass {allocated!r}." + ) + return manifest + + +def _harmonization_receipt( + assembled: Frame, + *, + shares: Mapping[str, float], + anchor_mass: float, + incoming_masses: Mapping[str, float], +) -> dict[str, dict[str, float]]: + household = assembled.table("household") + channel_values = household[support_channel_column("household")].astype(str) + weights = np.asarray(assembled.weights_for("household").values, dtype=np.float64) + receipt: dict[str, dict[str, float]] = {} + for channel, share in shares.items(): + incoming = float(incoming_masses[channel]) + allocated = float(weights[channel_values.eq(channel).to_numpy()].sum()) + receipt[channel] = { + "share": float(share), + "incoming_mass": incoming, + "allocated_mass": allocated, + "declared_allocation": float(share) * anchor_mass, + "scale_factor": allocated / incoming, + } + return receipt + + +def _ids_sha256(ids: np.ndarray) -> str: + payload = json.dumps( + [int(value) for value in np.asarray(ids).tolist()], + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _validate_fraction(fraction: float, *, boundary: str | None = None) -> None: + prefix = f"{boundary}: " if boundary else "" + if ( + isinstance(fraction, bool) + or not isinstance(fraction, (int, float)) + or not np.isfinite(fraction) + or not 0.0 < float(fraction) <= 1.0 + ): + raise ValueError( + f"{prefix}ACS sample fraction must be a finite number in (0, 1]; " + f"got {fraction!r}." + ) + + +def _validate_seed(seed: int) -> None: + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError( + f"ACS sample seed must be a non-negative integer; got {seed!r}." + ) + + +def _json_ready(value: object) -> dict[str, object]: + def thaw(item: object) -> object: + if isinstance(item, Mapping): + return {str(key): thaw(nested) for key, nested in item.items()} + if isinstance(item, (list, tuple)): + return [thaw(nested) for nested in item] + if isinstance(item, np.generic): + return item.item() + return item + + if not isinstance(value, Mapping): + raise TypeError("Stacked spine receipts must be mappings.") + return {str(key): thaw(item) for key, item in value.items()} + + +# --------------------------------------------------------------------------- +# Cross-origin gap-fill (charter item 2) +# --------------------------------------------------------------------------- + +_GAP_FILL_ASEC_TO_ACS = "asec_survey_to_acs" +_GAP_FILL_ACS_TO_ASEC = "acs_housing_to_asec" +_GAP_FILL_HOUSING_FAMILY = "housing" +_STACKED_AUTHORITY_ID = "us_stacked_spine_authority" +_STACKED_AUTHORITY_VERSION = 1 +_CANONICAL_AUTHORITY_FORM = "CANONICAL" +_NONCANONICAL_AUTHORITY_FORM = "NON-CANONICAL" + + +def _freeze_target_families(target_families: TargetFamilies) -> TargetFamilies: + """Recursively freeze an entity/family/target declaration.""" + + if not isinstance(target_families, Mapping): + raise TypeError("Target families must be a mapping.") + frozen_entities: dict[str, Mapping[str, tuple[str, ...]]] = {} + for entity, families in target_families.items(): + if not isinstance(entity, str) or not entity.strip(): + raise ValueError("Target-family entity names must be non-empty strings.") + if not isinstance(families, Mapping): + raise TypeError(f"Target families for {entity!r} must be a mapping.") + frozen_families: dict[str, tuple[str, ...]] = {} + for family, targets in families.items(): + if not isinstance(family, str) or not family.strip(): + raise ValueError("Target-family names must be non-empty strings.") + frozen_targets = tuple(targets) + if any( + not isinstance(target, str) or not target.strip() + for target in frozen_targets + ): + raise ValueError( + f"Target family {entity}/{family} contains an invalid target name." + ) + frozen_families[family] = frozen_targets + frozen_entities[entity] = MappingProxyType(frozen_families) + return MappingProxyType(frozen_entities) + + +@dataclass(frozen=True) +class GapFillDirection: + """One declared cross-origin fill: recipient origin <- donor origin. + + Activation authority is declared here, not inferred from nullness: the + named recipient channel's rows are the only rows the direction may fill, + and the named donor channel's native rows are the only donor evidence. + The transfer machinery itself stays spine-blind; this owner-level + declaration is what makes the run-7 silent-skip class impossible — a + direction either fills its declared families on its declared rows or + fails by name. + """ + + name: str + recipient_channel: str + donor_channel: str + target_families: TargetFamilies + + def __post_init__(self) -> None: + for label, value in ( + ("name", self.name), + ("recipient_channel", self.recipient_channel), + ("donor_channel", self.donor_channel), + ): + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"GapFillDirection.{label} must be a non-empty string." + ) + if self.recipient_channel == self.donor_channel: + raise ValueError( + "GapFillDirection must fill across origins; recipient and " + f"donor are both {self.donor_channel!r}." + ) + if not isinstance(self.target_families, Mapping) or not any( + families for families in self.target_families.values() + ): + raise ValueError( + f"GapFillDirection {self.name!r} declares no target families." + ) + object.__setattr__( + self, + "target_families", + _freeze_target_families(self.target_families), + ) + + +@dataclass(frozen=True) +class GapFillResult: + """The gap-filled stacked spine plus per-direction receipts.""" + + frame: Frame + receipt: Mapping[str, object] + transfer_results: Mapping[str, AcsTransferResult] = field(default_factory=dict) + + +def _build_stacked_gap_fill_plan( + families: TargetFamilies, +) -> tuple[GapFillDirection, ...]: + """Build a deeply frozen direction plan from a declared surface.""" + + survey_families: dict[str, dict[str, tuple[str, ...]]] = {} + housing_families: dict[str, dict[str, tuple[str, ...]]] = {} + for entity, entity_families in families.items(): + for family, targets in entity_families.items(): + bucket = ( + housing_families + if family == _GAP_FILL_HOUSING_FAMILY + else survey_families + ) + bucket.setdefault(entity, {})[family] = tuple(targets) + directions: list[GapFillDirection] = [] + if survey_families: + directions.append( + GapFillDirection( + name=_GAP_FILL_ASEC_TO_ACS, + recipient_channel=ACS_STACKED_SUPPORT_CHANNEL, + donor_channel=BASE_ASEC_SUPPORT_CHANNEL, + target_families=survey_families, + ) + ) + if housing_families: + directions.append( + GapFillDirection( + name=_GAP_FILL_ACS_TO_ASEC, + recipient_channel=BASE_ASEC_SUPPORT_CHANNEL, + donor_channel=ACS_STACKED_SUPPORT_CHANNEL, + target_families=housing_families, + ) + ) + return tuple(directions) + + +ORIGIN_BATTERY_METRIC_KINDS = ( + "boolean_incidence", + "rare_incidence", + "monetary_sign_separated", + "categorical_tvd", +) + + +@dataclass(frozen=True) +class _BatterySupportProfile: + profile_id: str + version: int + min_effective_support: int + + +@dataclass(frozen=True) +class _StackedAuthority: + """One digest-carrying, deeply immutable stacked-spine authority bundle.""" + + authority_id: str + version: int + gap_fill_plan: tuple[GapFillDirection, ...] + declared_surface: TargetFamilies + metric_registry: Mapping[tuple[str, str, str, int], str] + support_profile: _BatterySupportProfile + declared_component_sha256: Mapping[str, str] + declared_sha256: str + declared_form: str + + def __post_init__(self) -> None: + if not isinstance(self.authority_id, str) or not self.authority_id.strip(): + raise ValueError("Stacked authority_id must be a non-empty string.") + if isinstance(self.version, bool) or not isinstance(self.version, int): + raise ValueError("Stacked authority version must be an integer.") + plan = tuple(self.gap_fill_plan) + if any(not isinstance(direction, GapFillDirection) for direction in plan): + raise TypeError("Stacked authority plans require GapFillDirection values.") + object.__setattr__(self, "gap_fill_plan", plan) + object.__setattr__( + self, + "declared_surface", + _freeze_target_families(self.declared_surface), + ) + object.__setattr__( + self, + "metric_registry", + _freeze_metric_registry(self.metric_registry), + ) + if not isinstance(self.support_profile, _BatterySupportProfile): + raise TypeError( + "Stacked authority support_profile must be a _BatterySupportProfile." + ) + component_digests = dict(self.declared_component_sha256) + if set(component_digests) != { + "gap_fill_plan", + "declared_surface", + "metric_registry", + "support_profile", + }: + raise ValueError( + "Stacked authority must carry every component's declared digest." + ) + for name, digest in component_digests.items(): + _validate_sha256(digest, boundary=f"Stacked authority {name}") + object.__setattr__( + self, + "declared_component_sha256", + MappingProxyType(component_digests), + ) + _validate_sha256(self.declared_sha256, boundary="Stacked authority") + if self.declared_form not in { + _CANONICAL_AUTHORITY_FORM, + _NONCANONICAL_AUTHORITY_FORM, + }: + raise ValueError(f"Unknown stacked authority form {self.declared_form!r}.") + + +def _validate_sha256(value: object, *, boundary: str) -> None: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise ValueError(f"{boundary} digest must be a lowercase sha256.") + + +def _freeze_metric_registry( + registry: Mapping[tuple[str, str, str, int], str], +) -> Mapping[tuple[str, str, str, int], str]: + if not isinstance(registry, Mapping): + raise TypeError("The origin-battery metric registry must be a mapping.") + frozen: dict[tuple[str, str, str, int], str] = {} + for key, metric in registry.items(): + if ( + not isinstance(key, tuple) + or len(key) != 4 + or any(not isinstance(value, str) or not value for value in key[:3]) + or isinstance(key[3], bool) + or not isinstance(key[3], int) + or key[3] < 0 + ): + raise ValueError(f"Invalid origin-battery metric key {key!r}.") + if metric not in ORIGIN_BATTERY_METRIC_KINDS: + raise ValueError( + f"Origin-battery target {_battery_target_label(key)} declares " + f"unknown metric {metric!r}." + ) + frozen[key] = metric + return MappingProxyType(frozen) + + +def _surface_target_keys( + surface: TargetFamilies, +) -> tuple[tuple[str, str, str, int], ...]: + return tuple( + sorted( + (entity, family, target, 0) + for entity, families in surface.items() + for family, targets in families.items() + for target in targets + ) + ) + + +def _plan_target_keys( + plan: Sequence[GapFillDirection], +) -> tuple[tuple[str, str, str, int], ...]: + return tuple( + sorted( + (entity, family, target, 0) + for direction in plan + for entity, families in direction.target_families.items() + for family, targets in families.items() + for target in targets + ) + ) + + +def _surface_payload(surface: TargetFamilies) -> dict[str, object]: + return { + entity: {family: list(targets) for family, targets in families.items()} + for entity, families in surface.items() + } + + +def _plan_payload(plan: Sequence[GapFillDirection]) -> list[dict[str, object]]: + return [ + { + "name": direction.name, + "recipient_channel": direction.recipient_channel, + "donor_channel": direction.donor_channel, + "target_families": _surface_payload(direction.target_families), + } + for direction in plan + ] + + +def _metric_registry_payload( + registry: Mapping[tuple[str, str, str, int], str], +) -> list[dict[str, object]]: + return [ + { + "entity": entity, + "family": family, + "column": column, + "clone_index": clone_index, + "metric": registry[(entity, family, column, clone_index)], + } + for entity, family, column, clone_index in sorted(registry) + ] + + +def _support_profile_payload(profile: _BatterySupportProfile) -> dict[str, object]: + return { + "min_effective_support": profile.min_effective_support, + "profile_id": profile.profile_id, + "version": profile.version, + } + + +def _authority_component_payloads( + *, + gap_fill_plan: Sequence[GapFillDirection], + declared_surface: TargetFamilies, + metric_registry: Mapping[tuple[str, str, str, int], str], + support_profile: _BatterySupportProfile, +) -> dict[str, object]: + return { + "gap_fill_plan": _plan_payload(gap_fill_plan), + "declared_surface": _surface_payload(declared_surface), + "metric_registry": _metric_registry_payload(metric_registry), + "support_profile": _support_profile_payload(support_profile), + } + + +def _canonical_sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + + +def _authority_live_digests( + authority: _StackedAuthority, +) -> tuple[dict[str, str], str]: + payloads = _authority_component_payloads( + gap_fill_plan=authority.gap_fill_plan, + declared_surface=authority.declared_surface, + metric_registry=authority.metric_registry, + support_profile=authority.support_profile, + ) + component_digests = { + name: _canonical_sha256(payload) for name, payload in payloads.items() + } + bundle_digest = _canonical_sha256( + { + "authority_id": authority.authority_id, + "version": authority.version, + "components": payloads, + } + ) + return component_digests, bundle_digest + + +def _make_stacked_authority( + *, + authority_id: str, + version: int, + gap_fill_plan: Sequence[GapFillDirection], + declared_surface: TargetFamilies, + metric_registry: Mapping[tuple[str, str, str, int], str], + support_profile: _BatterySupportProfile, + declared_form: str, + declared_component_sha256: Mapping[str, str] | None = None, + declared_sha256: str | None = None, +) -> _StackedAuthority: + frozen_plan = tuple(gap_fill_plan) + frozen_surface = _freeze_target_families(declared_surface) + frozen_registry = _freeze_metric_registry(metric_registry) + component_payloads = _authority_component_payloads( + gap_fill_plan=frozen_plan, + declared_surface=frozen_surface, + metric_registry=frozen_registry, + support_profile=support_profile, + ) + live_components = { + name: _canonical_sha256(payload) for name, payload in component_payloads.items() + } + live_bundle = _canonical_sha256( + { + "authority_id": authority_id, + "version": version, + "components": component_payloads, + } + ) + return _StackedAuthority( + authority_id=authority_id, + version=version, + gap_fill_plan=frozen_plan, + declared_surface=frozen_surface, + metric_registry=frozen_registry, + support_profile=support_profile, + declared_component_sha256=( + live_components + if declared_component_sha256 is None + else declared_component_sha256 + ), + declared_sha256=live_bundle if declared_sha256 is None else declared_sha256, + declared_form=declared_form, + ) + + +def _canonical_metric_registry( + surface: TargetFamilies, +) -> Mapping[tuple[str, str, str, int], str]: + boolean_columns = { + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "is_incapable_of_self_care", + } + registry: dict[tuple[str, str, str, int], str] = {} + for key in _surface_target_keys(surface): + _entity, family, column, _clone_index = key + if ( + family in {"model_required_boolean", "benefit_participation"} + or column in boolean_columns + ): + metric = "boolean_incidence" + elif ( + family == "model_required_discrete" + or column == "first_home_mortgage_origination_year" + ): + metric = "categorical_tvd" + else: + metric = "monetary_sign_separated" + registry[key] = metric + return MappingProxyType(registry) + + +CANONICAL_STACKED_DECLARED_SURFACE = _freeze_target_families( + declared_acs_transfer_target_families() +) +CANONICAL_STACKED_GAP_FILL_PLAN = _build_stacked_gap_fill_plan( + CANONICAL_STACKED_DECLARED_SURFACE +) +CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY = _canonical_metric_registry( + CANONICAL_STACKED_DECLARED_SURFACE +) +CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE = _BatterySupportProfile( + profile_id="us_stacked_origin_battery_support", + version=1, + min_effective_support=5, +) + +_CANONICAL_STACKED_DECLARED_SURFACE_ANCHOR = CANONICAL_STACKED_DECLARED_SURFACE +_CANONICAL_STACKED_GAP_FILL_PLAN_ANCHOR = CANONICAL_STACKED_GAP_FILL_PLAN +_CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY_ANCHOR = ( + CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY +) +_CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE_ANCHOR = ( + CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE +) + +# Active module-level authority references are intentionally separate from the +# immutable anchors. Rebinding any active reference is detected at evaluation, +# and the live content digest is receipted rather than trusting a stale hash. +_STACKED_DECLARED_SURFACE = CANONICAL_STACKED_DECLARED_SURFACE +_STACKED_GAP_FILL_PLAN = CANONICAL_STACKED_GAP_FILL_PLAN +_BATTERY_METRIC_REGISTRY = CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY +_BATTERY_SUPPORT_PROFILE = CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE + +_CANONICAL_STACKED_AUTHORITY = _make_stacked_authority( + authority_id=_STACKED_AUTHORITY_ID, + version=_STACKED_AUTHORITY_VERSION, + gap_fill_plan=_CANONICAL_STACKED_GAP_FILL_PLAN_ANCHOR, + declared_surface=_CANONICAL_STACKED_DECLARED_SURFACE_ANCHOR, + metric_registry=_CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY_ANCHOR, + support_profile=_CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE_ANCHOR, + declared_form=_CANONICAL_AUTHORITY_FORM, +) +_CANONICAL_STACKED_AUTHORITY_ANCHOR = _CANONICAL_STACKED_AUTHORITY + + +def _production_stacked_authority( + *, + _canonical_authority: _StackedAuthority = _CANONICAL_STACKED_AUTHORITY, + _canonical_plan: tuple[GapFillDirection, ...] = CANONICAL_STACKED_GAP_FILL_PLAN, + _canonical_surface: TargetFamilies = CANONICAL_STACKED_DECLARED_SURFACE, + _canonical_registry: Mapping[ + tuple[str, str, str, int], str + ] = CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY, + _canonical_profile: _BatterySupportProfile = ( + CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE + ), +) -> _StackedAuthority: + identity = ( + _STACKED_GAP_FILL_PLAN is _canonical_plan + and _STACKED_DECLARED_SURFACE is _canonical_surface + and _BATTERY_METRIC_REGISTRY is _canonical_registry + and _BATTERY_SUPPORT_PROFILE is _canonical_profile + ) + if identity: + return _canonical_authority + return _make_stacked_authority( + authority_id=_STACKED_AUTHORITY_ID, + version=_STACKED_AUTHORITY_VERSION, + gap_fill_plan=_STACKED_GAP_FILL_PLAN, + declared_surface=_STACKED_DECLARED_SURFACE, + metric_registry=_BATTERY_METRIC_REGISTRY, + support_profile=_BATTERY_SUPPORT_PROFILE, + declared_form=_CANONICAL_AUTHORITY_FORM, + declared_component_sha256=_canonical_authority.declared_component_sha256, + declared_sha256=_canonical_authority.declared_sha256, + ) + + +def _metric_registry_for_surface( + surface: TargetFamilies, +) -> Mapping[tuple[str, str, str, int], str]: + inferred = _canonical_metric_registry(surface) + return MappingProxyType( + { + key: CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY.get( + key, + inferred[key], + ) + for key in _surface_target_keys(surface) + } + ) + + +def _make_test_stacked_authority( + *, + declared_surface: TargetFamilies | None = None, + gap_fill_plan: Sequence[GapFillDirection] | None = None, + metric_registry: Mapping[tuple[str, str, str, int], str] | None = None, + support_profile: _BatterySupportProfile | None = None, +) -> _StackedAuthority: + """Explicit test-only seam; every receipt is marked non-canonical.""" + + surface = ( + CANONICAL_STACKED_DECLARED_SURFACE + if declared_surface is None + else declared_surface + ) + plan = CANONICAL_STACKED_GAP_FILL_PLAN if gap_fill_plan is None else gap_fill_plan + registry = ( + _metric_registry_for_surface(surface) + if metric_registry is None + else metric_registry + ) + return _make_stacked_authority( + authority_id=f"{_STACKED_AUTHORITY_ID}.test", + version=_STACKED_AUTHORITY_VERSION, + gap_fill_plan=plan, + declared_surface=surface, + metric_registry=registry, + support_profile=( + CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE + if support_profile is None + else support_profile + ), + declared_form=_NONCANONICAL_AUTHORITY_FORM, + ) + + +def stacked_gap_fill_plan() -> tuple[GapFillDirection, ...]: + """Return the immutable canonical two-direction stacked gap-fill plan.""" + + return _STACKED_GAP_FILL_PLAN + + +def _direction_target_index( + plan: Sequence[GapFillDirection], +) -> dict[tuple[str, str, str, int], GapFillDirection]: + index: dict[tuple[str, str, str, int], GapFillDirection] = {} + for direction in plan: + for entity, families in direction.target_families.items(): + for family, targets in families.items(): + for target in targets: + index[(entity, family, target, 0)] = direction + return index + + +def _direction_signature(direction: GapFillDirection) -> tuple[str, str, str]: + return ( + direction.name, + direction.recipient_channel, + direction.donor_channel, + ) + + +def _authority_receipt( + authority: _StackedAuthority, + *, + _canonical_authority: _StackedAuthority = _CANONICAL_STACKED_AUTHORITY, +) -> dict[str, object]: + """Receipt live content, claimed digests, identity, and component counts.""" + + live_components, live_bundle = _authority_live_digests(authority) + component_integrity = { + name: digest == authority.declared_component_sha256[name] + for name, digest in live_components.items() + } + integrity = all(component_integrity.values()) and ( + live_bundle == authority.declared_sha256 + ) + canonical_identity = authority is _canonical_authority + canonical_content = ( + authority.authority_id == _STACKED_AUTHORITY_ID + and authority.version == _STACKED_AUTHORITY_VERSION + and live_components == dict(_canonical_authority.declared_component_sha256) + and live_bundle == _canonical_authority.declared_sha256 + ) + canonical = ( + authority.declared_form == _CANONICAL_AUTHORITY_FORM + and canonical_identity + and canonical_content + and integrity + ) + support = _support_profile_payload(authority.support_profile) + components: dict[str, dict[str, object]] = { + "gap_fill_plan": { + "sha256": live_components["gap_fill_plan"], + "declared_sha256": authority.declared_component_sha256["gap_fill_plan"], + "target_count": len(_plan_target_keys(authority.gap_fill_plan)), + "direction_count": len(authority.gap_fill_plan), + "digest_matches_declared": component_integrity["gap_fill_plan"], + }, + "declared_surface": { + "sha256": live_components["declared_surface"], + "declared_sha256": authority.declared_component_sha256["declared_surface"], + "target_count": len(_surface_target_keys(authority.declared_surface)), + "entity_count": len(authority.declared_surface), + "digest_matches_declared": component_integrity["declared_surface"], + }, + "metric_registry": { + "sha256": live_components["metric_registry"], + "declared_sha256": authority.declared_component_sha256["metric_registry"], + "target_count": len(authority.metric_registry), + "digest_matches_declared": component_integrity["metric_registry"], + }, + "support_profile": { + **support, + "sha256": live_components["support_profile"], + "declared_sha256": authority.declared_component_sha256["support_profile"], + "digest_matches_declared": component_integrity["support_profile"], + }, + } + return { + "authority_id": authority.authority_id, + "version": authority.version, + "authority_form": ( + _CANONICAL_AUTHORITY_FORM if canonical else _NONCANONICAL_AUTHORITY_FORM + ), + "declared_authority_form": authority.declared_form, + "canonical": canonical, + "production_manifest_permitted": canonical, + "canonical_identity": canonical_identity, + "canonical_content": canonical_content, + "integrity_valid": integrity, + "sha256": live_bundle, + "declared_sha256": authority.declared_sha256, + "digest_matches_declared": live_bundle == authority.declared_sha256, + "components": components, + } + + +def _authority_validation_failures( + authority: _StackedAuthority, + *, + production: bool, + _canonical_plan: tuple[GapFillDirection, ...] = CANONICAL_STACKED_GAP_FILL_PLAN, + _canonical_registry: Mapping[ + tuple[str, str, str, int], str + ] = CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY, + _canonical_profile: _BatterySupportProfile = ( + CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE + ), +) -> list[str]: + receipt = _authority_receipt(authority) + failures: list[str] = [] + surface_targets = _surface_target_keys(authority.declared_surface) + plan_targets = _plan_target_keys(authority.gap_fill_plan) + duplicate_surface_targets = sorted( + target for target, count in Counter(surface_targets).items() if count > 1 + ) + duplicate_plan_targets = sorted( + target for target, count in Counter(plan_targets).items() if count > 1 + ) + if duplicate_surface_targets: + failures.append( + "declared surface repeats target(s): " + + ", ".join( + _battery_target_label(target) for target in duplicate_surface_targets + ) + + "." + ) + if duplicate_plan_targets: + failures.append( + "gap-fill plan repeats target(s): " + + ", ".join( + _battery_target_label(target) for target in duplicate_plan_targets + ) + + "." + ) + for name, label in ( + ("gap_fill_plan", "gap-fill plan"), + ("declared_surface", "declared surface"), + ("metric_registry", "metric registry"), + ("support_profile", "support profile"), + ): + component = receipt["components"][name] + if not component["digest_matches_declared"]: + failures.append( + f"{label} live-content digest mismatch: declared " + f"{component['declared_sha256']}, computed {component['sha256']}." + ) + if not receipt["digest_matches_declared"]: + failures.append( + "stacked authority live-content digest mismatch: declared " + f"{receipt['declared_sha256']}, computed {receipt['sha256']}." + ) + + canonical_directions = _direction_target_index(_canonical_plan) + for target, direction in _direction_target_index(authority.gap_fill_plan).items(): + canonical_direction = canonical_directions.get(target) + if canonical_direction is not None and _direction_signature( + direction + ) != _direction_signature(canonical_direction): + failures.append( + f"canonical gap-fill direction mismatch for " + f"{_battery_target_label(target)}: authoritative " + f"{_direction_signature(canonical_direction)!r}, got " + f"{_direction_signature(direction)!r}." + ) + for target, metric in authority.metric_registry.items(): + canonical_metric = _canonical_registry.get(target) + if canonical_metric is not None and metric != canonical_metric: + failures.append( + f"declared battery target {_battery_target_label(target)} must " + f"use authoritative metric {canonical_metric!r}, got {metric!r}." + ) + if authority.support_profile != _canonical_profile: + failures.append( + "support profile differs from the canonical stacked battery profile." + ) + if production and receipt["canonical_identity"] is not True: + failures.append("canonical stacked authority identity mismatch.") + if production and receipt["canonical_content"] is not True: + failures.append("canonical stacked authority live content mismatch.") + if production and not receipt["canonical"]: + failures.append( + "non-canonical stacked authority is forbidden in production manifests." + ) + return failures + + +def _validate_production_authority_receipt( + receipt: Mapping[str, object], + *, + boundary: str, + _canonical_authority: _StackedAuthority = _CANONICAL_STACKED_AUTHORITY, +) -> None: + """Terminally reject any non-canonical authority at artifact emission.""" + + expected = _authority_receipt(_canonical_authority) + if dict(receipt) != expected: + raise ValueError( + f"{boundary}: non-canonical stacked authority is forbidden; " + "production manifest emission is forbidden." + ) + + +def _validate_test_authority(authority: _StackedAuthority, *, boundary: str) -> None: + """Keep the explicit fixture seam visibly and terminally non-production.""" + + receipt = _authority_receipt(authority) + if ( + authority.declared_form != _NONCANONICAL_AUTHORITY_FORM + or receipt["authority_form"] != _NONCANONICAL_AUTHORITY_FORM + or receipt["canonical_identity"] is not False + ): + raise ValueError(f"{boundary} requires a NON-CANONICAL test authority.") + + +def _validate_stacked_gate_manifest_details( + gate_name: str, + details: Mapping[str, object], + *, + _canonical_surface: TargetFamilies = CANONICAL_STACKED_DECLARED_SURFACE, + _canonical_plan: tuple[GapFillDirection, ...] = CANONICAL_STACKED_GAP_FILL_PLAN, + _canonical_registry: Mapping[ + tuple[str, str, str, int], str + ] = CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY, +) -> None: + """Validate gate-specific receipts against the captured canonical doctrine.""" + + boundary = f"Gate {gate_name!r} manifest emission" + authority = details.get("authority") + if not isinstance(authority, Mapping): + raise ValueError( + f"{boundary}: no stacked authority receipt; production manifest " + "emission is forbidden." + ) + _validate_production_authority_receipt(authority, boundary=boundary) + authority_sha256 = authority["sha256"] + plan_sha256 = authority["components"]["gap_fill_plan"]["sha256"] + surface_sha256 = authority["components"]["declared_surface"]["sha256"] + expected_keys = _surface_target_keys(_canonical_surface) + + def reject(reason: str) -> None: + raise ValueError( + f"{boundary}: {reason}; production manifest emission is forbidden." + ) + + if gate_name == _COMPLETENESS_GATE_NAME: + expected_labels = { + f"{entity}/{family}/{column}" + for entity, family, column, _clone_index in expected_keys + } + direction_by_label = { + f"{entity}/{family}/{column}": direction + for direction in _canonical_plan + for entity, families in direction.target_families.items() + for family, columns in families.items() + for column in columns + } + targets = details.get("targets") + if details.get("declared_targets") != 90: + reject("canonical completeness receipt must declare exactly 90 targets") + if not isinstance(targets, Mapping) or set(targets) != expected_labels: + reject("canonical completeness receipt target surface mismatch") + allowed_target_forms = { + "observed_complete", + "missing_declared_entity", + "missing_declared_target", + "origin_exact_recipient", + "mixed_proven_absence", + "unproven", + } + for label, target_receipt in targets.items(): + if not isinstance(target_receipt, Mapping): + reject(f"{label} target receipt is not a mapping") + if ( + target_receipt.get("authority_sha256") != authority_sha256 + or target_receipt.get("plan_sha256") != plan_sha256 + or target_receipt.get("surface_sha256") != surface_sha256 + ): + reject(f"{label} target receipt is not bound to canonical authority") + authority_form = target_receipt.get("authority_form") + if authority_form not in allowed_target_forms: + reject(f"{label} declares invalid authority form {authority_form!r}") + proven = target_receipt.get("proven", {}) + if not isinstance(proven, Mapping): + reject(f"{label} proven-absence receipts are not a mapping") + for cell, proof_receipt in proven.items(): + if not isinstance(proof_receipt, Mapping): + reject(f"{label} {cell} proof receipt is not a mapping") + direction = direction_by_label[label] + cell_channel, separator, _clone_role = str(cell).partition("/clone_") + if ( + not separator + or cell_channel != direction.recipient_channel + or proof_receipt.get("authority_form") != "origin_exact_recipient" + or proof_receipt.get("authority_sha256") != authority_sha256 + or proof_receipt.get("plan_sha256") != plan_sha256 + or proof_receipt.get("surface_sha256") != surface_sha256 + or proof_receipt.get("declared_direction") != direction.name + or proof_receipt.get("declared_donor_channel") + != direction.donor_channel + or proof_receipt.get("declared_recipient_channel") + != direction.recipient_channel + ): + reject( + f"{label} {cell} proof is not recipient-exact canonical authority" + ) + return + + if gate_name == _BATTERY_GATE_NAME: + expected_labels = {_battery_target_label(target) for target in expected_keys} + expected_plan = { + "plan_id": "stacked_gap_fill_plan", + "version": authority["version"], + "sha256": plan_sha256, + } + comparisons = details.get("comparisons") + if ( + details.get("declared_target_count") != 90 + or details.get("registered_target_count") != 90 + or details.get("missing_declared_targets") != [] + or details.get("extra_registered_targets") != [] + ): + reject("canonical battery coverage receipt must bind all 90 targets") + if details.get("declared_plan") != expected_plan: + reject("canonical battery plan receipt mismatch") + if details.get("support_profile") != authority["components"]["support_profile"]: + reject("canonical battery support-profile receipt mismatch") + if not isinstance(comparisons, Mapping) or set(comparisons) != expected_labels: + reject("canonical battery comparison surface mismatch") + for target, metric in _canonical_registry.items(): + label = _battery_target_label(target) + comparison = comparisons[label] + if ( + not isinstance(comparison, Mapping) + or comparison.get("metric") != metric + ): + reject(f"{label} comparison must use canonical metric {metric!r}") + return + + reject("unknown stacked authority gate") + + +_canonical_surface_keys = _surface_target_keys( + _CANONICAL_STACKED_DECLARED_SURFACE_ANCHOR +) +if ( + len(_canonical_surface_keys) != 90 + or len(set(_canonical_surface_keys)) != 90 + or set(_plan_target_keys(_CANONICAL_STACKED_GAP_FILL_PLAN_ANCHOR)) + != set(_canonical_surface_keys) + or set(_CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY_ANCHOR) + != set(_canonical_surface_keys) +): + raise RuntimeError( + "Canonical stacked authority must bind one plan, surface, and metric " + "for exactly 90 unique targets." + ) + + +def gap_fill_stacked_spine( + frame: Frame, + *, + seed: int = 0, + n_estimators: int = 100, + max_targets_per_fit: int = DEFAULT_ACS_TRANSFER_MAX_TARGETS_PER_FIT, + target_banks: Mapping[str, AcsTransferTargetBank] | None = None, +) -> GapFillResult: + """Run the canonical stacked gap-fill plan with no caller authority.""" + + return _gap_fill_stacked_spine_evaluate( + frame, + authority=_production_stacked_authority(), + production=True, + seed=seed, + n_estimators=n_estimators, + max_targets_per_fit=max_targets_per_fit, + target_banks=target_banks, + ) + + +def _gap_fill_stacked_spine_with_test_authority( + frame: Frame, + *, + authority: _StackedAuthority, + seed: int = 0, + n_estimators: int = 100, + max_targets_per_fit: int = DEFAULT_ACS_TRANSFER_MAX_TARGETS_PER_FIT, + target_banks: Mapping[str, AcsTransferTargetBank] | None = None, +) -> GapFillResult: + """Explicit non-production seam for fixture-sized authority surfaces.""" + + _validate_test_authority(authority, boundary="stacked gap-fill test seam") + return _gap_fill_stacked_spine_evaluate( + frame, + authority=authority, + production=False, + seed=seed, + n_estimators=n_estimators, + max_targets_per_fit=max_targets_per_fit, + target_banks=target_banks, + ) + + +def _gap_fill_stacked_spine_evaluate( + frame: Frame, + *, + authority: _StackedAuthority, + production: bool, + seed: int, + n_estimators: int, + max_targets_per_fit: int, + target_banks: Mapping[str, AcsTransferTargetBank] | None, +) -> GapFillResult: + """Gap-fill survey-specific fields cross-origin on the stacked spine. + + Runs before any clone operator: every row still carries clone index zero, + so filled values are cloned into the PUF arm afterwards and the single + PUF pass conditions on observed predictors for every origin. + + Per direction, in order: + + 1. **Activation authority** — declared, then verified: every target + column must exist, the donor origin's rows must observe it completely, + and every null cell must lie on the declared recipient origin. A + null anywhere else is a named terminal failure, so absence can never + silently reroute or skip a family (populace#578 audit item 2). + 2. **Authoritative donors** — the donor frame passed to the spine-blind + transfer is this owner's projection of the donor origin's native rows + (audit item 3); ``donor_channel=None`` marks the deliberate + whole-donor fit of that projection. + 3. **Banked transfer** — the reviewed #608 target-at-a-time banking + machinery is reused unchanged via ``target_banks[direction.name]``. + 4. **Post-verification** — donor-origin cells must be byte-identical + before and after, and no null may remain on authorized rows beyond + the transfer's receipted unmodeled rows. + + Returns a :class:`GapFillResult` whose receipt records, per direction and + target, the authorized-null, imputed, unmodeled, and residual-null + counts alongside the transfer's fit provenance. + """ + + authority_receipt = _authority_receipt(authority) + authority_failures = _authority_validation_failures( + authority, + production=production, + ) + if authority_failures: + raise ValueError( + "Stacked gap-fill authority validation failed:\n " + + "\n ".join(authority_failures) + ) + if production: + _validate_production_authority_receipt( + authority_receipt, + boundary="stacked gap-fill entry", + ) + validate_stacked_spine_frame(frame, boundary="stacked gap-fill entry") + directions = authority.gap_fill_plan + if not directions: + raise ValueError("Stacked gap-fill requires at least one direction.") + names = [direction.name for direction in directions] + if len(set(names)) != len(names): + raise ValueError(f"Stacked gap-fill directions repeat names: {names}.") + if target_banks is not None: + unknown_banks = sorted(set(target_banks) - set(names)) + if unknown_banks: + raise ValueError( + f"target_banks name unknown gap-fill direction(s): {unknown_banks}." + ) + + person_clone = frame.table("person")[support_clone_index_column("person")] + if not person_clone.eq(0).all(): + raise ValueError( + "Stacked gap-fill must run before clone operators; found nonzero " + "person support clone indices." + ) + + current = frame + receipts: dict[str, object] = {} + transfer_results: dict[str, AcsTransferResult] = {} + for direction in directions: + pre_counts = _verify_gap_fill_activation_authority( + current, + direction=direction, + ) + donor = _origin_projection(current, channel=direction.donor_channel) + donor_snapshot = { + entity: _direction_targets_snapshot( + current, + entity=entity, + targets=targets, + channel=direction.donor_channel, + ) + for entity, targets in _direction_entity_targets(direction).items() + } + result = transfer_acs_inputs( + current, + donor, + target_families=direction.target_families, + donor_channel=None, + seed=seed, + n_estimators=n_estimators, + max_targets_per_fit=max_targets_per_fit, + target_bank=(target_banks or {}).get(direction.name), + ) + transfer_results[direction.name] = result + current = result.frame + receipts[direction.name] = _verify_gap_fill_outcome( + current, + direction=direction, + pre_counts=pre_counts, + donor_snapshot=donor_snapshot, + result=result, + ) + + validate_stacked_spine_frame(current, boundary="stacked gap-fill output") + return GapFillResult( + frame=current, + receipt={"authority": authority_receipt, "directions": receipts}, + transfer_results=transfer_results, + ) + + +def _direction_entity_targets( + direction: GapFillDirection, +) -> dict[str, tuple[str, ...]]: + result: dict[str, tuple[str, ...]] = {} + for entity, families in direction.target_families.items(): + collected: list[str] = [] + for targets in families.values(): + collected.extend(targets) + result[entity] = tuple(collected) + return result + + +def _origin_projection(frame: Frame, *, channel: str) -> Frame: + """Project one origin's native lineages as a standalone donor frame.""" + + person = frame.table("person") + mask = ( + person[support_channel_column("person")].astype(str).eq(channel) + & person[support_clone_index_column("person")].eq(0) + ).to_numpy() + if not mask.any(): + raise ValueError( + f"Stacked spine has no native person rows for origin {channel!r}." + ) + return frame.select(mask) + + +def _direction_targets_snapshot( + frame: Frame, + *, + entity: str, + targets: Sequence[str], + channel: str, +) -> pd.DataFrame: + table = frame.table(entity) + mask = table[support_channel_column(entity)].astype(str).eq(channel) + present = [target for target in targets if target in table.columns] + return table.loc[mask, present].copy(deep=True) + + +def _canonical_donor_series_payload( + series: pd.Series, + *, + boundary: str, +) -> tuple[object, ...]: + """Return a canonical, byte-aware identity payload for one donor target.""" + + column = "__stacked_gap_fill_donor_target__" + canonical_table = canonicalize_table_string_dtypes( + series.to_frame(name=column), + boundary=boundary, + table_name=f"donor_target[{series.name!r}]", + ) + canonical = canonical_table[column] + canonical.name = series.name + values = canonical.to_numpy(copy=False) + if ( + not pd.api.types.is_extension_array_dtype(canonical.dtype) + and not values.dtype.hasobject + ): + encoding = "raw_numpy_c_order" + value_payload = np.ascontiguousarray(values).tobytes(order="C") + else: + encoding = "independent_scalar_pickle_protocol_5" + semantic_values = canonical.to_numpy( + dtype=object, + copy=True, + ).tolist() + value_payload = _semantic_scalar_sequence_payload( + semantic_values, + boundary=f"{boundary} values", + ) + return ( + canonical.shape, + _index_identity_payload( + canonical.index, + boundary=f"{boundary} index", + ), + _semantic_scalar_payload( + canonical.name, + boundary=f"{boundary} series name", + ), + ( + _qualified_type_name(canonical.dtype), + pickle.dumps(canonical.dtype, protocol=5), + ), + encoding, + value_payload, + ) + + +def _qualified_type_name(value: object) -> str: + value_type = type(value) + return f"{value_type.__module__}.{value_type.__qualname__}" + + +def _semantic_scalar_payload( + value: object, + *, + boundary: str, +) -> tuple[str, bytes]: + """Serialize one supported scalar without a cross-value pickle memo.""" + + supported = ( + value is None + or value is pd.NA + or value is pd.NaT + or isinstance( + value, + ( + str, + bytes, + bool, + int, + float, + complex, + np.generic, + pd.Timestamp, + pd.Timedelta, + pd.Period, + pd.Interval, + ), + ) + ) + if not supported: + raise TypeError( + f"{boundary}: donor byte identity found unsupported semantic " + f"scalar type {_qualified_type_name(value)!r}." + ) + return ( + _qualified_type_name(value), + pickle.dumps(value, protocol=5), + ) + + +def _semantic_scalar_sequence_payload( + values: Sequence[object], + *, + boundary: str, +) -> tuple[tuple[str, bytes], ...]: + payloads: list[tuple[str, bytes]] = [] + for position, value in enumerate(values): + payload = _semantic_scalar_payload( + value, + boundary=f"{boundary} position {position}", + ) + payloads.append(payload) + return tuple(payloads) + + +def _index_identity_payload( + index: pd.Index, + *, + boundary: str, +) -> tuple[object, ...]: + """Return exact index authority without list-wide object serialization.""" + + names = tuple( + _semantic_scalar_payload( + name, + boundary=f"{boundary} name {position}", + ) + for position, name in enumerate(index.names) + ) + index_type = _qualified_type_name(index) + if isinstance(index, pd.MultiIndex): + levels = tuple( + _index_identity_payload( + level, + boundary=f"{boundary} level {position}", + ) + for position, level in enumerate(index.levels) + ) + codes = tuple( + ( + code.dtype.str, + code.shape, + np.ascontiguousarray(code).tobytes(order="C"), + ) + for code in index.codes + ) + return (index_type, names, "multiindex_levels_codes", levels, codes) + + dtype = index.dtype + dtype_authority = ( + _qualified_type_name(dtype), + pickle.dumps(dtype, protocol=5), + ) + values = index.to_numpy(copy=False) + if not pd.api.types.is_extension_array_dtype(dtype) and not values.dtype.hasobject: + encoding = "raw_numpy_c_order" + value_payload: object = ( + values.shape, + np.ascontiguousarray(values).tobytes(order="C"), + ) + else: + encoding = "independent_scalar_pickle_protocol_5" + semantic_values = index.to_numpy(dtype=object, copy=True).tolist() + value_payload = _semantic_scalar_sequence_payload( + semantic_values, + boundary=f"{boundary} values", + ) + return (index_type, names, dtype_authority, encoding, value_payload) + + +def _verify_gap_fill_activation_authority( + frame: Frame, + *, + direction: GapFillDirection, +) -> dict[tuple[str, str], dict[str, int]]: + """Verify declared activation authority before any modeling runs.""" + + failures: list[str] = [] + counts: dict[tuple[str, str], dict[str, int]] = {} + for entity, families in direction.target_families.items(): + table = frame.table(entity) + channel = table[support_channel_column(entity)].astype(str) + recipient_rows = channel.eq(direction.recipient_channel) + donor_rows = channel.eq(direction.donor_channel) + recipient_count = int(recipient_rows.sum()) + donor_count = int(donor_rows.sum()) + if recipient_count == 0: + failures.append( + f"{direction.name}/{entity}: declared recipient channel " + f"{direction.recipient_channel!r} has no live rows." + ) + if donor_count == 0: + failures.append( + f"{direction.name}/{entity}: declared donor channel " + f"{direction.donor_channel!r} has no live rows." + ) + for family, targets in families.items(): + for target in targets: + label = f"{direction.name}/{entity}/{family}/{target}" + if target not in table.columns: + failures.append( + f"{label}: declared gap-fill target column is absent " + "from the stacked spine." + ) + continue + null_mask = table[target].isna() + donor_nulls = int((null_mask & donor_rows).sum()) + unauthorized_nulls = int( + (null_mask & ~recipient_rows & ~donor_rows).sum() + ) + if donor_nulls: + failures.append( + f"{label}: donor origin {direction.donor_channel!r} " + f"has {donor_nulls} null cell(s); donors must observe " + "every declared target." + ) + if unauthorized_nulls: + failures.append( + f"{label}: {unauthorized_nulls} null cell(s) lie " + "outside the declared recipient origin " + f"{direction.recipient_channel!r}." + ) + counts[(entity, target)] = { + "authorized_null_rows": int((null_mask & recipient_rows).sum()), + "recipient_rows": int(recipient_rows.sum()), + "donor_rows": int(donor_rows.sum()), + } + if failures: + raise ValueError( + "Stacked gap-fill activation authority failed:\n " + "\n ".join(failures) + ) + return counts + + +def _verify_gap_fill_outcome( + frame: Frame, + *, + direction: GapFillDirection, + pre_counts: Mapping[tuple[str, str], Mapping[str, int]], + donor_snapshot: Mapping[str, pd.DataFrame], + result: AcsTransferResult, +) -> dict[str, object]: + """Verify donor invariance and residual nulls; build the direction receipt.""" + + failures: list[str] = [] + imputed_by_target = { + (record.entity, record.column): record for record in result.imputed_inputs + } + target_receipts: dict[str, dict[str, object]] = {} + for entity, families in direction.target_families.items(): + table = frame.table(entity) + channel = table[support_channel_column(entity)].astype(str) + recipient_rows = channel.eq(direction.recipient_channel) + donor_after = { + entity_name: _direction_targets_snapshot( + frame, + entity=entity_name, + targets=targets, + channel=direction.donor_channel, + ) + for entity_name, targets in _direction_entity_targets(direction).items() + }[entity] + for family, targets in families.items(): + for target in targets: + label = f"{direction.name}/{entity}/{family}/{target}" + before = donor_snapshot[entity].get(target) + after = donor_after.get(target) + if ( + before is None + or after is None + or _canonical_donor_series_payload( + before, + boundary=f"{label} donor identity before transfer", + ) + != _canonical_donor_series_payload( + after, + boundary=f"{label} donor identity after transfer", + ) + ): + failures.append( + f"{label}: donor byte identity failed for origin " + f"{direction.donor_channel!r}; canonical donor payload " + "changed during gap-fill transfer." + ) + null_mask = table[target].isna() + residual_nulls = int((null_mask & recipient_rows).sum()) + outside_nulls = int((null_mask & ~recipient_rows).sum()) + if outside_nulls: + failures.append( + f"{label}: {outside_nulls} null cell(s) appeared " + "outside the declared recipient origin during the " + "transfer." + ) + record = imputed_by_target.get((entity, target)) + imputed = record.imputed_recipient_rows if record else 0 + unmodeled = record.unmodeled_recipient_rows if record else 0 + pre = pre_counts[(entity, target)] + authorized = pre["authorized_null_rows"] + if residual_nulls != unmodeled: + failures.append( + f"{label}: residual-null equation failed: " + f"residual_null_rows={residual_nulls} != " + f"unmodeled_rows={unmodeled}." + ) + if authorized != imputed + unmodeled: + failures.append( + f"{label}: activation accounting equation failed: " + f"authorized_null_rows={authorized} != " + f"imputed_rows={imputed} + unmodeled_rows={unmodeled}." + ) + target_receipts[f"{entity}/{family}/{target}"] = { + "authorized_null_rows": authorized, + "imputed_rows": imputed, + "unmodeled_rows": unmodeled, + "residual_null_rows": residual_nulls, + } + if failures: + raise ValueError( + "Stacked gap-fill outcome verification failed:\n " + "\n ".join(failures) + ) + return { + "recipient_channel": direction.recipient_channel, + "donor_channel": direction.donor_channel, + "donor_selection": "owner_projection_of_native_donor_rows", + "resolved_donor_channel": result.resolved_donor_channel, + "targets": target_receipts, + "deferred_inputs": list(result.deferred_inputs), + "fit_records": [ + {"fit_name": record.fit_name, "weight_kind": record.weight_kind} + for record in result.fit_records + ], + } + + +# --------------------------------------------------------------------------- +# The single PUF pass over the stacked spine (charter item 3) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class StackedPufPassResult: + """The post-PUF stacked frame plus attachment and fit receipts.""" + + frame: Frame + receipt: Mapping[str, object] + + +def run_stacked_puf_pass( + frame: Frame, + donor_tax_units: pd.DataFrame, + *, + clone_attachment_fraction: float, + clone_attachment_seed: int, + predictors: Sequence[str] | None = None, + person_outputs: Sequence[str] | None = None, + tax_unit_outputs: Sequence[str] | None = None, + seed: int = 0, + n_estimators: int = 100, + fit_records: list[FitWeightRecord] | None = None, + tail_bound_diagnostics: list[dict[str, object]] | None = None, +) -> StackedPufPassResult: + """Run the one PUF pass over the gap-filled stacked spine. + + Order is the charter's: the spine must already be gap-filled (this entry + validates the stacked manifest and refuses cloned input), the PUF clone + arm attaches to a seeded whole-household sample of stacked households + (both origins; reusing the reviewed clone-routing discipline), and the + primary QRF then runs under both stacked doctrines — recipient predictors + must be complete (no zero-filled absence) and finalization preserves + nulls on every cell the pass does not own. + """ + + validate_stacked_spine_frame(frame, boundary="stacked PUF pass entry") + person_clone = frame.table("person")[support_clone_index_column("person")] + if not person_clone.eq(0).all(): + raise ValueError( + "The stacked PUF pass owns clone attachment; found nonzero person " + "support clone indices on its input." + ) + cloned = clone_us_frame_for_puf_support( + frame, + clone_attachment_fraction=clone_attachment_fraction, + clone_attachment_seed=clone_attachment_seed, + ) + attachment = validate_puf_clone_attachment( + cloned, + boundary="stacked PUF pass clone attachment", + expected_fraction=clone_attachment_fraction, + expected_seed=clone_attachment_seed, + ) + + kwargs: dict[str, object] = {} + if predictors is not None: + kwargs["predictors"] = tuple(predictors) + if person_outputs is not None: + kwargs["person_outputs"] = tuple(person_outputs) + if tax_unit_outputs is not None: + kwargs["tax_unit_outputs"] = tuple(tax_unit_outputs) + imputed = impute_us_puf_tax_detail_support( + cloned, + donor_tax_units, + seed=seed, + n_estimators=n_estimators, + fit_records=fit_records, + tail_bound_diagnostics=tail_bound_diagnostics, + require_complete_recipient_predictors=True, + absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + **kwargs, + ) + validate_stacked_spine_frame(imputed, boundary="stacked PUF pass output") + validate_puf_clone_attachment( + imputed, + boundary="stacked PUF pass output", + expected_fraction=clone_attachment_fraction, + expected_seed=clone_attachment_seed, + ) + + person = imputed.table("person") + channel = person[support_channel_column("person")].astype(str) + clone_index = person[support_clone_index_column("person")] + recipients_by_origin = { + origin: int((channel.eq(origin) & clone_index.eq(1)).sum()) + for origin in sorted(channel.unique()) + } + return StackedPufPassResult( + frame=imputed, + receipt={ + "clone_attachment": _json_ready(attachment), + "doctrines": { + "require_complete_recipient_predictors": True, + "absent_cells": PUF_ABSENT_CELLS_PRESERVE_NULLS, + }, + "recipient_person_rows_by_origin": recipients_by_origin, + }, + ) + + +# --------------------------------------------------------------------------- +# Pre-simulation completeness gate (charter item 4) +# --------------------------------------------------------------------------- + +_COMPLETENESS_GATE_NAME = "us_stacked_completeness" +_ANY_CHANNEL = "*" + + +@dataclass(frozen=True) +class AbsenceProof: + """An explicit source-by-role authority proof for permitted null cells. + + A declared target's cells may be null only where a proof names the exact + origin channel and clone role, with the reason recorded. ``"*"`` may + cover every origin only when no gap-fill direction declares a donor for + that target. This is the audit's item-5 contract: a target is banked or + imputed, or its absence carries an explicit source-by-role proof — silence + is never authority. + """ + + entity: str + column: str + channel: str + clone_index: int + reason: str + + def __post_init__(self) -> None: + for label, value in ( + ("entity", self.entity), + ("column", self.column), + ("channel", self.channel), + ("reason", self.reason), + ): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"AbsenceProof.{label} must be a non-empty string.") + if ( + isinstance(self.clone_index, bool) + or not isinstance(self.clone_index, int) + or self.clone_index < 0 + ): + raise ValueError("AbsenceProof.clone_index must be a non-negative integer.") + + +def stacked_completeness_gate( + frame: Frame, + *, + absence_proofs: Sequence[AbsenceProof] = (), +) -> GateResult: + """Evaluate the canonical declared surface with no caller authority.""" + + return _stacked_completeness_gate_evaluate( + frame, + authority=_production_stacked_authority(), + production=True, + absence_proofs=absence_proofs, + ) + + +def _stacked_completeness_gate_with_test_authority( + frame: Frame, + *, + authority: _StackedAuthority, + absence_proofs: Sequence[AbsenceProof] = (), +) -> GateResult: + """Explicit test-only completeness seam for a digested authority bundle.""" + + _validate_test_authority(authority, boundary="stacked completeness test seam") + return _stacked_completeness_gate_evaluate( + frame, + authority=authority, + production=False, + absence_proofs=absence_proofs, + ) + + +def _stacked_completeness_gate_evaluate( + frame: Frame, + *, + authority: _StackedAuthority, + production: bool, + absence_proofs: Sequence[AbsenceProof], + _canonical_gap_fill_plan: tuple[ + GapFillDirection, ... + ] = CANONICAL_STACKED_GAP_FILL_PLAN, +) -> GateResult: + """Prove every declared target is filled or carries absence authority. + + For every declared ``entity/family/target``: a missing column is a named + terminal failure (a whole registered family can never silently vanish + again — this is the check that would have caught run 7's 58-target skip); + a null cell is permitted only where an :class:`AbsenceProof` names its + origin channel and clone role, and every unproven null fails by name with + its per-origin, per-role counts. The canonical bundle's live gap-fill plan + is required authority: its donor direction disables wildcard proofs for + that target even at the explicitly non-canonical fixture seam. + """ + + authority_receipt = _authority_receipt(authority) + declared_surface = authority.declared_surface + declared_count = len(_surface_target_keys(declared_surface)) + failures = _authority_validation_failures(authority, production=production) + if declared_count == 0: + failures.append("declared stacked surface contains zero targets.") + if failures: + return GateResult( + name=_COMPLETENESS_GATE_NAME, + passed=False, + failures=tuple(failures), + details={ + "authority": authority_receipt, + "declared_targets": declared_count, + "targets": {}, + }, + ) + + declared_directions: dict[tuple[str, str], GapFillDirection] = {} + for direction in authority.gap_fill_plan: + for entity, targets in _direction_entity_targets(direction).items(): + for target in targets: + key = (entity, target) + previous = declared_directions.get(key) + if previous is not None and previous != direction: + raise ValueError( + "stacked authority plan assigns conflicting directions " + f"to {entity}/{target}: {previous.name!r} and " + f"{direction.name!r}." + ) + declared_directions[key] = direction + canonical_direction_keys: set[tuple[str, str]] = set() + for direction in _canonical_gap_fill_plan: + for entity, targets in _direction_entity_targets(direction).items(): + for target in targets: + key = (entity, target) + canonical_direction_keys.add(key) + declared_directions[key] = direction + + proof_index: dict[tuple[str, str], dict[tuple[str, int], str]] = {} + for proof in absence_proofs: + if not isinstance(proof, AbsenceProof): + raise ValueError( + "absence_proofs must contain AbsenceProof values, got " + f"{type(proof).__name__}." + ) + proof_index.setdefault((proof.entity, proof.column), {})[ + (proof.channel, proof.clone_index) + ] = proof.reason + + target_receipts: dict[str, dict[str, object]] = {} + authority_sha256 = authority_receipt["sha256"] + plan_sha256 = authority_receipt["components"]["gap_fill_plan"]["sha256"] + surface_sha256 = authority_receipt["components"]["declared_surface"]["sha256"] + + def authority_binding(authority_form: str) -> dict[str, object]: + return { + "authority_form": authority_form, + "authority_sha256": authority_sha256, + "plan_sha256": plan_sha256, + "surface_sha256": surface_sha256, + } + + for entity, families in declared_surface.items(): + if entity not in frame.entities: + failures.append( + f"{entity}: declared entity is absent from the stacked frame." + ) + for family, targets in families.items(): + for target in targets: + target_receipts[f"{entity}/{family}/{target}"] = { + "status": "missing_entity", + "null_rows": None, + **authority_binding("missing_declared_entity"), + } + continue + table = frame.table(entity) + channel = table[support_channel_column(entity)].astype(str) + clone_index = pd.to_numeric( + table[support_clone_index_column(entity)], + errors="raise", + ).astype("int64") + for family, targets in families.items(): + for target in targets: + label = f"{entity}/{family}/{target}" + if target not in table.columns: + failures.append( + f"{label}: declared target column is missing from the " + "pre-simulation pool; the registered family must " + "never silently vanish from the active bank." + ) + target_receipts[label] = { + "status": "missing", + "null_rows": None, + **authority_binding("missing_declared_target"), + } + continue + null_mask = table[target].isna() + if not null_mask.any(): + target_receipts[label] = { + "status": "complete", + "null_rows": 0, + **authority_binding("observed_complete"), + } + continue + proofs = proof_index.get((entity, target), {}) + declared_direction = declared_directions.get((entity, target)) + null_channels = channel.loc[null_mask] + null_clones = clone_index.loc[null_mask] + unproven: dict[str, int] = {} + proven: dict[str, dict[str, object]] = {} + grouped = ( + pd.DataFrame({"channel": null_channels, "clone_index": null_clones}) + .groupby(["channel", "clone_index"], sort=True) + .size() + ) + target_authority_forms: set[str] = set() + for (cell_channel, cell_clone), count in grouped.items(): + cell_channel = str(cell_channel) + cell_clone = int(cell_clone) + reason = proofs.get((cell_channel, cell_clone)) + authority_form = "origin_exact" + if declared_direction is not None and reason is not None: + if cell_channel != declared_direction.recipient_channel: + failures.append( + f"{label}: origin-exact authority proof is valid " + "only for declared recipient " + f"{declared_direction.recipient_channel!r}; " + f"{cell_channel!r} is declared donor " + f"{declared_direction.donor_channel!r}." + ) + reason = None + else: + authority_form = "origin_exact_recipient" + if reason is None and declared_direction is None: + reason = proofs.get((_ANY_CHANNEL, cell_clone)) + authority_form = "wildcard_no_declared_donor_plan" + key = f"{cell_channel}/clone_{cell_clone}" + if reason is None: + unproven[key] = int(count) + if declared_direction is not None: + canonical = (entity, target) in canonical_direction_keys + failures.append( + f"{label}: {int(count)} null cell(s) on {key} " + "require an origin-exact authority proof because " + f"{'canonical ' if canonical else ''}gap-fill plan " + f"{declared_direction.name!r} names donor " + f"{declared_direction.donor_channel!r} and recipient " + f"{declared_direction.recipient_channel!r}; wildcard " + "authority is forbidden." + ) + else: + proof_receipt: dict[str, object] = { + "null_rows": int(count), + "reason": reason, + **authority_binding(authority_form), + } + if declared_direction is not None: + proof_receipt.update( + { + "declared_direction": declared_direction.name, + "declared_donor_channel": ( + declared_direction.donor_channel + ), + "declared_recipient_channel": ( + declared_direction.recipient_channel + ), + } + ) + proven[key] = proof_receipt + target_authority_forms.add(authority_form) + if unproven: + failures.append( + f"{label}: {sum(unproven.values())} null cell(s) have " + "no source-by-role authority proof " + f"(by origin/role: {unproven})." + ) + if unproven: + target_authority_form = "unproven" + elif len(target_authority_forms) == 1: + target_authority_form = next(iter(target_authority_forms)) + else: + target_authority_form = "mixed_proven_absence" + target_receipts[label] = { + "status": "proven_absent" if not unproven else "unproven", + "null_rows": int(null_mask.sum()), + "proven": proven, + "unproven": unproven, + **authority_binding(target_authority_form), + } + return GateResult( + name=_COMPLETENESS_GATE_NAME, + passed=not failures, + failures=tuple(failures), + details={ + "authority": authority_receipt, + "declared_targets": declared_count, + "targets": target_receipts, + }, + ) + + +# --------------------------------------------------------------------------- +# By-origin battery (charter item 5) +# --------------------------------------------------------------------------- + +_BATTERY_GATE_NAME = "us_by_origin_battery" +_BATTERY_INCIDENCE_RATIO_BOUNDS = (0.8, 1.25) +_BATTERY_QUANTILES = (0.10, 0.25, 0.50, 0.75, 0.90) +_BATTERY_QUANTILE_ENVELOPE_TOLERANCE = 0.25 +_BATTERY_CATEGORICAL_TVD_TOLERANCE = 0.25 + + +@dataclass(frozen=True) +class OriginBatterySpec: + """Test-seam grouping for per-column battery metrics. + + Production never accepts these specs from a caller: it consumes the + immutable 90-column canonical registry. The explicit test-authority seam + groups its digested registry into specs so the comparison engine can reuse + the same loop. ``clone_index`` scopes a fixture comparison to one clone + role: 0 compares native rows and 1 compares a PUF arm. + """ + + entity: str + family: str + column_metrics: Mapping[str, str] + clone_index: int = 0 + + def __post_init__(self) -> None: + if not isinstance(self.entity, str) or not self.entity.strip(): + raise ValueError("OriginBatterySpec.entity must be a non-empty string.") + if not isinstance(self.family, str) or not self.family.strip(): + raise ValueError("OriginBatterySpec.family must be a non-empty string.") + if not isinstance(self.column_metrics, Mapping) or not self.column_metrics: + raise ValueError( + f"OriginBatterySpec {self.entity}/{self.family} declares no " + "column metrics." + ) + unknown = sorted( + { + metric + for metric in self.column_metrics.values() + if metric not in ORIGIN_BATTERY_METRIC_KINDS + } + ) + if unknown: + raise ValueError( + f"OriginBatterySpec {self.entity}/{self.family} declares " + f"unknown metric kind(s) {unknown}; expected one of " + f"{list(ORIGIN_BATTERY_METRIC_KINDS)}." + ) + object.__setattr__( + self, + "column_metrics", + MappingProxyType(dict(self.column_metrics)), + ) + if ( + isinstance(self.clone_index, bool) + or not isinstance(self.clone_index, int) + or self.clone_index < 0 + ): + raise ValueError( + "OriginBatterySpec.clone_index must be a non-negative integer." + ) + + +def by_origin_battery( + frame: Frame, +) -> GateResult: + """Run the canonical 90-target by-origin battery.""" + + return _by_origin_battery_evaluate( + frame, + authority=_production_stacked_authority(), + production=True, + ) + + +def _by_origin_battery_with_test_authority( + frame: Frame, + *, + authority: _StackedAuthority, +) -> GateResult: + """Explicit test-only battery seam for a digested authority bundle.""" + + _validate_test_authority(authority, boundary="by-origin battery test seam") + return _by_origin_battery_evaluate( + frame, + authority=authority, + production=False, + ) + + +def _battery_specs_from_metric_registry( + registry: Mapping[tuple[str, str, str, int], str], +) -> tuple[OriginBatterySpec, ...]: + grouped: dict[tuple[str, str, int], dict[str, str]] = {} + for (entity, family, column, clone_index), metric in registry.items(): + grouped.setdefault((entity, family, clone_index), {})[column] = metric + return tuple( + OriginBatterySpec( + entity=entity, + family=family, + clone_index=clone_index, + column_metrics=column_metrics, + ) + for (entity, family, clone_index), column_metrics in sorted(grouped.items()) + ) + + +def _by_origin_battery_evaluate( + frame: Frame, + *, + authority: _StackedAuthority, + production: bool, +) -> GateResult: + """Compare declared statistics between origins within the one spine. + + Replaces the retired spine-vs-spine agreement: the same comparisons, + scoped to gap-filled families, with per-family DECLARED metrics. The + tolerances are the chartered ones — incidence ratios within + ``[0.8, 1.25]``, conditional-quantile envelopes within ``0.25``, + categorical total-variation within ``0.25`` — deliberately NOT widened. + + Support-awareness is a validity domain, not a tolerance: a comparison + whose scope rows on either origin fall below the spec's minimum + effective support is receipted ``insufficient_support`` instead of + producing a fake confident verdict, and a quantile envelope is evaluated + only when both origins carry at least that many nonzero rows on the + compared leg. A tested rare comparison still fails on any one-sided + hole — the run-7 ``160,667x`` class fails under every profile because + both origins carry ample support. + """ + + authority_receipt = _authority_receipt(authority) + specs = _battery_specs_from_metric_registry(authority.metric_registry) + registered_targets = set(authority.metric_registry) + declared_target_set = set(_surface_target_keys(authority.declared_surface)) + missing_targets = tuple(sorted(declared_target_set - registered_targets)) + extra_targets = tuple(sorted(registered_targets - declared_target_set)) + registration_failures = _authority_validation_failures( + authority, + production=production, + ) + registration_failures.extend( + f"missing declared battery target {_battery_target_label(target)}." + for target in missing_targets + ) + registration_failures.extend( + f"metric registry target {_battery_target_label(target)} is outside the " + "declared surface." + for target in extra_targets + ) + if not specs: + registration_failures.append( + "The by-origin battery requires at least one declared metric." + ) + support_profile_receipt = authority_receipt["components"]["support_profile"] + coverage_details = { + "authority": authority_receipt, + "declared_target_count": len(declared_target_set), + "registered_target_count": len(registered_targets), + "missing_declared_targets": [ + _battery_target_label(target) for target in missing_targets + ], + "extra_registered_targets": [ + _battery_target_label(target) for target in extra_targets + ], + "declared_plan": { + "plan_id": "stacked_gap_fill_plan", + "version": authority.version, + "sha256": authority_receipt["components"]["gap_fill_plan"]["sha256"], + }, + "support_profile": support_profile_receipt, + } + if registration_failures: + return GateResult( + name=_BATTERY_GATE_NAME, + passed=False, + failures=tuple(registration_failures), + details={ + **coverage_details, + "registered_specs": len(specs), + "tested_comparisons": 0, + "untestable_comparisons": [], + "comparisons": {}, + }, + ) + validate_stacked_spine_frame(frame, boundary="by-origin battery") + + failures: list[str] = [] + comparisons: dict[str, object] = {} + untestable: list[str] = [] + tested = 0 + for spec in specs: + table = frame.table(spec.entity) + channel = table[support_channel_column(spec.entity)].astype(str) + clone_index = pd.to_numeric( + table[support_clone_index_column(spec.entity)], + errors="raise", + ).astype("int64") + weights = np.asarray( + frame.resolve_weights(spec.entity).values, + dtype=np.float64, + ) + scope = (clone_index.eq(spec.clone_index)).to_numpy() & (weights > 0.0) + left_rows = scope & channel.eq(BASE_ASEC_SUPPORT_CHANNEL).to_numpy() + right_rows = scope & channel.eq(ACS_STACKED_SUPPORT_CHANNEL).to_numpy() + for column, metric in spec.column_metrics.items(): + label = f"{spec.entity}/{spec.family}/{column}[clone_{spec.clone_index}]" + if column not in table.columns: + failures.append(f"{label}: registered column is absent from the frame.") + comparisons[label] = { + "status": "missing_column", + "metric": metric, + } + continue + series = table[column] + scoped_nulls = int(series.isna().to_numpy(dtype=bool)[scope].sum()) + if scoped_nulls: + failures.append( + f"{label}: {scoped_nulls} null value(s) inside the " + "comparison scope; the battery runs only on completed " + "surfaces." + ) + comparisons[label] = { + "status": "null_in_scope", + "metric": metric, + "null_rows": scoped_nulls, + } + continue + support = { + "asec": int(left_rows.sum()), + "acs": int(right_rows.sum()), + } + if min(support.values()) < authority.support_profile.min_effective_support: + comparisons[label] = { + "status": "insufficient_support", + "metric": metric, + "scope_rows": support, + } + untestable.append(label) + continue + tested += 1 + if metric == "categorical_tvd": + _battery_categorical_comparison( + label=label, + series=series, + left_rows=left_rows, + right_rows=right_rows, + weights=weights, + failures=failures, + comparisons=comparisons, + ) + continue + values = _battery_numeric_values( + label, + series, + metric=metric, + failures=failures, + ) + if values is None: + comparisons[label] = { + "status": "invalid_values", + "metric": metric, + } + continue + if metric in {"boolean_incidence", "rare_incidence"}: + _battery_incidence_comparison( + label=label, + metric=metric, + values=values, + left_rows=left_rows, + right_rows=right_rows, + weights=weights, + failures=failures, + comparisons=comparisons, + ) + else: + _battery_sign_separated_comparison( + label=label, + values=values, + left_rows=left_rows, + right_rows=right_rows, + weights=weights, + failures=failures, + comparisons=comparisons, + min_effective_support=( + authority.support_profile.min_effective_support + ), + ) + return GateResult( + name=_BATTERY_GATE_NAME, + passed=not failures, + failures=tuple(failures), + details={ + **coverage_details, + "tolerances": { + "incidence_ratio_bounds": list(_BATTERY_INCIDENCE_RATIO_BOUNDS), + "max_quantile_envelope_distance": ( + _BATTERY_QUANTILE_ENVELOPE_TOLERANCE + ), + "max_categorical_total_variation_distance": ( + _BATTERY_CATEGORICAL_TVD_TOLERANCE + ), + }, + "registered_specs": len(specs), + "tested_comparisons": tested, + "untestable_comparisons": sorted(untestable), + "comparisons": comparisons, + }, + ) + + +def _battery_target_label(target: tuple[str, str, str, int]) -> str: + entity, family, column, clone_index = target + return f"{entity}/{family}/{column}[clone_{clone_index}]" + + +def _battery_numeric_values( + label: str, + series: pd.Series, + *, + metric: str, + failures: list[str], +) -> np.ndarray | None: + try: + values = pd.to_numeric(series, errors="raise").to_numpy(dtype=np.float64) + except (TypeError, ValueError): + failures.append(f"{label}: declared {metric} metric requires numeric values.") + return None + if metric == "boolean_incidence": + invalid = ~np.isin(values, (0.0, 1.0)) + if invalid.any(): + failures.append( + f"{label}: declared boolean incidence requires values in " + f"{{0, 1}}; found {int(invalid.sum())} other value(s)." + ) + return None + return values + + +def _battery_incidence_comparison( + *, + label: str, + metric: str, + values: np.ndarray, + left_rows: np.ndarray, + right_rows: np.ndarray, + weights: np.ndarray, + failures: list[str], + comparisons: dict[str, object], +) -> None: + left = _weighted_nonzero_incidence(values[left_rows], weights[left_rows]) + right = _weighted_nonzero_incidence(values[right_rows], weights[right_rows]) + record: dict[str, object] = { + "status": "tested", + "metric": metric, + "asec_incidence": left, + "acs_incidence": right, + "nonzero_rows": { + "asec": int((values[left_rows] != 0.0).sum()), + "acs": int((values[right_rows] != 0.0).sum()), + }, + } + comparisons[label] = record + if left == 0.0 and right == 0.0: + failures.append( + f"{label}: zero weighted incidence on both origins with adequate " + "support; the registered comparison is dead." + ) + record["status"] = "dead_comparison" + return + ratio = math.inf if left == 0.0 else right / left + record["incidence_ratio_acs_over_asec"] = ratio if math.isfinite(ratio) else "inf" + lower, upper = _BATTERY_INCIDENCE_RATIO_BOUNDS + if not lower <= ratio <= upper: + failures.append( + f"{label}: weighted incidence ratio {ratio:.6g} is outside " + f"[{lower:.6g}, {upper:.6g}] (asec={left:.6g}, acs={right:.6g})." + ) + + +def _battery_sign_separated_comparison( + *, + label: str, + values: np.ndarray, + left_rows: np.ndarray, + right_rows: np.ndarray, + weights: np.ndarray, + failures: list[str], + comparisons: dict[str, object], + min_effective_support: int, +) -> None: + record: dict[str, object] = { + "status": "tested", + "metric": "monetary_sign_separated", + "legs": {}, + } + comparisons[label] = record + lower, upper = _BATTERY_INCIDENCE_RATIO_BOUNDS + for leg_name, leg_mask in ( + ("positive", values > 0.0), + ("negative", values < 0.0), + ): + left_leg = leg_mask & left_rows + right_leg = leg_mask & right_rows + left_incidence = _weighted_mask_incidence( + left_leg[left_rows], weights[left_rows] + ) + right_incidence = _weighted_mask_incidence( + right_leg[right_rows], weights[right_rows] + ) + leg_record: dict[str, object] = { + "asec_incidence": left_incidence, + "acs_incidence": right_incidence, + "nonzero_rows": { + "asec": int(left_leg.sum()), + "acs": int(right_leg.sum()), + }, + } + record["legs"][leg_name] = leg_record + if left_incidence == 0.0 and right_incidence == 0.0: + leg_record["status"] = "absent_on_both_origins" + continue + ratio = math.inf if left_incidence == 0.0 else right_incidence / left_incidence + leg_record["incidence_ratio_acs_over_asec"] = ( + ratio if math.isfinite(ratio) else "inf" + ) + if not lower <= ratio <= upper: + failures.append( + f"{label}/{leg_name}: weighted {leg_name}-leg incidence ratio " + f"{ratio:.6g} is outside [{lower:.6g}, {upper:.6g}] " + f"(asec={left_incidence:.6g}, acs={right_incidence:.6g})." + ) + if ( + int(left_leg.sum()) < min_effective_support + or int(right_leg.sum()) < min_effective_support + ): + leg_record["quantile_envelope"] = "leg_insufficient_support" + continue + left_quantiles = _battery_conditional_quantiles( + np.abs(values[left_leg]), weights[left_leg] + ) + right_quantiles = _battery_conditional_quantiles( + np.abs(values[right_leg]), weights[right_leg] + ) + distance = _battery_quantile_envelope_distance(left_quantiles, right_quantiles) + leg_record["quantile_envelope_distance"] = distance + if distance > _BATTERY_QUANTILE_ENVELOPE_TOLERANCE: + failures.append( + f"{label}/{leg_name}: conditional-quantile envelope distance " + f"{distance:.6g} exceeds " + f"{_BATTERY_QUANTILE_ENVELOPE_TOLERANCE:.6g}." + ) + + +def _battery_categorical_comparison( + *, + label: str, + series: pd.Series, + left_rows: np.ndarray, + right_rows: np.ndarray, + weights: np.ndarray, + failures: list[str], + comparisons: dict[str, object], +) -> None: + values = series.to_numpy(dtype=object) + distributions: dict[str, dict[str, float]] = {} + for origin, rows in (("asec", left_rows), ("acs", right_rows)): + origin_weights = weights[rows] + total = float(origin_weights.sum()) + shares: dict[str, float] = {} + for value, weight in zip(values[rows], origin_weights, strict=True): + shares[str(value)] = shares.get(str(value), 0.0) + float(weight) + distributions[origin] = { + category: share / total for category, share in shares.items() + } + categories = sorted(set(distributions["asec"]) | set(distributions["acs"])) + distance = 0.5 * sum( + abs( + distributions["asec"].get(category, 0.0) + - distributions["acs"].get(category, 0.0) + ) + for category in categories + ) + comparisons[label] = { + "status": "tested", + "metric": "categorical_tvd", + "total_variation_distance": distance, + "category_shares": distributions, + } + if distance > _BATTERY_CATEGORICAL_TVD_TOLERANCE: + failures.append( + f"{label}: categorical total-variation distance {distance:.6g} " + f"exceeds {_BATTERY_CATEGORICAL_TVD_TOLERANCE:.6g}." + ) + + +def _weighted_nonzero_incidence(values: np.ndarray, weights: np.ndarray) -> float: + total = float(weights.sum()) + if total <= 0.0: + return 0.0 + return float(weights[values != 0.0].sum() / total) + + +def _weighted_mask_incidence(mask: np.ndarray, weights: np.ndarray) -> float: + total = float(weights.sum()) + if total <= 0.0: + return 0.0 + return float(weights[mask].sum() / total) + + +def _battery_conditional_quantiles( + values: np.ndarray, + weights: np.ndarray, +) -> np.ndarray: + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(weights[order]) + cumulative /= cumulative[-1] + positions = np.minimum( + np.searchsorted(cumulative, np.asarray(_BATTERY_QUANTILES), side="left"), + len(sorted_values) - 1, + ) + return sorted_values[positions] + + +def _battery_quantile_envelope_distance( + left: np.ndarray, + right: np.ndarray, +) -> float: + denominator = np.abs(left) + np.abs(right) + distances = np.divide( + 2.0 * np.abs(left - right), + denominator, + out=np.zeros_like(denominator), + where=denominator > 0.0, + ) + return float(np.max(distances)) diff --git a/packages/populace-build/tests/test_gates.py b/packages/populace-build/tests/test_gates.py index 09046e9f..efcdd39b 100644 --- a/packages/populace-build/tests/test_gates.py +++ b/packages/populace-build/tests/test_gates.py @@ -62,6 +62,29 @@ def test_report_aggregates(self) -> None: assert manifest["passed"] is False assert manifest["gates"]["b"]["failures"] == ["broke"] + def test_report_manifest_deep_copies_nested_details(self) -> None: + details = { + "authority": {"surface": {"sha256": "a" * 64}}, + "targets": [{"name": "person/income", "receipts": ["canonical"]}], + } + result = GateResult(name="authority", passed=True, details=details) + + manifest = GateReport((result,)).to_manifest() + manifest_details = manifest["gates"]["authority"]["details"] + + details["authority"]["surface"]["sha256"] = "b" * 64 + details["targets"][0]["receipts"].append("source-tampered") + assert manifest_details["authority"]["surface"]["sha256"] == "a" * 64 + assert manifest_details["targets"][0]["receipts"] == ["canonical"] + + manifest_details["authority"]["surface"]["sha256"] = "c" * 64 + manifest_details["targets"][0]["receipts"].append("manifest-tampered") + assert result.details["authority"]["surface"]["sha256"] == "b" * 64 + assert result.details["targets"][0]["receipts"] == [ + "canonical", + "source-tampered", + ] + class TestParityGate: def test_gap_fails_with_the_variable_named(self) -> None: diff --git a/packages/populace-build/tests/test_us_spine_blindness.py b/packages/populace-build/tests/test_us_spine_blindness.py index d96b9a21..c19e8e76 100644 --- a/packages/populace-build/tests/test_us_spine_blindness.py +++ b/packages/populace-build/tests/test_us_spine_blindness.py @@ -96,6 +96,10 @@ "puf_support.py", # Validates provenance at the clone boundary. "spine_agreement.py", # Pre-calibration distribution comparison. "spine_assembly.py", # New pre-operator assembly seam. + # Stacked-spine pilot (#578 revision): stacking, gap-fill donor + # routing, activation authority, the completeness gate, and the + # by-origin battery are origin-aware by charter. + "stacked_spine.py", "support_provenance.py", # Centralized provenance compatibility. "warm_start_selection.py", # Provenance reporting and recovery. } @@ -236,6 +240,7 @@ "spine_agreement.py", "spine_assembly.py", "spm_resources.py", + "stacked_spine.py", # Provenance owner (#578 revision); see owners list. "support_provenance.py", "take_up.py", "take_up_contract.py", diff --git a/packages/populace-build/tests/test_us_stacked_spine.py b/packages/populace-build/tests/test_us_stacked_spine.py new file mode 100644 index 00000000..7b5aba85 --- /dev/null +++ b/packages/populace-build/tests/test_us_stacked_spine.py @@ -0,0 +1,3258 @@ +"""Stacked-spine pilot: stack, doctrine, gap-fill, gates (populace#578 revision). + +The ratified increment-2 revision replaces the two-spine agreement construct +with ONE origin-labeled spine: ASEC plus a seeded ACS household sample, +cross-origin gap-fill with native predictors, a single PUF pass after +gap-fill, a pre-simulation completeness gate, and a by-origin battery with +per-family declared metrics. +""" + +from __future__ import annotations + +import hashlib +import inspect +import pickle +from collections import Counter +from copy import deepcopy +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +import populace.build.us_runtime.puf_support as puf_support_module +import populace.build.us_runtime.stacked_spine as stacked_spine_module +from populace.build.gates import GateReport, GateResult +from populace.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from populace.build.us_runtime.acs_transfer import ( + declared_acs_transfer_target_families, +) +from populace.build.us_runtime.acs_transfer_bank import AcsTransferTargetBankStore +from populace.build.us_runtime.puf_support import ( + PUF_ABSENT_CELLS_PRESERVE_NULLS, + PUF_CLONE_ATTACHMENT_MANIFEST_KEY, + clone_us_frame_for_puf_support, + finalize_us_puf_tax_detail_predictions, + prepare_us_puf_tax_detail_chain_inputs, + validate_puf_clone_attachment, +) +from populace.build.us_runtime.spine_assembly import assemble_spines +from populace.build.us_runtime.stacked_spine import ( + DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES, + STACKED_PILOT_ACS_SAMPLE_FRACTION, + STACKED_PILOT_ACS_SAMPLE_SEED, + STACKED_SPINE_MANIFEST_KEY, + AbsenceProof, + GapFillDirection, + OriginBatterySpec, + assemble_stacked_spine, + by_origin_battery, + gap_fill_stacked_spine, + run_stacked_puf_pass, + sample_acs_households, + stacked_completeness_gate, + stacked_gap_fill_plan, + validate_stacked_spine_frame, +) +from populace.build.us_runtime.support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, +) +from populace.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _source_frame( + *, + household_ids: list[int], + persons_per_household: dict[int, int] | None = None, + weights: list[float], + extra_person_columns: dict[str, float | str] | None = None, + extra_household_columns: dict[str, object] | None = None, + stratum: str, +) -> Frame: + """Build one pre-assembly source frame of one-tax-unit households.""" + + persons_per_household = persons_per_household or {} + person_household: list[int] = [] + for household_id in household_ids: + person_household.extend( + [household_id] * persons_per_household.get(household_id, 1) + ) + person_count = len(person_household) + person_household_array = np.asarray(person_household, dtype=np.int64) + group_offsets = { + "tax_unit": 100_000, + "spm_unit": 200_000, + "family": 300_000, + "marital_unit": 400_000, + } + person = pd.DataFrame( + { + "person_id": np.arange(1, person_count + 1, dtype=np.int64) + + household_ids[0] * 1_000, + "person_household_id": person_household_array, + "age": np.linspace(25.0, 70.0, person_count), + } + ) + for group, offset in group_offsets.items(): + person[f"person_{group}_id"] = person_household_array + offset + for column, value in (extra_person_columns or {}).items(): + person[column] = value + + household_array = np.asarray(household_ids, dtype=np.int64) + household = pd.DataFrame( + { + "household_id": household_array, + "state_fips": np.full(len(household_ids), 6, dtype=np.int64), + } + ) + for column, value in (extra_household_columns or {}).items(): + household[column] = value + tables: dict[str, pd.DataFrame] = {"person": person, "household": household} + for group, offset in group_offsets.items(): + tables[group] = pd.DataFrame({f"{group}_id": household_array + offset}) + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), + WeightKind.DESIGN, + ) + }, + pd.Series([stratum] * person_count, dtype=object), + ) + + +def _asec_source() -> Frame: + return _source_frame( + household_ids=[11, 12], + persons_per_household={11: 2}, + weights=[300.0, 100.0], + extra_person_columns={"asec_detail_income": 40.0}, + stratum="asec_2024", + ) + + +def _acs_source() -> Frame: + return _source_frame( + household_ids=list(range(101, 111)), + persons_per_household={103: 3, 107: 2}, + weights=[float(10 * position) for position in range(1, 11)], + extra_person_columns={"acs_native_aggregate": 15.0}, + extra_household_columns={"puma": "0600101"}, + stratum="acs_2024_1yr", + ) + + +def test_stacked_assembly_is_deterministic_and_floor_exact() -> None: + first = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.25, + acs_sample_seed=578, + ) + repeated = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.25, + acs_sample_seed=578, + ) + + sample = first.receipt["acs_sample"] + assert sample["eligible_household_count"] == 10 + assert sample["requested_household_count"] == 2 + assert sample["realized_household_count"] == 2 + assert sample["exact_count_rule"] == "floor(fraction * eligible)" + assert first.receipt["acs_sample_fraction"] == 0.25 + assert first.receipt["acs_sample_seed"] == 578 + assert ( + first.receipt["acs_sample"]["selected_household_ids_sha256"] + == repeated.receipt["acs_sample"]["selected_household_ids_sha256"] + ) + for entity in first.frame.entities: + assert_frame_equal(first.frame.table(entity), repeated.frame.table(entity)) + np.testing.assert_array_equal( + first.frame.weights_for("household").values, + repeated.frame.weights_for("household").values, + ) + + manifest = first.frame.metadata[STACKED_SPINE_MANIFEST_KEY] + assert manifest["acs_sample_fraction"] == 0.25 + assert manifest["acs_sample_seed"] == 578 + assert validate_stacked_spine_frame( + first.frame, + boundary="determinism fixture", + ) + + +def test_stacked_assembly_seed_and_fraction_bind_identity() -> None: + base = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.25, + acs_sample_seed=578, + ) + changed_seed = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.25, + acs_sample_seed=579, + ) + wider_fraction = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.55, + acs_sample_seed=578, + ) + + assert ( + base.receipt["acs_sample"]["selected_household_ids_sha256"] + != changed_seed.receipt["acs_sample"]["selected_household_ids_sha256"] + ) + assert wider_fraction.receipt["acs_sample"]["realized_household_count"] == 5 + + +@pytest.mark.parametrize( + ("mutate", "match"), + ( + ( + lambda manifest: manifest["acs_sample"].__setitem__( + "realized_household_count", 3 + ), + "realized household count", + ), + ( + lambda manifest: manifest["acs_sample"].__setitem__( + "selected_household_ids_sha256", "0" * 64 + ), + "selection digest", + ), + ( + lambda manifest: manifest.__setitem__("acs_sample_fraction", 0.35), + "violates floor", + ), + ( + lambda manifest: manifest.__setitem__("acs_sample_seed", "578"), + "acs_sample_seed", + ), + ( + lambda manifest: manifest.pop("acs_sample"), + "sample receipt", + ), + ), +) +def test_stacked_manifest_mutations_fail_closed(mutate, match) -> None: + result = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=0.25, + acs_sample_seed=578, + ) + stacked = result.frame + manifest = { + key: ( + { + nested_key: ( + dict(nested_value) + if isinstance(nested_value, dict) + else nested_value + ) + for nested_key, nested_value in value.items() + } + if isinstance(value, dict) + else value + ) + for key, value in result.receipt.items() + } + mutate(manifest) + tampered = Frame( + {entity: stacked.table(entity) for entity in stacked.entities}, + stacked.schema, + {entity: stacked.weights_for(entity) for entity in stacked.weighted_entities}, + stacked.strata, + mass_log=stacked.mass_log, + metadata={**stacked.metadata, STACKED_SPINE_MANIFEST_KEY: manifest}, + ) + + with pytest.raises(ValueError, match=match): + validate_stacked_spine_frame(tampered, boundary="tampered fixture") + + +def test_sample_acs_households_takes_whole_lineages() -> None: + acs = _acs_source() + sampled, receipt = sample_acs_households(acs, fraction=0.55, seed=7) + + assert receipt["requested_household_count"] == 5 + selected_households = set(sampled.table("household")["household_id"].tolist()) + person = sampled.table("person") + assert set(person["person_household_id"]) == selected_households + full_person = acs.table("person") + for household_id in selected_households: + expected = int((full_person["person_household_id"] == household_id).sum()) + actual = int((person["person_household_id"] == household_id).sum()) + assert actual == expected + for group in ("tax_unit", "spm_unit", "family", "marital_unit"): + assert set(sampled.table(group)[f"{group}_id"]) == set( + person[f"person_{group}_id"] + ) + + +def test_sample_floor_zero_fails_closed() -> None: + with pytest.raises(ValueError, match="floors to zero"): + sample_acs_households(_acs_source(), fraction=0.05, seed=1) + + +def test_sample_rejects_provenance_carrying_source() -> None: + stacked = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=1.0, + acs_sample_seed=0, + ).frame + with pytest.raises(ValueError, match="already carries support provenance"): + sample_acs_households(stacked, fraction=0.5, seed=1) + + +def test_weight_harmonization_matches_share_math() -> None: + asec = _asec_source() + acs = _acs_source() + result = assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=0.25, + acs_sample_seed=578, + ) + stacked = result.frame + anchor_mass = float(asec.weights_for("household").total) + + household = stacked.table("household") + weights = stacked.weights_for("household").values + channel = household[support_channel_column("household")] + asec_mass = float(weights[channel.eq("asec").to_numpy()].sum()) + acs_mass = float(weights[channel.eq("acs").to_numpy()].sum()) + assert np.isclose(asec_mass, 0.5 * anchor_mass, rtol=1e-12) + assert np.isclose(acs_mass, 0.5 * anchor_mass, rtol=1e-12) + assert float(stacked.weights_for("household").total) == anchor_mass + + harmonization = result.receipt["weight_harmonization"] + sampled_mass = result.receipt["acs_sample"]["sampled_household_mass"] + assert np.isclose( + harmonization["acs"]["scale_factor"], + 0.5 * anchor_mass / sampled_mass, + rtol=1e-12, + ) + assert harmonization["asec"]["incoming_mass"] == anchor_mass + assert np.isclose( + harmonization["asec"]["scale_factor"], + 0.5, + rtol=1e-12, + ) + + acs_weights = weights[channel.eq("acs").to_numpy()] + source_ids = household.loc[ + channel.eq("acs").to_numpy(), + spine_source_id_column("household"), + ].to_numpy() + incoming_by_id = dict( + zip( + acs.table("household")["household_id"].tolist(), + acs.weights_for("household").values.tolist(), + strict=True, + ) + ) + incoming = np.asarray( + [incoming_by_id[int(value)] for value in source_ids], + dtype=np.float64, + ) + np.testing.assert_allclose( + acs_weights, + incoming * harmonization["acs"]["scale_factor"], + rtol=1e-9, + ) + + +def test_weight_harmonization_receipts_use_the_live_selected_anchor() -> None: + result = assemble_stacked_spine( + _asec_source(), + _acs_source(), + acs_sample_fraction=1.0, + acs_sample_seed=0, + mass_anchor_channel="acs", + ) + + assert result.frame.weights_for("household").total == 550.0 + harmonization = result.receipt["weight_harmonization"] + for channel in ("asec", "acs"): + assert harmonization[channel]["declared_allocation"] == 275.0 + assert harmonization[channel]["allocated_mass"] == 275.0 + + metadata = { + **result.frame.metadata, + STACKED_SPINE_MANIFEST_KEY: deepcopy(result.receipt), + } + metadata[STACKED_SPINE_MANIFEST_KEY]["weight_harmonization"]["asec"][ + "declared_allocation" + ] = 200.0 + forged = Frame( + {entity: result.frame.table(entity) for entity in result.frame.entities}, + result.frame.schema, + { + entity: result.frame.weights_for(entity) + for entity in result.frame.weighted_entities + }, + result.frame.strata, + mass_log=result.frame.mass_log, + metadata=metadata, + ) + with pytest.raises( + ValueError, + match=( + "declared 'asec' allocation 200.0 differs from share 0.5 times " + "live anchor mass 550.0" + ), + ): + validate_stacked_spine_frame(forged, boundary="forged allocation") + + +def test_fraction_one_matches_plain_assembly() -> None: + asec = _asec_source() + acs = _acs_source() + result = assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=1.0, + acs_sample_seed=123, + ) + plain = assemble_spines( + {"asec": _asec_source(), "acs": _acs_source()}, + household_mass_shares=dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES), + mass_anchor_channel="asec", + ) + + sample = result.receipt["acs_sample"] + assert sample["realized_household_count"] == sample["eligible_household_count"] + for entity in plain.entities: + assert_frame_equal(result.frame.table(entity), plain.table(entity)) + np.testing.assert_array_equal( + result.frame.weights_for("household").values, + plain.weights_for("household").values, + ) + + +def test_selection_digest_uses_raw_spine_ids_under_collision_remap() -> None: + asec = _source_frame( + household_ids=[101, 102], + weights=[300.0, 100.0], + extra_person_columns={"asec_detail_income": 40.0}, + stratum="asec_2024", + ) + result = assemble_stacked_spine( + asec, + _acs_source(), + acs_sample_fraction=1.0, + acs_sample_seed=0, + ) + + household = result.frame.table("household") + channel = household[support_channel_column("household")] + clone_index = household[support_clone_index_column("household")] + native_acs = channel.eq("acs") & clone_index.eq(0) + remapped = household.loc[native_acs, "household_id"].to_numpy() + raw = household.loc[native_acs, spine_source_id_column("household")].to_numpy() + assert not np.array_equal(np.sort(remapped), np.sort(raw)) + assert validate_stacked_spine_frame( + result.frame, + boundary="collision fixture", + ) + + +def _asec_detail_source() -> Frame: + """ASEC arm with observed tax-detail sentinels and PUF-pass predictors.""" + + frame = _source_frame( + household_ids=[11, 12], + persons_per_household={11: 2}, + weights=[300.0, 100.0], + stratum="asec_2024", + ) + person = frame.table("person").copy() + person["employment_income_before_lsr"] = np.asarray([50_000.0, 20_000.0, 35_000.0]) + person["taxable_interest_income"] = np.asarray([100.0, 0.0, 200.0]) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +def _cloned_stacked_fixture() -> Frame: + stacked = assemble_stacked_spine( + _asec_detail_source(), + _acs_source(), + acs_sample_fraction=1.0, + acs_sample_seed=0, + ).frame + return clone_us_frame_for_puf_support(stacked) + + +def _finalize_fixture_predictions( + cloned: Frame, +) -> tuple[pd.DataFrame, pd.DataFrame]: + tax_unit = cloned.table("tax_unit") + puf_mask = tax_unit[support_clone_index_column("tax_unit")].eq(1).to_numpy() + predictions = pd.DataFrame( + { + "taxable_interest_income": np.full(int(puf_mask.sum()), 100.0), + "health_savings_account_ald": np.full(int(puf_mask.sum()), 750.0), + }, + index=tax_unit.index[puf_mask], + ) + donor = pd.DataFrame( + { + "taxable_interest_income": [100.0, 200.0, 100.0, 150.0], + "health_savings_account_ald": [750.0, 500.0, 250.0, 1_000.0], + "weight": [1.0, 1.0, 1.0, 1.0], + } + ) + return predictions, donor + + +def test_finalize_preserve_nulls_keeps_unowned_cells_null() -> None: + cloned = _cloned_stacked_fixture() + predictions, donor = _finalize_fixture_predictions(cloned) + before_person = cloned.table("person").copy(deep=True) + + finalized = finalize_us_puf_tax_detail_predictions( + cloned, + donor, + predictions.copy(), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + + person = finalized.table("person") + channel = person[support_channel_column("person")] + clone_index = person[support_clone_index_column("person")] + native_acs = channel.eq("acs") & clone_index.eq(0) + native_asec = channel.eq("asec") & clone_index.eq(0) + detail = clone_index.eq(1) + assert person.loc[native_acs, "taxable_interest_income"].isna().all() + pd.testing.assert_series_equal( + person.loc[native_asec, "taxable_interest_income"], + before_person.loc[native_asec.to_numpy(), "taxable_interest_income"], + check_names=False, + ) + assert person.loc[detail, "taxable_interest_income"].notna().all() + + tax_unit = finalized.table("tax_unit") + tax_unit_clone = tax_unit[support_clone_index_column("tax_unit")] + assert tax_unit.loc[tax_unit_clone.eq(0), "health_savings_account_ald"].isna().all() + assert ( + tax_unit.loc[tax_unit_clone.eq(1), "health_savings_account_ald"].notna().all() + ) + + +def test_finalize_legacy_zero_fill_reproduces_the_audited_defect() -> None: + """Pin the run-7 boundary: legacy finalization reads absence as zero.""" + + cloned = _cloned_stacked_fixture() + predictions, donor = _finalize_fixture_predictions(cloned) + + finalized = finalize_us_puf_tax_detail_predictions( + cloned, + donor, + predictions.copy(), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + ) + + person = finalized.table("person") + channel = person[support_channel_column("person")] + clone_index = person[support_clone_index_column("person")] + native_acs = channel.eq("acs") & clone_index.eq(0) + assert person.loc[native_acs, "taxable_interest_income"].eq(0.0).all() + tax_unit = finalized.table("tax_unit") + tax_unit_clone = tax_unit[support_clone_index_column("tax_unit")] + assert ( + tax_unit.loc[tax_unit_clone.eq(0), "health_savings_account_ald"].eq(0.0).all() + ) + + +def test_finalize_legacy_zero_fill_matches_run_7_protocol_5_byte_pin() -> None: + """Pin every legacy run-7 output byte, not only its zero-region meaning.""" + + cloned = _cloned_stacked_fixture() + predictions, donor = _finalize_fixture_predictions(cloned) + finalized = finalize_us_puf_tax_detail_predictions( + cloned, + donor, + predictions.copy(), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + ) + + actual = ( + finalized.table("person")["taxable_interest_income"].to_numpy(copy=True), + finalized.table("tax_unit")["health_savings_account_ald"].to_numpy(copy=True), + ) + actual_bytes = pickle.dumps(actual, protocol=5) + assert actual_bytes[:2] == b"\x80\x05" + assert hashlib.sha256(actual_bytes).hexdigest() == ( + "f2dc86f630a41c7b0eb060d7cd630bfadda423ae26c4311403116e3e6cb7720e" + ) + + +def test_preserve_nulls_sparsification_never_rewrites_native_rows() -> None: + cloned = _cloned_stacked_fixture() + predictions, donor = _finalize_fixture_predictions(cloned) + donor = donor.assign( + taxable_interest_income=[100.0, 0.0, 0.0, 0.0], + ) + before_person = cloned.table("person").copy(deep=True) + + finalized = finalize_us_puf_tax_detail_predictions( + cloned, + donor, + predictions.copy(), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + + person = finalized.table("person") + clone_index = person[support_clone_index_column("person")] + channel = person[support_channel_column("person")] + native = clone_index.eq(0) + native_asec = native & channel.eq("asec") + pd.testing.assert_series_equal( + person.loc[native_asec, "taxable_interest_income"], + before_person.loc[native_asec.to_numpy(), "taxable_interest_income"], + check_names=False, + ) + assert ( + person.loc[native & channel.eq("acs"), "taxable_interest_income"].isna().all() + ) + detail_units = ( + person.loc[clone_index.eq(1)] + .groupby("person_tax_unit_id", sort=False)["taxable_interest_income"] + .sum() + ) + assert (detail_units == 0.0).any() + + +def test_strict_recipient_predictors_fail_closed_on_absence() -> None: + cloned = _cloned_stacked_fixture() + donor = pd.DataFrame( + { + "employment_income": [45_000.0, 8_000.0], + "taxable_interest_income": [120.0, 30.0], + "weight": [1.0, 1.0], + } + ) + + with pytest.raises(ValueError, match="puf_predictor_employment_income"): + prepare_us_puf_tax_detail_chain_inputs( + cloned, + donor, + predictors=("puf_predictor_employment_income",), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=(), + require_complete_recipient_predictors=True, + ) + + legacy = prepare_us_puf_tax_detail_chain_inputs( + cloned, + donor, + predictors=("puf_predictor_employment_income",), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=(), + ) + assert not legacy.recipient_features.isna().any().any() + tax_unit = cloned.table("tax_unit") + detail_mask = tax_unit[support_clone_index_column("tax_unit")].eq(1).to_numpy() + acs_detail = ( + tax_unit.loc[detail_mask, support_channel_column("tax_unit")] + .eq("acs") + .to_numpy() + ) + zero_filled = legacy.recipient_features.loc[ + acs_detail, "puf_predictor_employment_income" + ] + assert zero_filled.eq(0.0).all() + + +def test_strict_recipient_predictors_reject_null_filing_status_before_coercion( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A missing filing status is terminal before status-code conversion.""" + + cloned = _cloned_stacked_fixture() + tax_unit = cloned.table("tax_unit") + tax_unit["filing_status_input"] = "SINGLE" + puf_mask = tax_unit[support_clone_index_column("tax_unit")].eq(1) + tax_unit.loc[tax_unit.index[puf_mask][0], "filing_status_input"] = np.nan + donor = pd.DataFrame( + { + "puf_predictor_filing_status_code": [1.0, 2.0], + "taxable_interest_income": [120.0, 30.0], + "weight": [1.0, 1.0], + } + ) + + def reject_early_coercion(_values: object) -> np.ndarray: + raise AssertionError("filing-status coercion ran before recipient validation") + + monkeypatch.setattr( + puf_support_module, + "_filing_status_codes", + reject_early_coercion, + ) + with pytest.raises( + ValueError, + match=r"puf_predictor_filing_status_code.*1.*recipient rows", + ): + prepare_us_puf_tax_detail_chain_inputs( + cloned, + donor, + predictors=("puf_predictor_filing_status_code",), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=(), + require_complete_recipient_predictors=True, + ) + + +def _asec_gap_source() -> Frame: + """ASEC arm observing survey detail plus the native donor analogs.""" + + frame = _source_frame( + household_ids=[11, 12, 13, 14], + persons_per_household={11: 2, 13: 2}, + weights=[300.0, 100.0, 200.0, 150.0], + stratum="asec_2024", + ) + person = frame.table("person").copy() + count = len(person) + person["is_female"] = np.asarray([False, True, True, False, True, False]) + person["is_household_head"] = np.asarray([True, False, True, True, False, True]) + person["employment_income_before_lsr"] = np.linspace(10_000.0, 60_000.0, count) + person["unemployment_compensation"] = np.asarray( + [0.0, 1_200.0, 0.0, 3_600.0, 0.0, 2_400.0] + ) + person["is_disabled"] = np.asarray([False, False, True, False, False, True]) + for column, base in ( + ("taxable_interest_income", 100.0), + ("tax_exempt_interest_income", 0.0), + ("qualified_dividend_income", 50.0), + ("non_qualified_dividend_income", 25.0), + ("rental_income", 0.0), + ("estate_income", 0.0), + ): + person[column] = np.linspace(base, base * 2 if base else 0.0, count) + household = frame.table("household").copy() + household["tenure_type"] = pd.Series( + ["OWNED_WITH_MORTGAGE", "RENTED", "OWNED_OUTRIGHT", "RENTED"], + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + tables["household"] = household + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +def _acs_gap_source() -> Frame: + """ACS arm observing housing plus the honest native aggregates.""" + + frame = _source_frame( + household_ids=list(range(101, 111)), + persons_per_household={103: 2}, + weights=[float(10 * position) for position in range(1, 11)], + stratum="acs_2024_1yr", + ) + person = frame.table("person").copy() + count = len(person) + person["is_female"] = np.asarray([position % 2 == 0 for position in range(count)]) + person["is_household_head"] = ~person["person_household_id"].duplicated() + person["employment_income_before_lsr"] = np.linspace(8_000.0, 90_000.0, count) + person["acs_interest_dividend_rental_income"] = np.asarray( + [0.0, 400.0, 0.0, 150.0, 900.0, 0.0, 250.0, 0.0, 3_000.0, 120.0, 60.0] + ) + person["pre_subsidy_rent"] = np.asarray( + [ + 0.0, + 14_400.0, + 0.0, + 0.0, + 9_600.0, + 12_000.0, + 0.0, + 18_000.0, + 0.0, + 7_200.0, + 15_600.0, + ] + ) + household = frame.table("household").copy() + household["tenure_type"] = pd.Series( + [ + "OWNED_OUTRIGHT", + "RENTED", + "OWNED_WITH_MORTGAGE", + "OWNED_OUTRIGHT", + "RENTED", + "RENTED", + "OWNED_WITH_MORTGAGE", + "RENTED", + "OWNED_OUTRIGHT", + "RENTED", + ], + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + tables["household"] = household + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +_GAP_FILL_TEST_PLAN = ( + GapFillDirection( + name="asec_survey_to_acs", + recipient_channel="acs", + donor_channel="asec", + target_families={ + "person": { + "model_required_numeric": ("unemployment_compensation",), + "model_required_boolean": ("is_disabled",), + } + }, + ), + GapFillDirection( + name="acs_housing_to_asec", + recipient_channel="asec", + donor_channel="acs", + target_families={"person": {"housing": ("pre_subsidy_rent",)}}, + ), +) + + +def test_production_entrypoints_take_no_authority_parameters() -> None: + production_entrypoints = ( + gap_fill_stacked_spine, + stacked_completeness_gate, + by_origin_battery, + ) + authority_parameter_tokens = { + "authority", + "canonical", + "declared", + "metric", + "metrics", + "plan", + "profile", + "registry", + "support", + "surface", + } + + for entrypoint in production_entrypoints: + authority_parameters = { + parameter + for parameter in inspect.signature(entrypoint).parameters + if authority_parameter_tokens.intersection(parameter.split("_")) + } + assert not authority_parameters, ( + f"{entrypoint.__name__} exposes caller-controlled authority " + f"parameter(s): {sorted(authority_parameters)}" + ) + + +def test_canonical_authority_objects_are_deeply_immutable() -> None: + plan = stacked_spine_module.CANONICAL_STACKED_GAP_FILL_PLAN + surface = stacked_spine_module.CANONICAL_STACKED_DECLARED_SURFACE + registry = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY + profile = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE + + assert isinstance(plan, tuple) + with pytest.raises(TypeError): + plan[0].target_families["person"]["model_required_numeric"] = () + with pytest.raises(TypeError): + surface["person"]["model_required_numeric"] = () + with pytest.raises(TypeError): + registry[("person", "puf_tax_itemization", "taxable_interest_income", 0)] = ( + "rare_incidence" + ) + with pytest.raises(FrozenInstanceError): + profile.min_effective_support = 50 + + +def test_canonical_metric_registry_covers_the_declared_90_target_split() -> None: + surface = stacked_spine_module.CANONICAL_STACKED_DECLARED_SURFACE + registry = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY + surface_targets = { + (entity, family, target, 0) + for entity, families in surface.items() + for family, targets in families.items() + for target in targets + } + + assert len(surface_targets) == 90 + assert set(registry) == surface_targets + assert Counter(registry.values()) == { + "monetary_sign_separated": 62, + "boolean_incidence": 26, + "categorical_tvd": 2, + } + assert ( + registry[("person", "puf_tax_itemization", "taxable_interest_income", 0)] + == "monetary_sign_separated" + ) + + +def test_explicit_test_seams_reject_the_canonical_authority() -> None: + authority = stacked_spine_module._production_stacked_authority() + frame = _stacked_gap_fixture() + + with pytest.raises(ValueError, match="NON-CANONICAL test authority"): + stacked_spine_module._gap_fill_stacked_spine_with_test_authority( + frame, + authority=authority, + ) + with pytest.raises(ValueError, match="NON-CANONICAL test authority"): + stacked_spine_module._stacked_completeness_gate_with_test_authority( + frame, + authority=authority, + ) + with pytest.raises(ValueError, match="NON-CANONICAL test authority"): + stacked_spine_module._by_origin_battery_with_test_authority( + frame, + authority=authority, + ) + + +def _surface_from_gap_fill_plan( + plan: tuple[GapFillDirection, ...], +) -> dict[str, dict[str, tuple[str, ...]]]: + surface: dict[str, dict[str, tuple[str, ...]]] = {} + for direction in plan: + for entity, families in direction.target_families.items(): + for family, targets in families.items(): + surface.setdefault(entity, {})[family] = tuple(targets) + return surface + + +def _gap_fill_with_test_authority( + frame: Frame, + *, + plan: tuple[GapFillDirection, ...], + **kwargs: object, +): + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface=_surface_from_gap_fill_plan(plan), + gap_fill_plan=plan, + ) + return stacked_spine_module._gap_fill_stacked_spine_with_test_authority( + frame, + authority=authority, + **kwargs, + ) + + +def _completeness_with_test_authority( + frame: Frame, + *, + declared_surface: dict[str, dict[str, tuple[str, ...]]], + declared_gap_fill_plan: tuple[GapFillDirection, ...], + absence_proofs: tuple[AbsenceProof, ...] = (), +): + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface=declared_surface, + gap_fill_plan=declared_gap_fill_plan, + ) + return stacked_spine_module._stacked_completeness_gate_with_test_authority( + frame, + authority=authority, + absence_proofs=absence_proofs, + ) + + +def _battery_with_test_authority( + frame: Frame, + *, + registry: tuple[OriginBatterySpec, ...], +): + surface = { + entity: {family: tuple(targets) for family, targets in families.items()} + for entity, families in ( + stacked_spine_module.CANONICAL_STACKED_DECLARED_SURFACE.items() + ) + } + metrics: dict[tuple[str, str, str, int], str] = {} + for spec in registry: + family_targets = list(surface.setdefault(spec.entity, {}).get(spec.family, ())) + for column, metric in spec.column_metrics.items(): + if column not in family_targets: + family_targets.append(column) + metrics[(spec.entity, spec.family, column, spec.clone_index)] = metric + surface[spec.entity][spec.family] = tuple(family_targets) + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface=surface, + metric_registry=metrics, + ) + return stacked_spine_module._by_origin_battery_with_test_authority( + frame, + authority=authority, + ) + + +def _stacked_gap_fixture() -> Frame: + return assemble_stacked_spine( + _asec_gap_source(), + _acs_gap_source(), + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame + + +def test_gap_fill_plan_covers_declared_families_exactly() -> None: + plan = stacked_gap_fill_plan() + assert [direction.name for direction in plan] == [ + "asec_survey_to_acs", + "acs_housing_to_asec", + ] + survey, housing = plan + assert survey.recipient_channel == "acs" + assert survey.donor_channel == "asec" + assert housing.recipient_channel == "asec" + assert housing.donor_channel == "acs" + assert set(housing.target_families) == {"person"} + assert housing.target_families["person"] == {"housing": ("pre_subsidy_rent",)} + + declared = declared_acs_transfer_target_families() + recombined: dict[str, dict[str, tuple[str, ...]]] = {} + for direction in plan: + for entity, families in direction.target_families.items(): + for family, targets in families.items(): + recombined.setdefault(entity, {})[family] = tuple(targets) + assert recombined == { + entity: {family: tuple(targets) for family, targets in families.items()} + for entity, families in declared.items() + } + + +def test_gap_fill_fills_both_directions_with_authority_receipts() -> None: + stacked = _stacked_gap_fixture() + result = _gap_fill_with_test_authority( + stacked, + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + ) + + person = result.frame.table("person") + channel = person[support_channel_column("person")] + acs_rows = channel.eq("acs") + asec_rows = channel.eq("asec") + for column in ("unemployment_compensation", "is_disabled"): + assert person.loc[acs_rows, column].notna().all() + assert person.loc[asec_rows, "pre_subsidy_rent"].notna().all() + + before_person = stacked.table("person") + for column in ("unemployment_compensation", "is_disabled"): + pd.testing.assert_series_equal( + person.loc[asec_rows, column], + before_person.loc[asec_rows.to_numpy(), column], + check_names=False, + ) + pd.testing.assert_series_equal( + person.loc[acs_rows, "pre_subsidy_rent"], + before_person.loc[acs_rows.to_numpy(), "pre_subsidy_rent"], + check_names=False, + ) + + directions = result.receipt["directions"] + survey = directions["asec_survey_to_acs"] + assert survey["donor_selection"] == "owner_projection_of_native_donor_rows" + assert survey["resolved_donor_channel"] is None + unemployment = survey["targets"][ + "person/model_required_numeric/unemployment_compensation" + ] + assert unemployment["authorized_null_rows"] == int(acs_rows.sum()) + assert unemployment["imputed_rows"] == int(acs_rows.sum()) + assert unemployment["residual_null_rows"] == 0 + housing = directions["acs_housing_to_asec"] + rent = housing["targets"]["person/housing/pre_subsidy_rent"] + assert rent["imputed_rows"] == int(asec_rows.sum()) + assert rent["residual_null_rows"] == 0 + + survey_transfer = result.transfer_results["asec_survey_to_acs"] + native_predictor_used = any( + "__acs_transfer_interest_dividend_rental_income" + in pattern.observed_optional_predictors + for record in survey_transfer.imputed_inputs + for pattern in record.patterns + ) + assert native_predictor_used + + +def test_gap_fill_outcome_rejects_forged_residual_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A transfer cannot receipt unmodeled rows after filling every hole.""" + + transfer = stacked_spine_module.transfer_acs_inputs + + def forge_unmodeled_rows(*args: object, **kwargs: object) -> object: + result = transfer(*args, **kwargs) + return replace( + result, + imputed_inputs=tuple( + replace(record, unmodeled_recipient_rows=1) + if record.column == "unemployment_compensation" + else record + for record in result.imputed_inputs + ), + ) + + monkeypatch.setattr( + stacked_spine_module, + "transfer_acs_inputs", + forge_unmodeled_rows, + ) + with pytest.raises( + ValueError, + match=( + "residual-null equation failed: residual_null_rows=0 != unmodeled_rows=1" + ), + ) as error: + _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + ) + assert ( + "activation accounting equation failed: authorized_null_rows=11 " + "!= imputed_rows=11 + unmodeled_rows=1" in str(error.value) + ) + + +def test_gap_fill_rejects_signed_zero_donor_byte_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Donor identity distinguishes equal-valued IEEE-754 signed zeros.""" + + stacked = _stacked_gap_fixture() + person = stacked.table("person") + channel_column = support_channel_column("person") + donor_rows = person[channel_column].astype(str).eq("asec") + donor_index = person.index[ + donor_rows & person["unemployment_compensation"].eq(0.0) + ][0] + person.loc[donor_index, "unemployment_compensation"] = -0.0 + assert np.signbit(person.loc[donor_index, "unemployment_compensation"]) + + transfer = stacked_spine_module.transfer_acs_inputs + + def flip_donor_signed_zero(*args: object, **kwargs: object) -> object: + result = transfer(*args, **kwargs) + result_person = result.frame.table("person").copy(deep=True) + assert np.signbit(result_person.loc[donor_index, "unemployment_compensation"]) + result_person.loc[donor_index, "unemployment_compensation"] = 0.0 + tables = { + entity: result.frame.table(entity) for entity in result.frame.entities + } + tables["person"] = result_person + changed = Frame( + tables, + result.frame.schema, + { + entity: result.frame.weights_for(entity) + for entity in result.frame.weighted_entities + }, + result.frame.strata, + mass_log=result.frame.mass_log, + metadata=result.frame.metadata, + ) + return replace(result, frame=changed) + + monkeypatch.setattr( + stacked_spine_module, + "transfer_acs_inputs", + flip_donor_signed_zero, + ) + with pytest.raises( + ValueError, + match=( + r"asec_survey_to_acs/person/model_required_numeric/" + r"unemployment_compensation: donor byte identity failed.*" + r"canonical donor payload changed" + ), + ): + _gap_fill_with_test_authority( + stacked, + plan=(_GAP_FILL_TEST_PLAN[0],), + seed=578, + n_estimators=10, + ) + + +def test_donor_byte_identity_canonicalizes_semantic_strings() -> None: + index = pd.Index([7, 3], name="donor_row") + object_strings = pd.Series( + ["RENTED", None], + index=index, + name="tenure_type", + dtype=object, + ) + canonical_strings = object_strings.astype(CANONICAL_STRING_DTYPE) + + assert stacked_spine_module._canonical_donor_series_payload( + object_strings, + boundary="object-string donor identity", + ) == stacked_spine_module._canonical_donor_series_payload( + canonical_strings, + boundary="canonical-string donor identity", + ) + + unchanged_controls = ( + pd.Series([True, False], name="native_bool", dtype=bool), + pd.Series([True, pd.NA], name="nullable_bool", dtype="boolean"), + pd.Series( + pd.Categorical(["a", None], categories=["a", "b"]), + name="native_category", + ), + pd.Series([None, np.nan], name="all_null_object", dtype=object), + ) + for control in unchanged_controls: + assert stacked_spine_module._canonical_donor_series_payload( + control, + boundary=f"{control.name} donor identity before", + ) == stacked_spine_module._canonical_donor_series_payload( + control.copy(deep=True), + boundary=f"{control.name} donor identity after", + ) + + mixed = pd.Series( + ["RENTED", 1], + name="mixed_tenure_type", + dtype=object, + ) + with pytest.raises(TypeError, match="semantic strings cannot mix"): + stacked_spine_module._canonical_donor_series_payload( + mixed, + boundary="mixed-object donor identity", + ) + + +def test_donor_byte_identity_ignores_string_object_alias_topology() -> None: + shared = "".join(["dynamically", "-", "allocated", "-", "donor"]) + separate = [ + "".join(["dynamically", "-", "allocated", "-", "donor"]) for _ in range(2) + ] + before = pd.Series([shared, shared], name="native_label", dtype=object) + after = pd.Series(separate, name="native_label", dtype=object) + assert before.iloc[0] is before.iloc[1] + assert after.iloc[0] is not after.iloc[1] + + assert stacked_spine_module._canonical_donor_series_payload( + before, + boundary="aliased-string donor identity before", + ) == stacked_spine_module._canonical_donor_series_payload( + after, + boundary="aliased-string donor identity after", + ) + + +def test_donor_byte_identity_accepts_semantic_boolean_object_scalars() -> None: + semantic_booleans = pd.Series( + [True, np.bool_(False), None], + name="is_disabled", + dtype=object, + ) + + assert stacked_spine_module._canonical_donor_series_payload( + semantic_booleans, + boundary="semantic-boolean donor identity before", + ) == stacked_spine_module._canonical_donor_series_payload( + semantic_booleans.copy(deep=True), + boundary="semantic-boolean donor identity after", + ) + + +def test_gap_fill_activation_authority_fails_closed_on_donor_nulls() -> None: + stacked = _stacked_gap_fixture() + person = stacked.table("person").copy() + channel = person[support_channel_column("person")] + poke = person.index[channel.eq("asec")][1] + person.loc[poke, "unemployment_compensation"] = np.nan + tables = {entity: stacked.table(entity) for entity in stacked.entities} + tables["person"] = person + poked = Frame( + tables, + stacked.schema, + {entity: stacked.weights_for(entity) for entity in stacked.weighted_entities}, + stacked.strata, + mass_log=stacked.mass_log, + metadata=stacked.metadata, + ) + + with pytest.raises(ValueError, match="donors must observe"): + _gap_fill_with_test_authority(poked, plan=_GAP_FILL_TEST_PLAN, seed=578) + + +@pytest.mark.parametrize( + ("recipient_channel", "donor_channel", "role", "missing_channel"), + ( + ("acx_typo", "asec", "recipient", "acx_typo"), + ("acs", "asec_typo", "donor", "asec_typo"), + ), +) +def test_gap_fill_activation_authority_rejects_nonlive_declared_channel( + recipient_channel: str, + donor_channel: str, + role: str, + missing_channel: str, +) -> None: + plan = ( + GapFillDirection( + name="nonlive_channel", + recipient_channel=recipient_channel, + donor_channel=donor_channel, + target_families={"person": {"complete": ("age",)}}, + ), + ) + + with pytest.raises( + ValueError, + match=f"declared {role} channel {missing_channel!r} has no live rows", + ): + _gap_fill_with_test_authority(_stacked_gap_fixture(), plan=plan, seed=578) + + +def test_gap_fill_fails_closed_on_missing_target_column() -> None: + stacked = _stacked_gap_fixture() + plan = ( + GapFillDirection( + name="asec_survey_to_acs", + recipient_channel="acs", + donor_channel="asec", + target_families={ + "person": {"model_required_numeric": ("veterans_benefits",)} + }, + ), + ) + with pytest.raises(ValueError, match="veterans_benefits.*absent"): + _gap_fill_with_test_authority(stacked, plan=plan, seed=578) + + +def test_gap_fill_rejects_cloned_frames() -> None: + cloned = clone_us_frame_for_puf_support(_stacked_gap_fixture()) + with pytest.raises(ValueError, match="before clone operators"): + _gap_fill_with_test_authority(cloned, plan=_GAP_FILL_TEST_PLAN, seed=578) + + +def test_gap_fill_banks_per_target_via_608_store(tmp_path) -> None: + identity = {"pilot": "stacked-gap-fill", "seed": 578} + banks = { + "asec_survey_to_acs": AcsTransferTargetBankStore( + tmp_path / "survey", + identity=identity, + ), + "acs_housing_to_asec": AcsTransferTargetBankStore( + tmp_path / "housing", + identity=identity, + ), + } + first = _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + target_banks=banks, + ) + survey_files = sorted((tmp_path / "survey" / "targets").glob("*.h5")) + housing_files = sorted((tmp_path / "housing" / "targets").glob("*.h5")) + assert len(survey_files) == 2 + assert len(housing_files) == 1 + + resumed_banks = { + "asec_survey_to_acs": AcsTransferTargetBankStore( + tmp_path / "survey", + identity=identity, + ), + "acs_housing_to_asec": AcsTransferTargetBankStore( + tmp_path / "housing", + identity=identity, + ), + } + second = _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + target_banks=resumed_banks, + ) + for column in ("unemployment_compensation", "is_disabled", "pre_subsidy_rent"): + pd.testing.assert_series_equal( + first.frame.table("person")[column], + second.frame.table("person")[column], + ) + survey_receipt = resumed_banks["asec_survey_to_acs"].receipt() + assert survey_receipt["targets"] + + +def test_clone_attachment_is_seeded_exact_and_pair_weighted() -> None: + stacked = _stacked_gap_fixture() + attached = clone_us_frame_for_puf_support( + stacked, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + ) + repeated = clone_us_frame_for_puf_support( + stacked, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + ) + changed_seed = clone_us_frame_for_puf_support( + stacked, + clone_attachment_fraction=0.5, + clone_attachment_seed=579, + ) + + manifest = attached.metadata[PUF_CLONE_ATTACHMENT_MANIFEST_KEY] + assert manifest["eligible_household_count"] == 14 + assert manifest["requested_household_count"] == 7 + assert manifest["realized_household_count"] == 7 + assert manifest["exact_count_rule"] == "floor(fraction * eligible)" + assert ( + manifest["selected_household_source_ids_sha256"] + == repeated.metadata[PUF_CLONE_ATTACHMENT_MANIFEST_KEY][ + "selected_household_source_ids_sha256" + ] + ) + assert ( + manifest["selected_household_source_ids_sha256"] + != changed_seed.metadata[PUF_CLONE_ATTACHMENT_MANIFEST_KEY][ + "selected_household_source_ids_sha256" + ] + ) + + household = attached.table("household") + clone_index = household[support_clone_index_column("household")] + assert int(clone_index.eq(0).sum()) == 14 + assert int(clone_index.eq(1).sum()) == 7 + weights = attached.weights_for("household").values + assert np.isclose( + float(weights.sum()), + float(stacked.weights_for("household").total), + rtol=1e-12, + ) + source_column = household.columns[household.columns.str.endswith("_source_id")][0] + attached_ids = set( + household.loc[clone_index.eq(1), source_column].astype(int).tolist() + ) + incoming = dict( + zip( + stacked.table("household")[source_column].astype(int).tolist(), + stacked.weights_for("household").values.tolist(), + strict=True, + ) + ) + for row, weight in zip(household.itertuples(index=False), weights, strict=True): + source_id = int(getattr(row, source_column)) + expected = ( + incoming[source_id] / 2.0 + if source_id in attached_ids + else incoming[source_id] + ) + assert np.isclose(weight, expected, rtol=1e-12) + + assert validate_puf_clone_attachment(attached, boundary="attachment fixture") + + +def test_clone_attachment_fraction_one_matches_full_clone() -> None: + stacked = _stacked_gap_fixture() + full = clone_us_frame_for_puf_support(stacked) + attached = clone_us_frame_for_puf_support( + stacked, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + + assert attached.schema == full.schema + assert attached.entities == full.entities + assert attached.links == full.links + assert attached.weighted_entities == full.weighted_entities + for entity in full.entities: + assert_frame_equal( + attached.table(entity), + full.table(entity), + check_exact=True, + ) + for link in full.links: + assert_frame_equal( + attached.link(link), + full.link(link), + check_exact=True, + ) + for entity in full.weighted_entities: + attached_weights = attached.weights_for(entity) + full_weights = full.weights_for(entity) + assert attached_weights.kind == full_weights.kind + assert attached_weights.values.dtype == full_weights.values.dtype + assert attached_weights.values.shape == full_weights.values.shape + assert attached_weights.values.tobytes( + order="C" + ) == full_weights.values.tobytes(order="C") + pd.testing.assert_series_equal(attached.strata, full.strata, check_exact=True) + assert attached.mass_log == full.mass_log + assert PUF_CLONE_ATTACHMENT_MANIFEST_KEY not in full.metadata + assert PUF_CLONE_ATTACHMENT_MANIFEST_KEY not in attached.metadata + assert attached.metadata == full.metadata + receipt = validate_puf_clone_attachment( + attached, + boundary="fraction-one identity fixture", + expected_fraction=1.0, + expected_seed=0, + ) + assert receipt["authority_form"] == "full_clone_identity_no_manifest" + assert receipt["eligible_household_count"] == 14 + assert receipt["realized_household_count"] == 14 + with pytest.raises(ValueError, match="clone attachment manifest.*is absent"): + validate_puf_clone_attachment( + attached, + boundary="fraction-one identity without declared expectation", + ) + + +def test_full_clone_identity_validation_fails_closed() -> None: + attached = clone_us_frame_for_puf_support( + _stacked_gap_fixture(), + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + tables = {entity: attached.table(entity) for entity in attached.entities} + tables.update({link: attached.link(link) for link in attached.links}) + weights = { + entity: attached.weights_for(entity) for entity in attached.weighted_entities + } + household_weights = attached.weights_for("household") + tampered_values = household_weights.values.copy() + household_clone = attached.table("household")[ + support_clone_index_column("household") + ] + first_detail = int(np.flatnonzero(household_clone.eq(1).to_numpy())[0]) + tampered_values[first_detail] += 1.0 + weights["household"] = Weights(tampered_values, household_weights.kind) + tampered = Frame( + tables, + attached.schema, + weights, + attached.strata, + mass_log=attached.mass_log, + metadata=attached.metadata, + ) + with pytest.raises(ValueError, match="full-clone identity failed"): + validate_puf_clone_attachment( + tampered, + boundary="tampered full clone", + expected_fraction=1.0, + expected_seed=0, + ) + + asymmetric_metadata = Frame( + tables, + attached.schema, + {entity: attached.weights_for(entity) for entity in attached.weighted_entities}, + attached.strata, + mass_log=attached.mass_log, + metadata={ + **attached.metadata, + PUF_CLONE_ATTACHMENT_MANIFEST_KEY: {"unexpected": True}, + }, + ) + with pytest.raises(ValueError, match="full-clone metadata symmetry failed"): + validate_puf_clone_attachment( + asymmetric_metadata, + boundary="metadata-asymmetric full clone", + expected_fraction=1.0, + expected_seed=0, + ) + + +def test_clone_attachment_configuration_fails_closed() -> None: + stacked = _stacked_gap_fixture() + with pytest.raises(ValueError, match="provided together"): + clone_us_frame_for_puf_support(stacked, clone_attachment_fraction=0.5) + with pytest.raises(ValueError, match="assembled frame"): + clone_us_frame_for_puf_support( + _asec_gap_source(), + clone_attachment_fraction=0.5, + clone_attachment_seed=1, + ) + with pytest.raises(ValueError, match="floors to zero"): + clone_us_frame_for_puf_support( + stacked, + clone_attachment_fraction=0.01, + clone_attachment_seed=1, + ) + + +def test_clone_attachment_manifest_mutation_fails_closed() -> None: + attached = clone_us_frame_for_puf_support( + _stacked_gap_fixture(), + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + ) + manifest = { + key: value + for key, value in attached.metadata[PUF_CLONE_ATTACHMENT_MANIFEST_KEY].items() + } + manifest["realized_household_count"] = 8 + manifest["requested_household_count"] = 8 + tampered = Frame( + {entity: attached.table(entity) for entity in attached.entities}, + attached.schema, + {entity: attached.weights_for(entity) for entity in attached.weighted_entities}, + attached.strata, + mass_log=attached.mass_log, + metadata={ + **attached.metadata, + PUF_CLONE_ATTACHMENT_MANIFEST_KEY: manifest, + }, + ) + with pytest.raises(ValueError, match="violates floor"): + validate_puf_clone_attachment(tampered, boundary="tampered attachment") + + +def test_run_stacked_puf_pass_imputes_only_the_attached_arm() -> None: + gap_filled = _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + ).frame + donor = pd.DataFrame( + { + "employment_income": [45_000.0, 8_000.0, 70_000.0, 22_000.0], + "taxable_interest_income": [120.0, 30.0, 900.0, 0.0], + "weight": [1.0, 1.0, 1.0, 1.0], + } + ) + result = run_stacked_puf_pass( + gap_filled, + donor, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + predictors=("puf_predictor_employment_income",), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=(), + seed=578, + n_estimators=10, + ) + + person = result.frame.table("person") + channel = person[support_channel_column("person")].astype(str) + clone_index = person[support_clone_index_column("person")] + assert person.loc[clone_index.eq(1), "taxable_interest_income"].notna().all() + assert ( + person.loc[clone_index.eq(0) & channel.eq("acs"), "taxable_interest_income"] + .isna() + .all() + ) + by_origin = result.receipt["recipient_person_rows_by_origin"] + assert set(by_origin) == {"asec", "acs"} + assert all(count > 0 for count in by_origin.values()) + assert result.receipt["doctrines"]["absent_cells"] == "preserve_nulls" + + with pytest.raises(ValueError, match="clone attachment"): + run_stacked_puf_pass( + result.frame, + donor, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + ) + + +def test_run_stacked_puf_pass_fraction_one_receipts_out_of_frame_identity() -> None: + gap_filled = _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + ).frame + donor = pd.DataFrame( + { + "employment_income": [45_000.0, 8_000.0, 70_000.0, 22_000.0], + "taxable_interest_income": [120.0, 30.0, 900.0, 0.0], + "weight": [1.0, 1.0, 1.0, 1.0], + } + ) + result = run_stacked_puf_pass( + gap_filled, + donor, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + predictors=("puf_predictor_employment_income",), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=(), + seed=578, + n_estimators=10, + ) + + assert PUF_CLONE_ATTACHMENT_MANIFEST_KEY not in result.frame.metadata + attachment = result.receipt["clone_attachment"] + assert attachment["authority_form"] == "full_clone_identity_no_manifest" + assert attachment["clone_attachment_fraction"] == 1.0 + assert attachment["clone_attachment_seed"] == 0 + assert attachment["eligible_household_count"] == 14 + assert attachment["realized_household_count"] == 14 + + +def _completed_stacked_frame() -> Frame: + """A stacked fixture whose declared surface is fully observed.""" + + return assemble_stacked_spine( + _asec_gap_source(), + _acs_gap_source(), + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame + + +def test_completeness_gate_passes_on_filled_and_proven_surface() -> None: + gap_filled = _gap_fill_with_test_authority( + _stacked_gap_fixture(), + plan=_GAP_FILL_TEST_PLAN, + seed=578, + n_estimators=10, + ).frame + surface = { + "person": { + "model_required_numeric": ("unemployment_compensation",), + "model_required_boolean": ("is_disabled",), + "housing": ("pre_subsidy_rent",), + } + } + result = _completeness_with_test_authority( + gap_filled, + declared_surface=surface, + declared_gap_fill_plan=_GAP_FILL_TEST_PLAN, + ) + assert result.passed + assert result.details["declared_targets"] == 3 + statuses = { + label: receipt["status"] for label, receipt in result.details["targets"].items() + } + assert set(statuses.values()) == {"complete"} + + +def test_completeness_gate_names_a_silently_missing_family() -> None: + """The run-7 catcher: a declared family with no columns fails by name.""" + + surface = { + "person": { + "puf_tax_itemization": ( + "taxable_interest_income", + "qualified_dividend_income", + ), + } + } + stacked = _stacked_gap_fixture() + tables = {entity: stacked.table(entity) for entity in stacked.entities} + person = tables["person"].drop(columns=["taxable_interest_income"]) + tables["person"] = person + dropped = Frame( + tables, + stacked.schema, + {entity: stacked.weights_for(entity) for entity in stacked.weighted_entities}, + stacked.strata, + mass_log=stacked.mass_log, + metadata=stacked.metadata, + ) + + result = _completeness_with_test_authority( + dropped, + declared_surface=surface, + declared_gap_fill_plan=_GAP_FILL_TEST_PLAN, + ) + assert not result.passed + missing = [ + failure + for failure in result.failures + if "puf_tax_itemization/taxable_interest_income" in failure + and "missing" in failure + ] + assert missing + unproven = [ + failure + for failure in result.failures + if "qualified_dividend_income" in failure and "authority proof" in failure + ] + assert unproven + + +def test_completeness_gate_requires_source_role_proofs_for_nulls() -> None: + stacked = _stacked_gap_fixture() + surface = {"person": {"puf_tax_itemization": ("taxable_interest_income",)}} + + unproven = _completeness_with_test_authority( + stacked, + declared_surface=surface, + declared_gap_fill_plan=(), + ) + assert not unproven.passed + assert any( + "acs/clone_0" in failure and "authority proof" in failure + for failure in unproven.failures + ) + + proven = _completeness_with_test_authority( + stacked, + declared_surface=surface, + declared_gap_fill_plan=(), + absence_proofs=( + AbsenceProof( + entity="person", + column="taxable_interest_income", + channel="acs", + clone_index=0, + reason="pending asec_survey_to_acs gap-fill", + ), + ), + ) + assert proven.passed + receipt = proven.details["targets"][ + "person/puf_tax_itemization/taxable_interest_income" + ] + assert receipt["status"] == "proven_absent" + assert receipt["proven"]["acs/clone_0"]["reason"] == ( + "pending asec_survey_to_acs gap-fill" + ) + + +def test_completeness_gate_rejects_wildcard_for_declared_donor_hole() -> None: + stacked = _stacked_gap_fixture() + surface = {"person": {"model_required_numeric": ("unemployment_compensation",)}} + wildcard = AbsenceProof( + entity="person", + column="unemployment_compensation", + channel="*", + clone_index=0, + reason="WILDCARD LAUNDER", + ) + + laundered = _completeness_with_test_authority( + stacked, + declared_surface=surface, + declared_gap_fill_plan=_GAP_FILL_TEST_PLAN, + absence_proofs=(wildcard,), + ) + assert not laundered.passed + assert any( + "origin-exact authority proof" in failure + and "asec_survey_to_acs" in failure + and "donor 'asec'" in failure + for failure in laundered.failures + ) + + exact = _completeness_with_test_authority( + stacked, + declared_surface=surface, + declared_gap_fill_plan=_GAP_FILL_TEST_PLAN, + absence_proofs=(replace(wildcard, channel="acs", reason="DECLARED EXACT"),), + ) + assert exact.passed + proof = exact.details["targets"][ + "person/model_required_numeric/unemployment_compensation" + ]["proven"]["acs/clone_0"] + assert proof["authority_form"] == "origin_exact_recipient" + assert proof["declared_direction"] == "asec_survey_to_acs" + assert proof["declared_donor_channel"] == "asec" + + +def test_completeness_gate_empty_plan_cannot_launder_canonical_target() -> None: + surface = {"person": {"model_required_numeric": ("unemployment_compensation",)}} + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface=surface, + gap_fill_plan=(), + ) + result = stacked_spine_module._stacked_completeness_gate_with_test_authority( + _stacked_gap_fixture(), + authority=authority, + absence_proofs=( + AbsenceProof( + entity="person", + column="unemployment_compensation", + channel="*", + clone_index=0, + reason="EMPTY PLAN LAUNDER", + ), + ), + ) + + assert not result.passed + assert any( + "person/model_required_numeric/unemployment_compensation" in failure + and "canonical gap-fill plan" in failure + and "recipient 'acs'" in failure + and "donor 'asec'" in failure + and "wildcard authority is forbidden" in failure + for failure in result.failures + ) + + +def test_completeness_gate_empty_surface_is_terminal() -> None: + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface={}, + gap_fill_plan=(), + metric_registry={}, + ) + result = stacked_spine_module._stacked_completeness_gate_with_test_authority( + _stacked_gap_fixture(), + authority=authority, + ) + + assert not result.passed + assert result.details["declared_targets"] == 0 + assert any( + "declared stacked surface contains zero targets" in failure + for failure in result.failures + ) + + +def test_completeness_gate_rejects_origin_exact_proof_for_declared_donor() -> None: + stacked = _stacked_gap_fixture() + person = stacked.table("person").copy() + channel = person[support_channel_column("person")].astype(str) + donor_index = person.index[channel.eq("asec")][0] + person.loc[donor_index, "unemployment_compensation"] = np.nan + tables = {entity: stacked.table(entity) for entity in stacked.entities} + tables["person"] = person + donor_hole = Frame( + tables, + stacked.schema, + {entity: stacked.weights_for(entity) for entity in stacked.weighted_entities}, + stacked.strata, + mass_log=stacked.mass_log, + metadata=stacked.metadata, + ) + surface = {"person": {"model_required_numeric": ("unemployment_compensation",)}} + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface=surface, + gap_fill_plan=_GAP_FILL_TEST_PLAN, + ) + result = stacked_spine_module._stacked_completeness_gate_with_test_authority( + donor_hole, + authority=authority, + absence_proofs=( + AbsenceProof( + entity="person", + column="unemployment_compensation", + channel="asec", + clone_index=0, + reason="DONOR-ORIGIN LAUNDER", + ), + AbsenceProof( + entity="person", + column="unemployment_compensation", + channel="acs", + clone_index=0, + reason="DECLARED RECIPIENT", + ), + ), + ) + + assert not result.passed + assert any( + "person/model_required_numeric/unemployment_compensation" in failure + and "origin-exact authority proof is valid only for declared recipient 'acs'" + in failure + and "declared donor 'asec'" in failure + for failure in result.failures + ) + target = result.details["targets"][ + "person/model_required_numeric/unemployment_compensation" + ] + assert "asec/clone_0" not in target["proven"] + + +def test_completeness_receipts_bind_live_authority_per_target() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = stacked_completeness_gate(frame) + assert canonical.passed, canonical.failures + assert canonical.details["declared_targets"] == 90 + authority = canonical.details["authority"] + assert authority["authority_form"] == "CANONICAL" + assert authority["canonical"] is True + battery = by_origin_battery(frame) + assert battery.passed, battery.failures + assert battery.details["authority"] == authority + canonical_manifest = GateReport((canonical, battery)).to_manifest() + assert canonical_manifest["passed"] is True + assert ( + stacked_spine_module._authority_receipt( + stacked_spine_module._production_stacked_authority() + ) + == authority + ) + plan_sha256 = authority["components"]["gap_fill_plan"]["sha256"] + surface_sha256 = authority["components"]["declared_surface"]["sha256"] + for receipt in canonical.details["targets"].values(): + assert receipt["authority_form"] == "observed_complete" + assert receipt["plan_sha256"] == plan_sha256 + assert receipt["surface_sha256"] == surface_sha256 + + stacked = _stacked_gap_fixture() + person = stacked.table("person").copy() + person["test_unplanned_absence"] = np.nan + tables = {entity: stacked.table(entity) for entity in stacked.entities} + tables["person"] = person + custom_frame = Frame( + tables, + stacked.schema, + {entity: stacked.weights_for(entity) for entity in stacked.weighted_entities}, + stacked.strata, + mass_log=stacked.mass_log, + metadata=stacked.metadata, + ) + custom_authority = stacked_spine_module._make_test_stacked_authority( + declared_surface={"person": {"test_only": ("test_unplanned_absence",)}}, + gap_fill_plan=(), + ) + custom = stacked_spine_module._stacked_completeness_gate_with_test_authority( + custom_frame, + authority=custom_authority, + absence_proofs=( + AbsenceProof( + entity="person", + column="test_unplanned_absence", + channel="*", + clone_index=0, + reason="test-only unplanned surface", + ), + ), + ) + assert custom.passed, custom.failures + custom_top = custom.details["authority"] + assert custom_top["authority_form"] == "NON-CANONICAL" + custom_target = custom.details["targets"]["person/test_only/test_unplanned_absence"] + assert custom_target["authority_form"] == "wildcard_no_declared_donor_plan" + assert ( + custom_target["plan_sha256"] + == custom_top["components"]["gap_fill_plan"]["sha256"] + ) + assert ( + custom_target["surface_sha256"] + == custom_top["components"]["declared_surface"]["sha256"] + ) + with pytest.raises( + ValueError, + match="non-canonical stacked authority is forbidden", + ): + stacked_spine_module._validate_production_authority_receipt( + custom_top, + boundary="test production artifact", + ) + with pytest.raises(ValueError, match="production manifest emission is forbidden"): + GateReport((custom,)).to_manifest() + custom.details["authority"]["production_manifest_permitted"] = True + with pytest.raises(ValueError, match="production manifest emission is forbidden"): + GateReport((custom,)).to_manifest() + forged_receipt = deepcopy(custom_top) + forged_receipt.update( + { + "authority_id": "us_stacked_spine_authority", + "authority_form": "CANONICAL", + "declared_authority_form": "CANONICAL", + "canonical": True, + "canonical_identity": True, + "canonical_content": True, + "integrity_valid": True, + "digest_matches_declared": True, + "production_manifest_permitted": True, + "declared_sha256": forged_receipt["sha256"], + } + ) + for component in forged_receipt["components"].values(): + component["declared_sha256"] = component["sha256"] + component["digest_matches_declared"] = True + forged_result = replace(custom, details={"authority": forged_receipt}) + with pytest.raises(ValueError, match="production manifest emission is forbidden"): + GateReport((forged_result,)).to_manifest() + + +def test_self_digested_partial_authority_cannot_forge_production_identity() -> None: + surface = {"person": {"test_only": ("unemployment_compensation",)}} + forged = stacked_spine_module._make_stacked_authority( + authority_id="us_stacked_spine_authority", + version=1, + gap_fill_plan=(), + declared_surface=surface, + metric_registry={ + ("person", "test_only", "unemployment_compensation", 0): ( + "monetary_sign_separated" + ) + }, + support_profile=(stacked_spine_module.CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE), + declared_form="CANONICAL", + ) + result = stacked_spine_module._stacked_completeness_gate_evaluate( + _stacked_gap_fixture(), + authority=forged, + production=True, + absence_proofs=(), + ) + + assert not result.passed + assert result.details["authority"]["canonical_identity"] is False + assert result.details["authority"]["canonical_content"] is False + assert any( + "canonical stacked authority identity mismatch" in failure + for failure in result.failures + ) + with pytest.raises(ValueError, match="production manifest emission is forbidden"): + GateReport((result,)).to_manifest() + + +def test_rebound_anchor_aliases_cannot_replace_captured_canonical_authority( + monkeypatch: pytest.MonkeyPatch, +) -> None: + surface = {"person": {"test_only": ("unemployment_compensation",)}} + forged = stacked_spine_module._make_stacked_authority( + authority_id="us_stacked_spine_authority", + version=1, + gap_fill_plan=(), + declared_surface=surface, + metric_registry={ + ("person", "test_only", "unemployment_compensation", 0): ( + "monetary_sign_separated" + ) + }, + support_profile=(stacked_spine_module.CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE), + declared_form="CANONICAL", + ) + rebound = { + "_CANONICAL_STACKED_DECLARED_SURFACE_ANCHOR": forged.declared_surface, + "_CANONICAL_STACKED_GAP_FILL_PLAN_ANCHOR": forged.gap_fill_plan, + "_CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY_ANCHOR": forged.metric_registry, + "_CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE_ANCHOR": (forged.support_profile), + "_CANONICAL_STACKED_AUTHORITY_ANCHOR": forged, + "_STACKED_DECLARED_SURFACE": forged.declared_surface, + "_STACKED_GAP_FILL_PLAN": forged.gap_fill_plan, + "_BATTERY_METRIC_REGISTRY": forged.metric_registry, + "_BATTERY_SUPPORT_PROFILE": forged.support_profile, + } + for name, value in rebound.items(): + monkeypatch.setattr(stacked_spine_module, name, value) + + result = stacked_completeness_gate(_stacked_gap_fixture()) + + assert not result.passed + assert result.details["authority"]["canonical"] is False + assert result.details["authority"]["canonical_identity"] is False + assert any( + "canonical stacked authority identity mismatch" in failure + for failure in result.failures + ) + + +@pytest.mark.parametrize( + "mutation", + ( + "support_threshold", + "surface_count", + "direction_count", + "target_binding", + ), +) +def test_stacked_manifest_rejects_pre_emission_nested_receipt_mutation( + mutation: str, +) -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + result = stacked_completeness_gate(frame) + assert result.passed, result.failures + authority = result.details["authority"] + if mutation == "support_threshold": + authority["components"]["support_profile"]["min_effective_support"] = 50 + elif mutation == "surface_count": + authority["components"]["declared_surface"]["target_count"] = 0 + elif mutation == "direction_count": + authority["components"]["gap_fill_plan"]["direction_count"] = 0 + else: + target = next(iter(result.details["targets"].values())) + target["authority_form"] = "wildcard_no_declared_donor_plan" + target["plan_sha256"] = "0" * 64 + + with pytest.raises( + ValueError, + match="details changed after evaluation.*manifest emission is forbidden", + ): + GateReport((result,)).to_manifest() + + +@pytest.mark.parametrize("replacement", ({}, None)) +def test_stacked_manifest_requires_the_authority_receipt( + replacement: object, +) -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = stacked_completeness_gate(frame) + details = deepcopy(dict(canonical.details)) + if replacement is None: + details["authority"] = None + else: + details.pop("authority") + missing = replace(canonical, details=details) + + with pytest.raises( + ValueError, + match="no stacked authority receipt.*manifest emission is forbidden", + ): + GateReport((missing,)).to_manifest() + + +def test_fresh_gate_result_cannot_graft_canonical_authority_onto_test_surface() -> None: + frame = _stacked_gap_fixture() + person = frame.table("person").copy() + person["test_unplanned_absence"] = np.nan + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + custom_frame = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + test_authority = stacked_spine_module._make_test_stacked_authority( + declared_surface={"person": {"test_only": ("test_unplanned_absence",)}}, + gap_fill_plan=(), + ) + custom = stacked_spine_module._stacked_completeness_gate_with_test_authority( + custom_frame, + authority=test_authority, + absence_proofs=( + AbsenceProof( + entity="person", + column="test_unplanned_absence", + channel="*", + clone_index=0, + reason="test-only wildcard", + ), + ), + ) + canonical = stacked_completeness_gate( + _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + ) + grafted_details = deepcopy(dict(custom.details)) + grafted_details["authority"] = deepcopy(canonical.details["authority"]) + grafted = replace(custom, details=grafted_details) + + with pytest.raises( + ValueError, + match="must declare exactly 90 targets.*manifest emission is forbidden", + ): + GateReport((grafted,)).to_manifest() + + +def test_fresh_gate_result_cannot_forge_a_donor_origin_proof() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = stacked_completeness_gate(frame) + details = deepcopy(dict(canonical.details)) + authority = details["authority"] + binding = { + "authority_sha256": authority["sha256"], + "plan_sha256": authority["components"]["gap_fill_plan"]["sha256"], + "surface_sha256": authority["components"]["declared_surface"]["sha256"], + } + label = "person/puf_tax_itemization/taxable_interest_income" + details["targets"][label] = { + "status": "proven_absent", + "null_rows": 1, + "authority_form": "origin_exact_recipient", + **binding, + "proven": { + "asec/clone_0": { + "null_rows": 1, + "reason": "DONOR-ORIGIN FORGERY", + "authority_form": "origin_exact_recipient", + **binding, + "declared_direction": "asec_survey_to_acs", + "declared_donor_channel": "asec", + "declared_recipient_channel": "acs", + } + }, + "unproven": {}, + } + forged = replace(canonical, details=details) + + with pytest.raises( + ValueError, + match="asec/clone_0 proof is not recipient-exact.*emission is forbidden", + ): + GateReport((forged,)).to_manifest() + + +def test_fresh_battery_result_cannot_forge_canonical_coverage_receipts() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = by_origin_battery(frame) + details = deepcopy(dict(canonical.details)) + details["declared_target_count"] = 1 + details["registered_target_count"] = 1 + details["comparisons"] = { + "person/test_only/fake[clone_0]": { + "status": "tested", + "metric": "rare_incidence", + } + } + forged = replace(canonical, details=details) + + with pytest.raises( + ValueError, + match="coverage receipt must bind all 90 targets.*emission is forbidden", + ): + GateReport((forged,)).to_manifest() + + +def test_fresh_battery_result_cannot_relabel_a_canonical_metric() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = by_origin_battery(frame) + details = deepcopy(dict(canonical.details)) + label = "person/puf_tax_itemization/taxable_interest_income[clone_0]" + details["comparisons"][label]["metric"] = "rare_incidence" + forged = replace(canonical, details=details) + + with pytest.raises( + ValueError, + match=( + "taxable_interest_income.*canonical metric " + "'monetary_sign_separated'.*emission is forbidden" + ), + ): + GateReport((forged,)).to_manifest() + + +def test_noncanonical_stacked_receipt_cannot_escape_under_a_renamed_gate() -> None: + authority = stacked_spine_module._make_test_stacked_authority( + declared_surface={"person": {"test_only": ("unemployment_compensation",)}}, + gap_fill_plan=(), + ) + custom = stacked_spine_module._stacked_completeness_gate_with_test_authority( + _stacked_gap_fixture(), + authority=authority, + ) + renamed = replace(custom, name="renamed_stacked_completeness") + + with pytest.raises( + ValueError, + match="unrecognized gate name.*manifest emission is forbidden", + ): + GateReport((renamed,)).to_manifest() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("authority_form", "NON-CANONICAL"), + ("declared_authority_form", "NON-CANONICAL"), + ("digest_matches_declared", False), + ), +) +def test_stripped_noncanonical_receipt_cannot_escape_under_a_renamed_gate( + field: str, + value: object, +) -> None: + stripped = GateResult( + name="renamed_stacked_completeness", + passed=True, + details={"authority": {field: value}}, + ) + + with pytest.raises( + ValueError, + match="unrecognized gate name.*manifest emission is forbidden", + ): + GateReport((stripped,)).to_manifest() + + +def test_completeness_gate_wildcard_proof_covers_every_origin() -> None: + attached = clone_us_frame_for_puf_support( + _stacked_gap_fixture(), + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + person = attached.table("person").copy() + person["health_savings_account_ald_person_carrier"] = np.nan + clone_mask = person[support_clone_index_column("person")].eq(1) + person.loc[clone_mask, "health_savings_account_ald_person_carrier"] = 100.0 + tables = {entity: attached.table(entity) for entity in attached.entities} + tables["person"] = person + frame = Frame( + tables, + attached.schema, + {entity: attached.weights_for(entity) for entity in attached.weighted_entities}, + attached.strata, + mass_log=attached.mass_log, + metadata=attached.metadata, + ) + surface = { + "person": { + "puf_tax_itemization": ("health_savings_account_ald_person_carrier",) + } + } + + unproven = _completeness_with_test_authority( + frame, + declared_surface=surface, + declared_gap_fill_plan=(), + ) + assert not unproven.passed + + proven = _completeness_with_test_authority( + frame, + declared_surface=surface, + declared_gap_fill_plan=(), + absence_proofs=( + AbsenceProof( + entity="person", + column="health_savings_account_ald_person_carrier", + channel="*", + clone_index=0, + reason="base arm carries no PUF detail; engine default applies", + ), + ), + ) + assert proven.passed + wildcard_receipt = proven.details["targets"][ + "person/puf_tax_itemization/health_savings_account_ald_person_carrier" + ]["proven"]["asec/clone_0"] + assert wildcard_receipt["authority_form"] == "wildcard_no_declared_donor_plan" + + +def _declared_battery_metric(entity: str, family: str, target: str) -> str: + canonical = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY.get( + (entity, family, target, 0) + ) + if canonical is not None: + return canonical + if family in {"benefit_participation", "model_required_boolean"}: + return "boolean_incidence" + if family == "model_required_discrete": + return "categorical_tvd" + return "monetary_sign_separated" + + +def _complete_battery_registry( + *extras: OriginBatterySpec, +) -> tuple[OriginBatterySpec, ...]: + metrics: dict[tuple[str, str, int], dict[str, str]] = {} + for direction in stacked_gap_fill_plan(): + for entity, families in direction.target_families.items(): + for family, targets in families.items(): + bucket = metrics.setdefault((entity, family, 0), {}) + bucket.update( + { + target: _declared_battery_metric(entity, family, target) + for target in targets + } + ) + for spec in extras: + metrics.setdefault((spec.entity, spec.family, spec.clone_index), {}).update( + spec.column_metrics + ) + return tuple( + OriginBatterySpec( + entity=entity, + family=family, + clone_index=clone_index, + column_metrics=column_metrics, + ) + for (entity, family, clone_index), column_metrics in sorted(metrics.items()) + ) + + +def _with_declared_battery_defaults( + frame: Frame, + *, + preserve: frozenset[tuple[str, str]] = frozenset(), +) -> Frame: + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for direction in stacked_gap_fill_plan(): + for entity, families in direction.target_families.items(): + for targets in families.values(): + for target in targets: + if (entity, target) not in preserve: + tables[entity][target] = 1.0 + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _battery_frame(columns: dict[str, tuple[np.ndarray, np.ndarray]]) -> Frame: + """A stacked frame with hand-set asec/acs person columns. + + ``columns`` maps a column name to its (asec values, acs values) pair. + """ + + first_asec, first_acs = next(iter(columns.values())) + asec_count = len(first_asec) + acs_count = len(first_acs) + asec = _source_frame( + household_ids=list(range(11, 11 + asec_count)), + weights=[100.0] * asec_count, + stratum="asec_2024", + ) + acs = _source_frame( + household_ids=list(range(101, 101 + acs_count)), + weights=[100.0] * acs_count, + stratum="acs_2024_1yr", + ) + + def with_columns(frame: Frame, position: int) -> Frame: + frame = _with_declared_battery_defaults(frame) + person = frame.table("person").copy() + for column, values in columns.items(): + person[column] = values[position] + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + return assemble_stacked_spine( + with_columns(asec, 0), + with_columns(acs, 1), + acs_sample_fraction=1.0, + acs_sample_seed=0, + ).frame + + +def test_battery_boolean_incidence_is_declared_not_dispatched() -> None: + frame = _battery_frame( + { + "matched_flag": ( + np.asarray([1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0]), + np.asarray([1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0]), + ), + "object_backed_flag": ( + np.asarray( + [True, True, True, False, False, False, False, False], dtype=object + ), + np.asarray([True] + [False] * 10, dtype=object), + ), + } + ) + registry = ( + OriginBatterySpec( + entity="person", + family="model_required_boolean", + column_metrics={ + "matched_flag": "boolean_incidence", + "object_backed_flag": "boolean_incidence", + }, + ), + ) + result = _battery_with_test_authority( + frame, registry=_complete_battery_registry(*registry) + ) + + assert not result.passed + matched = result.details["comparisons"][ + "person/model_required_boolean/matched_flag[clone_0]" + ] + assert matched["status"] == "tested" + assert not any("matched_flag" in failure for failure in result.failures) + assert any( + "object_backed_flag" in failure and "incidence ratio" in failure + for failure in result.failures + ) + + +def test_battery_rejects_registry_omitting_declared_champva_before_comparisons() -> ( + None +): + champva = "has_champva_health_coverage_at_interview" + frame = _battery_frame( + { + champva: (np.ones(8), np.zeros(11)), + "healthy_control": ( + np.asarray([1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0]), + np.asarray([1.0] * 7 + [0.0] * 4), + ), + } + ) + result = _battery_with_test_authority( + frame, + registry=( + OriginBatterySpec( + entity="person", + family="healthy_control", + column_metrics={"healthy_control": "boolean_incidence"}, + ), + ), + ) + + assert not result.passed + assert result.details["tested_comparisons"] == 0 + assert any( + f"missing declared battery target person/model_required_boolean/{champva}" + in failure + for failure in result.failures + ) + assert any( + champva in target for target in result.details["missing_declared_targets"] + ) + + +def test_battery_taxable_interest_metric_cannot_be_relabelled_rare_incidence() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.arange(1.0, 9.0), + np.arange(101.0, 112.0), + ) + } + ) + registry = dict(stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY) + target = ("person", "puf_tax_itemization", "taxable_interest_income", 0) + registry[target] = "rare_incidence" + authority = stacked_spine_module._make_test_stacked_authority( + metric_registry=registry, + ) + result = stacked_spine_module._by_origin_battery_with_test_authority( + frame, + authority=authority, + ) + + assert not result.passed + assert result.details["tested_comparisons"] == 0 + metric_receipt = result.details["authority"]["components"]["metric_registry"] + assert metric_receipt["target_count"] == 90 + assert any( + "person/puf_tax_itemization/taxable_interest_income[clone_0]" in failure + and "authoritative metric 'monetary_sign_separated'" in failure + and "got 'rare_incidence'" in failure + for failure in result.failures + ) + + +def test_gap_fill_plan_digest_binds_direction_and_channels() -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + canonical = stacked_completeness_gate(frame) + canonical_sha256 = canonical.details["authority"]["components"]["gap_fill_plan"][ + "sha256" + ] + plan = stacked_spine_module.CANONICAL_STACKED_GAP_FILL_PLAN + altered = ( + replace( + plan[0], + name="rerouted_gap_fill", + recipient_channel="asec", + donor_channel="acs", + ), + *plan[1:], + ) + authority = stacked_spine_module._make_test_stacked_authority( + gap_fill_plan=altered, + ) + result = stacked_spine_module._stacked_completeness_gate_with_test_authority( + frame, + authority=authority, + ) + + assert not result.passed + receipt = result.details["authority"] + assert receipt["authority_form"] == "NON-CANONICAL" + assert receipt["components"]["gap_fill_plan"]["sha256"] != canonical_sha256 + assert any("canonical gap-fill direction mismatch" in f for f in result.failures) + + +def test_battery_sign_separated_catches_the_one_sided_hole() -> None: + """The run-7 signature: ample support, hollow ACS leg -> terminal.""" + + frame = _battery_frame( + { + "sentinel_amount": ( + np.asarray([500.0, 0.0, 1_200.0, 0.0, 800.0, 0.0, 2_000.0, 650.0]), + np.zeros(11), + ), + } + ) + registry = ( + OriginBatterySpec( + entity="person", + family="puf_tax_itemization", + column_metrics={"sentinel_amount": "monetary_sign_separated"}, + ), + ) + result = _battery_with_test_authority( + frame, registry=_complete_battery_registry(*registry) + ) + + assert not result.passed + assert any( + "sentinel_amount" in failure and "positive-leg incidence ratio" in failure + for failure in result.failures + ) + + +def test_battery_sign_separated_passes_matching_legs() -> None: + frame = _battery_frame( + { + "signed_amount": ( + np.asarray( + [ + 100.0, + -100.0, + 200.0, + -200.0, + 300.0, + -300.0, + 400.0, + -400.0, + 500.0, + -500.0, + 600.0, + -600.0, + ] + ), + np.asarray( + [ + 105.0, + -105.0, + 210.0, + -210.0, + 315.0, + -315.0, + 420.0, + -420.0, + 525.0, + -525.0, + 630.0, + -630.0, + ] + ), + ), + } + ) + registry = ( + OriginBatterySpec( + entity="person", + family="puf_tax_itemization", + column_metrics={"signed_amount": "monetary_sign_separated"}, + ), + ) + result = _battery_with_test_authority( + frame, registry=_complete_battery_registry(*registry) + ) + assert result.passed, result.failures + record = result.details["comparisons"][ + "person/puf_tax_itemization/signed_amount[clone_0]" + ] + assert record["legs"]["positive"]["quantile_envelope_distance"] <= 0.25 + assert record["legs"]["negative"]["quantile_envelope_distance"] <= 0.25 + + +def test_battery_support_awareness_and_dead_comparisons() -> None: + frame = _battery_frame( + { + "rare_flag": ( + np.asarray([0.0] * 8), + np.asarray([0.0] * 11), + ), + } + ) + dead = _battery_with_test_authority( + frame, + registry=_complete_battery_registry( + OriginBatterySpec( + entity="person", + family="take_up", + column_metrics={"rare_flag": "rare_incidence"}, + ), + ), + ) + assert not dead.passed + assert any("dead" in failure for failure in dead.failures) + assert dead.details["support_profile"]["profile_id"] == ( + "us_stacked_origin_battery_support" + ) + assert dead.details["support_profile"]["min_effective_support"] == 5 + assert len(dead.details["support_profile"]["sha256"]) == 64 + + +def test_battery_rebound_support_profile_with_stale_digest_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frame = _battery_frame( + { + "taxable_interest_income": ( + np.asarray([100.0] * 8), + np.asarray([100.0] * 11), + ) + } + ) + rebound = replace( + stacked_spine_module.CANONICAL_ORIGIN_BATTERY_SUPPORT_PROFILE, + min_effective_support=50, + ) + monkeypatch.setattr(stacked_spine_module, "_BATTERY_SUPPORT_PROFILE", rebound) + result = by_origin_battery(frame) + + assert not result.passed + assert result.details["tested_comparisons"] == 0 + assert any( + "support profile live-content digest mismatch" in failure + for failure in result.failures + ) + profile = result.details["authority"]["components"]["support_profile"] + assert profile["sha256"] == ( + "7ffd25d0bc4c7cca1a12b61171d8d433094a60fc56cf5a099564598841252af9" + ) + assert profile["sha256"] != profile["declared_sha256"] + + +def test_battery_rejects_caller_controlled_support_threshold() -> None: + with pytest.raises(TypeError, match="min_effective_support"): + OriginBatterySpec( + entity="person", + family="take_up", + column_metrics={"rare_flag": "rare_incidence"}, + min_effective_support=50, + ) + + +def test_battery_categorical_tvd_and_null_scope() -> None: + frame = _battery_frame( + { + "category_field": ( + np.asarray(["A", "A", "A", "B", "B", "B", "A", "B"], dtype=object), + np.asarray( + ["A", "B", "A", "B", "A", "B", "A", "B", "A", "B", "A"], + dtype=object, + ), + ), + "leaky_field": ( + np.asarray([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]), + np.asarray([1.0, np.nan, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0]), + ), + } + ) + result = _battery_with_test_authority( + frame, + registry=_complete_battery_registry( + OriginBatterySpec( + entity="person", + family="model_required_discrete", + column_metrics={ + "category_field": "categorical_tvd", + "leaky_field": "monetary_sign_separated", + }, + ) + ), + ) + assert not result.passed + assert any( + "leaky_field" in failure and "null value(s)" in failure + for failure in result.failures + ) + category = result.details["comparisons"][ + "person/model_required_discrete/category_field[clone_0]" + ] + assert category["total_variation_distance"] <= 0.25 + assert not any("category_field" in failure for failure in result.failures) + + +def test_battery_spec_rejects_undeclared_metric_kinds() -> None: + with pytest.raises(ValueError, match="unknown metric kind"): + OriginBatterySpec( + entity="person", + family="take_up", + column_metrics={"anything": "dtype_dispatch"}, + ) + + +def test_pilot_configuration_is_the_ratified_ten_percent() -> None: + assert STACKED_PILOT_ACS_SAMPLE_FRACTION == 0.10 + assert STACKED_PILOT_ACS_SAMPLE_SEED == 578 + + +def _asec_e2e_source() -> Frame: + frame = _source_frame( + household_ids=list(range(1_001, 1_041)), + weights=[100.0] * 40, + stratum="asec_2024", + ) + person = frame.table("person").copy() + count = len(person) + index = np.arange(count) + person["is_female"] = index % 2 == 0 + person["is_household_head"] = True + person["employment_income_before_lsr"] = 20_000.0 + 1_500.0 * index + person["unemployment_compensation"] = np.where(index % 4 == 0, 2_400.0, 0.0) + # Structurally learnable from a REQUIRED predictor with a stable share + # under any household subsample, so the gap-fill QRF reproduces the + # incidence on the seeded ACS sample without sampling-skew noise. + person["is_disabled"] = person["is_female"].to_numpy() + interest = np.where(index % 2 == 0, 1_200.0 + 40.0 * index, 0.0) + person["taxable_interest_income"] = interest + for column in ( + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "rental_income", + "estate_income", + ): + person[column] = 0.0 + household = frame.table("household").copy() + household["tenure_type"] = pd.Series( + ["RENTED" if position % 2 else "OWNED_WITH_MORTGAGE" for position in range(40)], + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + tables["household"] = household + tax_unit = tables["tax_unit"].copy() + tax_unit["health_savings_account_ald"] = np.where( + np.arange(len(tax_unit)) % 3 == 0, + 750.0, + 0.0, + ) + tables["tax_unit"] = tax_unit + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +def _acs_e2e_source() -> Frame: + frame = _source_frame( + household_ids=list(range(5_001, 5_041)), + weights=[100.0] * 40, + stratum="acs_2024_1yr", + ) + person = frame.table("person").copy() + count = len(person) + index = np.arange(count) + person["is_female"] = index % 2 == 1 + person["is_household_head"] = True + person["employment_income_before_lsr"] = 21_000.0 + 1_450.0 * index + person["acs_interest_dividend_rental_income"] = np.where( + index % 2 == 0, 1_250.0 + 42.0 * index, 0.0 + ) + person["pre_subsidy_rent"] = np.where(index % 2 == 1, 11_000.0 + 150.0 * index, 0.0) + household = frame.table("household").copy() + household["tenure_type"] = pd.Series( + ["RENTED" if position % 2 else "OWNED_OUTRIGHT" for position in range(40)], + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + tables["household"] = household + tax_unit = tables["tax_unit"].copy() + tax_unit["health_savings_account_ald"] = np.nan + tables["tax_unit"] = tax_unit + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +_E2E_GAP_FILL_PLAN = ( + GapFillDirection( + name="asec_survey_to_acs", + recipient_channel="acs", + donor_channel="asec", + target_families={ + "person": { + "puf_tax_itemization": ("taxable_interest_income",), + "model_required_numeric": ("unemployment_compensation",), + "model_required_boolean": ("is_disabled",), + }, + "tax_unit": { + "puf_tax_itemization": ("health_savings_account_ald",), + }, + }, + ), + GapFillDirection( + name="acs_housing_to_asec", + recipient_channel="asec", + donor_channel="acs", + target_families={"person": {"housing": ("pre_subsidy_rent",)}}, + ), +) + + +def test_end_to_end_stack_gap_fill_puf_pass_gates_and_battery(tmp_path) -> None: + """The pilot pipeline end to end: the ACS tax-detail hole is closed. + + Run 7's failure signature was a hollow ACS income surface: the ACS spine + carried ~zero taxable-interest incidence against ASEC's 45% because the + PUF family silently skipped. This walks the revised architecture at + fixture scale — stack, banked cross-origin gap-fill with native ACS + predictors, seeded clone attachment, one doctrine-mode PUF pass, the + completeness gate, and the terminal by-origin battery — and proves the + sentinel is healthy on ACS-origin rows. + """ + + stacked = assemble_stacked_spine( + _asec_e2e_source(), + _acs_e2e_source(), + acs_sample_fraction=0.5, + acs_sample_seed=578, + ).frame + + banks = { + direction.name: AcsTransferTargetBankStore( + tmp_path / direction.name, + identity={"lane": "stacked-e2e", "direction": direction.name}, + ) + for direction in _E2E_GAP_FILL_PLAN + } + gap_filled = _gap_fill_with_test_authority( + stacked, + plan=_E2E_GAP_FILL_PLAN, + seed=578, + n_estimators=12, + target_banks=banks, + ) + + donor = pd.DataFrame( + { + "employment_income": 25_000.0 + 6_000.0 * np.arange(8), + "taxable_interest_income": [ + 1_300.0, + 0.0, + 1_500.0, + 1_800.0, + 0.0, + 2_100.0, + 1_650.0, + 1_950.0, + ], + "health_savings_account_ald": [ + 500.0, + 0.0, + 750.0, + 1_000.0, + 250.0, + 0.0, + 800.0, + 600.0, + ], + "weight": [1.0] * 8, + } + ) + passed = run_stacked_puf_pass( + gap_filled.frame, + donor, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + predictors=( + "puf_predictor_employment_income", + "puf_predictor_taxable_interest_income", + ), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + seed=578, + n_estimators=12, + ) + + # The audit's failure mode is now a named terminal error, not a silent + # zero-fill: without gap-fill the strict doctrine refuses the PUF pass. + with pytest.raises(ValueError, match="puf_predictor_taxable_interest_income"): + run_stacked_puf_pass( + stacked, + donor, + clone_attachment_fraction=0.5, + clone_attachment_seed=578, + predictors=( + "puf_predictor_employment_income", + "puf_predictor_taxable_interest_income", + ), + person_outputs=("taxable_interest_income",), + tax_unit_outputs=("health_savings_account_ald",), + seed=578, + n_estimators=12, + ) + + declared_surface = { + "person": { + "puf_tax_itemization": ("taxable_interest_income",), + "model_required_numeric": ("unemployment_compensation",), + "model_required_boolean": ("is_disabled",), + "housing": ("pre_subsidy_rent",), + }, + "tax_unit": { + "puf_tax_itemization": ("health_savings_account_ald",), + }, + } + completeness = _completeness_with_test_authority( + passed.frame, + declared_surface=declared_surface, + declared_gap_fill_plan=_E2E_GAP_FILL_PLAN, + ) + assert completeness.passed, completeness.failures + + # Drop-a-family mutation: the gate names the vanished family. + mutated_surface = { + "person": { + **declared_surface["person"], + "puf_tax_itemization": ( + "taxable_interest_income", + "salt_refund_income", + ), + }, + "tax_unit": declared_surface["tax_unit"], + } + mutated = _completeness_with_test_authority( + passed.frame, + declared_surface=mutated_surface, + declared_gap_fill_plan=_E2E_GAP_FILL_PLAN, + ) + assert not mutated.passed + assert any( + "salt_refund_income" in failure and "missing" in failure + for failure in mutated.failures + ) + + registry = ( + OriginBatterySpec( + entity="person", + family="puf_tax_itemization", + column_metrics={"taxable_interest_income": "monetary_sign_separated"}, + ), + OriginBatterySpec( + entity="person", + family="model_required_numeric", + column_metrics={"unemployment_compensation": "monetary_sign_separated"}, + ), + OriginBatterySpec( + entity="person", + family="model_required_boolean", + column_metrics={"is_disabled": "boolean_incidence"}, + ), + OriginBatterySpec( + entity="person", + family="housing", + column_metrics={"pre_subsidy_rent": "monetary_sign_separated"}, + ), + ) + battery = _battery_with_test_authority( + _with_declared_battery_defaults( + passed.frame, + preserve=frozenset( + { + ("person", "taxable_interest_income"), + ("person", "unemployment_compensation"), + ("person", "is_disabled"), + ("person", "pre_subsidy_rent"), + } + ), + ), + registry=_complete_battery_registry(*registry), + ) + assert battery.passed, battery.failures + + sentinel = battery.details["comparisons"][ + "person/puf_tax_itemization/taxable_interest_income[clone_0]" + ] + assert sentinel["status"] == "tested" + positive = sentinel["legs"]["positive"] + assert positive["acs_incidence"] > 0.0 + assert 0.8 <= positive["incidence_ratio_acs_over_asec"] <= 1.25 + detail = passed.frame.table("person") + detail_clone = detail[support_clone_index_column("person")].eq(1) + detail_channel = detail[support_channel_column("person")].astype(str) + for origin in ("asec", "acs"): + origin_detail = detail.loc[ + detail_clone & detail_channel.eq(origin), + "taxable_interest_income", + ] + assert origin_detail.notna().all() + assert (origin_detail > 0.0).any()