diff --git a/README.md b/README.md index 2dffffae..96e509e8 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,9 @@ normal `uv run pytest` suite; the real-H5 mode above is a local/runbook step. ## Releasing & alerts -Publishing uploads the locally built `releases//` artifacts to the Hugging -Face dataset, tags the release, and updates `latest.json`. It runs on the build -machine (it needs the freshly built H5), so it isn't a CI step: +Standard publication uploads the locally built `releases//` artifacts to +the Hugging Face dataset, tags the release, and updates `latest.json`. It runs +on the build machine (it needs the freshly built H5), so it isn't a CI step: ```bash tools/publish_release.sh releases/ --repo-id policyengine/populace-us @@ -117,6 +117,17 @@ tools/publish_release.sh releases/ --repo-id policyengine/populace-us publish CLI posts a release alert to Slack — `#populace-us` or `#populace-uk`, chosen from the repo id. +US exact-k ladder candidates use a tag-only lane. Run +`tools/build_us_exact_k_ladder_release.py`, then execute the `publish_command` +recorded in `package_result.json`. That command includes `--create-tag`, +`--no-latest`, and `--tag-only`: it uploads the immutable release and creates its +tag without committing candidate artifacts or release copies to the production +main branch. The launcher also forces `--no-staging`, so the build writes neither +a production nor a staging pointer. The candidate is therefore available only +by its explicit release id or tag until a separate promotion updates +`latest.json`. Because Slack alerts are coupled to that production pointer +update, tag-only publication sends no release alert. + The alert is a **no-op unless the channel's incoming-webhook URL is set**, so configure it once on the build machine: diff --git a/changelog.d/578-exact-k-ladder-release.added.md b/changelog.d/578-exact-k-ladder-release.added.md new file mode 100644 index 00000000..5ee71297 --- /dev/null +++ b/changelog.d/578-exact-k-ladder-release.added.md @@ -0,0 +1,21 @@ +Added the US exact-k ladder release launcher for the full multispine pool, +57,240 households, and 20,000 households. It authenticates the ready pool and +frozen target register, requires strict improvement over the incumbent on the +shared weighted loss, preserves the full-pool identity arm, uses seeded +Sampford selection plus an original-weight HT-with-q refit for smaller points, +records the selection and calibration receipts, and emits exact-count release +IDs with immutable `--create-tag --no-latest --tag-only` publication plumbing +without moving a pointer or mutating canonical artifacts on the dataset main +branch. Canonical artifact filenames remain loadable after a later manual +promotion, and an optional hash-pinned SSI retry basis preserves the certified +hard-gate path. + +The release lane now authenticates the pool's own release id and scores the +incumbent from the same verified bytes whose SHA-256 enters the manifest. It +binds the frozen-register comparison to the incumbent's weighting version, +family multipliers, loss cap, and name-aligned loss-vector digest, and refuses +basis drift. The house builder independently enforces the `N`, `57240`, and +`20000` ladder, an explicit seed, requested-versus-realized row-count equality, +and disabled staging. Package receipts record that neither the production nor +staging pointer changes. Legacy no-pool builds retain their previous receipt, +console-output, and CLI contracts. diff --git a/packages/populace-build/src/populace/build/us_runtime/exact_k_ladder.py b/packages/populace-build/src/populace/build/us_runtime/exact_k_ladder.py new file mode 100644 index 00000000..c56916b4 --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/exact_k_ladder.py @@ -0,0 +1,375 @@ +"""Exact-cardinality calibration seam for the US release ladder. + +The multispine pool supplies one original importance-weighted frame. A +non-census ladder point first learns hard-concrete open probabilities on that +full frame, draws a seeded fixed-size Sampford support, and refits ordinary +calibration from the normalized Horvitz--Thompson ``w / q`` baseline. The +full-pool point skips the stochastic draw but still runs ordinary calibration +and emits the same six-scalar receipt shape with ``design="full-pool"``. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Mapping +from dataclasses import dataclass + +import numpy as np + +from populace.calibrate import ( + CONSERVE_MASS, + CalibrationResult, + L0RefitResult, + TargetSet, + assert_exact_k_support, + calibrate, + effective_sample_size, + refit_l0_selection, + select_exact_k, +) +from populace.frame import Frame + +__all__ = [ + "ExactKLadderCalibration", + "ExactKRealizedCountMismatchError", + "assert_exact_k_realized_count", + "calibrate_exact_k_ladder", + "exact_k_ladder_manifest_payload", +] + + +@dataclass(frozen=True) +class ExactKLadderCalibration: + """One exact-k selection/refit result and its release-sized receipts.""" + + result: CalibrationResult | L0RefitResult + support: np.ndarray + selected_inclusion_probabilities: np.ndarray + selection_receipt: dict[str, int | float | str] + refit_baseline_diagnostics: dict[str, int | float | str] + + +class ExactKRealizedCountMismatchError(RuntimeError): + """The calibrated export count differs from the requested release count.""" + + +def assert_exact_k_realized_count( + outcome: ExactKLadderCalibration, + k: int, +) -> int: + """Refuse an exact-k receipt whose calibrated frame realized another count.""" + + requested = _nonnegative_integer(k, name="k") + realized = int(outcome.result.frame.n("household")) + if realized != requested: + raise ExactKRealizedCountMismatchError( + "ExactKRealizedCountMismatchError: requested/realized household " + f"count mismatch: requested={requested}, realized={realized}." + ) + return realized + + +def exact_k_ladder_manifest_payload( + outcome: ExactKLadderCalibration, + *, + k: int, + seed: int, + pool: Mapping[str, object], + agreement_gate_reference: Mapping[str, object], + frozen_target_register: Mapping[str, object], +) -> dict[str, object]: + """Render the receipt block shared by diagnostics and release manifests.""" + + target = _nonnegative_integer(k, name="k") + random_seed = _nonnegative_integer(seed, name="seed") + assert_exact_k_realized_count(outcome, target) + receipt = outcome.selection_receipt + receipt_keys = { + "k", + "pi_hi", + "seed", + "certainty_count", + "boundary_pool_size", + "design", + } + if set(receipt) != receipt_keys: + raise RuntimeError( + "Exact-k selection receipt must remain the public six-scalar " + f"contract; got keys {sorted(receipt)}." + ) + if receipt["k"] != target or receipt["seed"] != random_seed: + raise RuntimeError( + "Exact-k selection receipt disagrees with the configured k or seed." + ) + return { + "k": target, + "seed": random_seed, + # Do not add launcher fields or hashes here: this nested mapping must + # round-trip the public #585 six-scalar receipt without reinterpretation. + "selection_receipt": dict(receipt), + "refit_baseline_diagnostics": dict(outcome.refit_baseline_diagnostics), + "pool": dict(pool), + "agreement_gate_reference": dict(agreement_gate_reference), + "frozen_target_register": dict(frozen_target_register), + } + + +def calibrate_exact_k_ladder( + frame: Frame, + targets: TargetSet, + *, + k: int, + pi_hi: float, + seed: int, + weight_entity: str = "household", + epochs: int = 256, + refit_epochs: int | None = None, + learning_rate: float = 0.02, + refit_learning_rate: float | None = None, + mass: str = CONSERVE_MASS, + max_weight_ratio: float | None = None, + l0_lambda: float = 0.0, + l2_lambda: float = 0.0, + refit_l2_lambda: float | None = None, + l2_anchor: str = "initial", + refit_l2_anchor: str | None = None, + init_mean: float = 0.999, + temperature: float = 0.25, + budget_iters: int = 10, + target_loss_weights: np.ndarray | None = None, + target_loss_scales: np.ndarray | None = None, + target_loss_cap: float = 10.0, + warm_start_weights: np.ndarray | None = None, + progress_callback: Callable[[dict[str, object]], None] | None = None, +) -> ExactKLadderCalibration: + """Calibrate one exact-count point from an original weighted pool. + + ``k == N`` is an identity support selection followed by ordinary + calibration. ``k < N`` uses the public exact-k API and passes its aligned + marginal inclusion probabilities into the public explicit-support refit. + That refit is the #585 authority for subsetting the *original* frame and + constructing the normalized ``w / q`` baseline. + """ + + pool_size = int(frame.n(weight_entity)) + target = _nonnegative_integer(k, name="k") + if target == 0: + raise ValueError("k must be positive for a release dataset.") + if target > pool_size: + raise ValueError( + f"k={target} exceeds the pool size {pool_size}; ladder selection " + "never clamps the requested cardinality." + ) + random_seed = _nonnegative_integer(seed, name="seed") + certainty_threshold = _probability(pi_hi, name="pi_hi") + refit_steps = epochs if refit_epochs is None else refit_epochs + refit_rate = learning_rate if refit_learning_rate is None else refit_learning_rate + refit_penalty = l2_lambda if refit_l2_lambda is None else refit_l2_lambda + refit_anchor = l2_anchor if refit_l2_anchor is None else refit_l2_anchor + if target < pool_size and warm_start_weights is not None: + raise ValueError( + "k dict[str, int | float | str]: + original_weights = frame.resolve_weights(weight_entity) + source = np.asarray(original_weights.values, dtype=np.float64) + selected_source = source[support] + q = np.asarray(selected_inclusion_probabilities, dtype=np.float64) + unnormalized = selected_source / q + normalization_factor = float(source.sum() / unnormalized.sum()) + expected_baseline = unnormalized * normalization_factor + observed_baseline = np.asarray(result.initial_weights, dtype=np.float64) + if not np.allclose( + observed_baseline, + expected_baseline, + rtol=1e-12, + atol=1e-12, + ): + raise RuntimeError( + "Exact-k refit baseline no longer equals the normalized original-frame " + "w/q projection." + ) + return { + "method": ( + "full_pool_original_frame_weights" + if full_pool + else "normalized_horvitz_thompson_w_over_q" + ), + "source_weight_kind": original_weights.kind.value, + "pool_size": int(source.size), + "selected_size": int(support.size), + "pool_weight_total": float(source.sum()), + "selected_original_weight_total": float(selected_source.sum()), + "unnormalized_ht_weight_total": float(unnormalized.sum()), + "normalization_factor": normalization_factor, + "refit_baseline_weight_total": float(observed_baseline.sum()), + "refit_baseline_minimum": float(observed_baseline.min()), + "refit_baseline_maximum": float(observed_baseline.max()), + "refit_baseline_effective_sample_size": float( + effective_sample_size(observed_baseline) + ), + "inclusion_probability_minimum": float(q.min()), + "inclusion_probability_maximum": float(q.max()), + } + + +def _nonnegative_integer(value: object, *, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int | np.integer): + raise ValueError(f"{name} must be a non-negative integer, got {value!r}.") + parsed = int(value) + if parsed < 0: + raise ValueError(f"{name} must be a non-negative integer, got {value!r}.") + return parsed + + +def _probability(value: object, *, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a finite value in [0, 1], got {value!r}.") + try: + parsed = float(value) + except (TypeError, ValueError): + raise ValueError( + f"{name} must be a finite value in [0, 1], got {value!r}." + ) from None + if not math.isfinite(parsed) or not 0.0 <= parsed <= 1.0: + raise ValueError(f"{name} must be a finite value in [0, 1], got {value!r}.") + return parsed + + +def _phase_callback( + callback: Callable[[dict[str, object]], None] | None, + phase: str, +) -> Callable[[dict[str, object]], None] | None: + if callback is None: + return None + + def with_phase(event: dict[str, object]) -> None: + callback({"phase": phase, **event}) + + return with_phase diff --git a/packages/populace-build/src/populace/build/us_runtime/h5_io.py b/packages/populace-build/src/populace/build/us_runtime/h5_io.py index e4b19040..9bec3105 100644 --- a/packages/populace-build/src/populace/build/us_runtime/h5_io.py +++ b/packages/populace-build/src/populace/build/us_runtime/h5_io.py @@ -13,9 +13,11 @@ import json import os import re +import shutil import uuid import warnings from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path import numpy as np @@ -25,12 +27,15 @@ from populace.frame.units import US_SCHEMA __all__ = [ + "AuthenticatedPoolH5", + "AuthenticatedPoolH5MismatchError", "LEGACY_NULLABLE_STAGING_ARTIFACT_KIND", "US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND", "US_MULTISPINE_POOL_H5_ARTIFACT_KIND", "US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND", "US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION", "load_legacy_calibrated_us_h5", + "load_simulation_ready_us_multispine_pool", "load_simulation_ready_us_multispine_pool_manifest", "read_nullable_us_h5_metadata", "write_nullable_us_h5", @@ -50,6 +55,83 @@ _LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") +class AuthenticatedPoolH5MismatchError(RuntimeError): + """A pool H5 no longer matches the bytes authenticated by its manifest.""" + + +@dataclass(frozen=True) +class AuthenticatedPoolH5: + """Immutable identity of the pool H5 authorized by one manifest buffer.""" + + path: Path + sha256: str + size_bytes: int + publication_run_id: str + manifest_sha256: str + + def verified_digest(self, *, consumer: str) -> str: + """Re-verify the pathname and return only the authenticated digest.""" + + try: + observed_sha256, observed_size_bytes = _file_sha256_and_size(self.path) + except OSError as exc: + raise AuthenticatedPoolH5MismatchError( + "AuthenticatedPoolH5MismatchError: authenticated pool H5 " + f"became unreadable at consumer {consumer!r}: {self.path}; " + f"expected sha256={self.sha256}, size_bytes={self.size_bytes}." + ) from exc + if observed_sha256 != self.sha256 or observed_size_bytes != self.size_bytes: + self._raise_mismatch( + consumer=consumer, + observed_sha256=observed_sha256, + observed_size_bytes=observed_size_bytes, + ) + return self.sha256 + + def copy_verified_to(self, destination: str | Path, *, consumer: str) -> Path: + """Atomically copy the authenticated bytes and reject a raced source.""" + + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") + try: + self.verified_digest(consumer=f"{consumer} source preflight") + _copy_file_bytes(self.path, temporary) + observed_sha256, observed_size_bytes = _file_sha256_and_size(temporary) + if observed_sha256 != self.sha256 or observed_size_bytes != self.size_bytes: + self._raise_mismatch( + consumer=f"{consumer} copied bytes", + observed_sha256=observed_sha256, + observed_size_bytes=observed_size_bytes, + ) + os.replace(temporary, destination) + except AuthenticatedPoolH5MismatchError: + temporary.unlink(missing_ok=True) + raise + except OSError as exc: + temporary.unlink(missing_ok=True) + raise AuthenticatedPoolH5MismatchError( + "AuthenticatedPoolH5MismatchError: authenticated pool H5 " + f"became unreadable at {consumer!r}: {self.path}; expected " + f"sha256={self.sha256}, size_bytes={self.size_bytes}." + ) from exc + return destination + + def _raise_mismatch( + self, + *, + consumer: str, + observed_sha256: str, + observed_size_bytes: int, + ) -> None: + raise AuthenticatedPoolH5MismatchError( + "AuthenticatedPoolH5MismatchError: authenticated pool H5 changed " + f"at consumer {consumer!r}: {self.path}; expected " + f"sha256={self.sha256}, size_bytes={self.size_bytes}, observed " + f"sha256={observed_sha256}, size_bytes={observed_size_bytes}." + ) + + def load_legacy_calibrated_us_h5(path: str | Path) -> Frame: """Load a legacy US single-year H5 as a calibrated-weight frame. @@ -88,17 +170,39 @@ def load_legacy_calibrated_us_h5(path: str | Path) -> Frame: def load_simulation_ready_us_multispine_pool_manifest( path: str | Path, + *, + expected_manifest_sha256: str | None = None, ) -> dict[str, object]: """Validate and return one ready manifest bound to its H5 and diagnostics. The manifest is the readiness authority. A caller cannot treat an H5 as ready merely because it exists: the manifest, nested artifact receipts, H5 metadata, diagnostics, and file digests must all bind the same - publication run. + publication run. When ``expected_manifest_sha256`` is supplied, this + function hashes and parses one byte buffer so a replacement cannot inherit + the authenticated manifest identity. """ + manifest, _ = _load_authenticated_us_multispine_pool_manifest( + path, + expected_manifest_sha256=expected_manifest_sha256, + ) + return manifest + + +def _load_authenticated_us_multispine_pool_manifest( + path: str | Path, + *, + expected_manifest_sha256: str | None = None, +) -> tuple[dict[str, object], AuthenticatedPoolH5]: + """Return the validated manifest and its authenticated pool-H5 identity.""" + manifest_path = Path(path) - manifest = _read_json_object(manifest_path, label="pool manifest") + manifest, manifest_sha256, _ = _read_json_object_with_identity( + manifest_path, + label="pool manifest", + expected_sha256=expected_manifest_sha256, + ) if ( manifest.get("artifact_kind") != US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND or manifest.get("schema_version") != US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION @@ -156,10 +260,11 @@ def load_simulation_ready_us_multispine_pool_manifest( pool_receipt, label=f"US multispine pool manifest {manifest_path}.pool_h5", ) - _require_matching_sha256( + pool_sha256, pool_size_bytes = _require_matching_sha256( pool_path, pool_receipt, label=f"US multispine pool manifest {manifest_path}.pool_h5", + require_size=True, ) h5_metadata = read_nullable_us_h5_metadata(pool_path) if h5_metadata.get("artifact_kind") != US_MULTISPINE_POOL_H5_ARTIFACT_KIND: @@ -206,7 +311,136 @@ def load_simulation_ready_us_multispine_pool_manifest( f"US multispine pool diagnostics {diagnostics_path} do not match " "the ready manifest publication." ) - return manifest + manifest_agreement_gate = _mapping( + manifest.get("agreement_gate"), + label=f"US multispine pool manifest {manifest_path}.agreement_gate", + ) + diagnostics_agreement_gate = _mapping( + diagnostics.get("agreement_gate"), + label=f"US multispine pool diagnostics {diagnostics_path}.agreement_gate", + ) + if diagnostics_agreement_gate != manifest_agreement_gate: + raise ValueError( + f"US multispine pool diagnostics {diagnostics_path} agreement-gate " + "verdict does not match the ready manifest." + ) + return manifest, AuthenticatedPoolH5( + path=pool_path.resolve(), + sha256=pool_sha256, + size_bytes=pool_size_bytes, + publication_run_id=publication_run_id, + manifest_sha256=manifest_sha256, + ) + + +def load_simulation_ready_us_multispine_pool( + path: str | Path, + *, + expected_manifest_sha256: str | None = None, +) -> tuple[Frame, dict[str, object], AuthenticatedPoolH5]: + """Load a manifest-bound multispine pool with its importance weights. + + The companion manifest remains the readiness authority. This loader first + validates the manifest/H5/agreement publication triple, then reads the + fixed-format entity tables and checks the H5 digest again after the read so + bytes changed concurrently cannot be treated as the validated pool. The + pool's household weights retain their ``IMPORTANCE`` provenance; the + legacy US loader deliberately labels historical datasets ``CALIBRATED`` + and is therefore not a valid consumer for this artifact. + + Returns: + The reconstructed pool :class:`~populace.frame.Frame`, the exact + manifest object whose artifact receipts authorized the load, and one + immutable identity object for every downstream H5 consumer. + """ + + manifest_path = Path(path) + manifest, authenticated_pool_h5 = _load_authenticated_us_multispine_pool_manifest( + manifest_path, + expected_manifest_sha256=expected_manifest_sha256, + ) + agreement_gate = _mapping( + manifest.get("agreement_gate"), + label=f"US multispine pool manifest {manifest_path}.agreement_gate", + ) + if agreement_gate.get("passed") is not True: + raise ValueError( + f"US multispine pool manifest {manifest_path} has no passing " + "agreement-gate verdict." + ) + + pool_path = authenticated_pool_h5.path + metadata = read_nullable_us_h5_metadata(pool_path) + stored_kind = metadata.get("household_weight_kind") + if stored_kind != WeightKind.IMPORTANCE.value: + raise ValueError( + f"US multispine pool H5 {pool_path} must carry importance weights, " + f"got {stored_kind!r}." + ) + + with pd.HDFStore(pool_path, mode="r") as store: + keys = {key.lstrip("/") for key in store.keys()} + missing = sorted(set(US_SCHEMA.entities) - keys) + if missing: + raise ValueError( + f"US multispine pool H5 {pool_path} is missing entity table(s): " + f"{missing}." + ) + tables = {entity: store[entity] for entity in US_SCHEMA.entities} + period = store[_TIME_PERIOD_KEY] + + if len(period) != 1 or period.tolist() != [manifest.get("period")]: + raise ValueError( + f"US multispine pool H5 {pool_path} period does not match its " + f"manifest: H5={period.tolist()!r}, manifest={manifest.get('period')!r}." + ) + household = tables["household"].copy() + if "household_weight" not in household: + raise ValueError( + f"US multispine pool H5 {pool_path} household table has no " + "household_weight column." + ) + household_weights = household.pop("household_weight").to_numpy(dtype=np.float64) + tables["household"] = household + frame = Frame( + tables, + US_SCHEMA, + { + "household": Weights( + household_weights, + WeightKind.IMPORTANCE, + ) + }, + ) + + provenance_counts = _mapping( + manifest.get("provenance_counts"), + label=f"US multispine pool manifest {manifest_path}.provenance_counts", + ) + household_counts = _mapping( + provenance_counts.get("household"), + label=( + f"US multispine pool manifest {manifest_path}.provenance_counts.household" + ), + ) + expected_households = household_counts.get("rows") + if ( + isinstance(expected_households, bool) + or not isinstance(expected_households, int) + or expected_households != frame.n("household") + ): + raise ValueError( + f"US multispine pool manifest {manifest_path} household row count " + f"{expected_households!r} does not match H5 count " + f"{frame.n('household')}." + ) + + # Close the validation/read time-of-check-to-time-of-use window. A file + # replacement during the HDF read must not inherit the first digest check. + authenticated_pool_h5.verified_digest( + consumer="pool loader post-HDF read", + ) + return frame, manifest, authenticated_pool_h5 def read_nullable_us_h5_metadata(path: str | Path) -> dict[str, object]: @@ -418,14 +652,50 @@ def _artifact_metadata( return metadata -def _read_json_object(path: Path, *, label: str) -> dict[str, object]: +def _read_json_object( + path: Path, + *, + label: str, + expected_sha256: str | None = None, +) -> dict[str, object]: + payload, _, _ = _read_json_object_with_identity( + path, + label=label, + expected_sha256=expected_sha256, + ) + return payload + + +def _read_json_object_with_identity( + path: Path, + *, + label: str, + expected_sha256: str | None = None, +) -> tuple[dict[str, object], str, int]: try: - payload = json.loads(Path(path).read_text(encoding="utf-8")) + raw = Path(path).read_bytes() except (OSError, TypeError, ValueError) as exc: raise ValueError(f"{label} {path} is not readable valid JSON.") from exc + observed_sha256 = hashlib.sha256(raw).hexdigest() + if expected_sha256 is not None: + if not isinstance(expected_sha256, str) or not _LOWERCASE_SHA256.fullmatch( + expected_sha256 + ): + raise ValueError( + f"Expected {label} SHA-256 must be 64 lowercase hexadecimal characters." + ) + if observed_sha256 != expected_sha256: + raise ValueError( + f"{label.capitalize()} SHA-256 mismatch for {path}: got " + f"{observed_sha256}, expected {expected_sha256}." + ) + try: + payload = json.loads(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} {path} is not readable valid JSON.") from exc if not isinstance(payload, dict): raise ValueError(f"{label} {path} must contain a JSON object.") - return payload + return payload, observed_sha256, len(raw) def _mapping(value: object, *, label: str) -> Mapping[str, object]: @@ -465,13 +735,42 @@ def _require_matching_sha256( receipt: Mapping[str, object], *, label: str, -) -> None: + require_size: bool = False, +) -> tuple[str, int]: expected = receipt.get("sha256") if not isinstance(expected, str) or not _LOWERCASE_SHA256.fullmatch(expected): raise ValueError(f"{label}.sha256 must be a lowercase SHA-256 digest.") + expected_size = receipt.get("size_bytes") + if require_size and ( + isinstance(expected_size, bool) + or not isinstance(expected_size, int) + or expected_size < 0 + ): + raise ValueError(f"{label}.size_bytes must be a non-negative integer.") + observed_sha256, observed_size_bytes = _file_sha256_and_size(path) + if observed_sha256 != expected: + raise ValueError(f"{label} SHA-256 does not match the published artifact.") + if require_size and observed_size_bytes != expected_size: + raise ValueError( + f"{label} size_bytes {observed_size_bytes} does not match the " + f"published artifact size {expected_size}." + ) + return observed_sha256, observed_size_bytes + + +def _file_sha256_and_size(path: Path) -> tuple[str, int]: digest = hashlib.sha256() + size_bytes = 0 with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) - if digest.hexdigest() != expected: - raise ValueError(f"{label} SHA-256 does not match the published artifact.") + size_bytes += len(chunk) + return digest.hexdigest(), size_bytes + + +def _copy_file_bytes(source: Path, destination: Path) -> None: + with ( + source.open("rb") as source_stream, + destination.open("xb") as destination_stream, + ): + shutil.copyfileobj(source_stream, destination_stream, length=1024 * 1024) diff --git a/packages/populace-build/tests/test_us_exact_k_ladder.py b/packages/populace-build/tests/test_us_exact_k_ladder.py new file mode 100644 index 00000000..4b2e2211 --- /dev/null +++ b/packages/populace-build/tests/test_us_exact_k_ladder.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from populace.build.us_runtime.exact_k_ladder import ( + calibrate_exact_k_ladder, + exact_k_ladder_manifest_payload, +) +from populace.calibrate import Target, TargetSet +from populace.frame import EntitySchema, Frame, WeightKind, Weights + + +def _fixture_pool() -> tuple[Frame, TargetSet]: + schema = EntitySchema(group_entities=("household",)) + weights = np.arange(1.0, 9.0) + measure = np.asarray([0.0, 1.0, 2.0, 3.0, 6.0, 9.0, 12.0, 20.0]) + frame = Frame( + { + "person": pd.DataFrame( + { + "person_id": range(8), + "person_household_id": range(8), + } + ), + "household": pd.DataFrame( + { + "household_id": range(8), + "fixture_measure": measure, + } + ), + }, + schema, + {"household": Weights(weights, WeightKind.IMPORTANCE)}, + ) + targets = TargetSet( + ( + Target( + name="fixture_measure", + entity="household", + value=float(weights @ measure) * 0.9, + measure="fixture_measure", + ), + ) + ) + return frame, targets + + +@pytest.mark.parametrize( + ("ladder_point", "k", "expected_design"), + ( + ("N", 8, "full-pool"), + ("57,240 fixture analogue", 6, "sampford"), + ("20,000 fixture analogue", 4, "sampford"), + ), +) +def test_each_ladder_point_selects_refits_and_emits_round_trip_receipt( + ladder_point: str, + k: int, + expected_design: str, +) -> None: + frame, targets = _fixture_pool() + + outcome = calibrate_exact_k_ladder( + frame, + targets, + k=k, + pi_hi=1.0, + seed=17, + epochs=3, + refit_epochs=3, + learning_rate=0.02, + max_weight_ratio=20.0, + l0_lambda=1e-8, + ) + + assert ladder_point + assert outcome.support.shape == (k,) + assert outcome.result.frame.n("household") == k + assert outcome.result.frame.weights_for("household").kind is WeightKind.CALIBRATED + assert outcome.selection_receipt["design"] == expected_design + assert outcome.selection_receipt["k"] == k + assert set(outcome.selection_receipt) == { + "k", + "pi_hi", + "seed", + "certainty_count", + "boundary_pool_size", + "design", + } + assert json.loads(json.dumps(outcome.selection_receipt)) == ( + outcome.selection_receipt + ) + assert outcome.refit_baseline_diagnostics["pool_weight_total"] == 36.0 + assert outcome.refit_baseline_diagnostics["refit_baseline_weight_total"] == 36.0 + + original = frame.weights_for("household").values[outcome.support] + q = outcome.selected_inclusion_probabilities + expected_baseline = original / q + expected_baseline *= 36.0 / expected_baseline.sum() + np.testing.assert_allclose( + outcome.result.initial_weights, + expected_baseline, + rtol=1e-12, + atol=1e-12, + ) + manifest_receipt = exact_k_ladder_manifest_payload( + outcome, + k=k, + seed=17, + pool={ + "release_id": "fixture-pool", + "manifest_sha256": "a" * 64, + }, + agreement_gate_reference={ + "passed": True, + "diagnostics_sha256": "b" * 64, + }, + frozen_target_register={ + "target_surface_sha256": "c" * 64, + "incumbent_diagnostics_sha256": "d" * 64, + }, + ) + round_trip = json.loads(json.dumps(manifest_receipt)) + assert round_trip["selection_receipt"] == outcome.selection_receipt + assert round_trip["k"] == k + assert round_trip["seed"] == 17 + assert round_trip["pool"]["manifest_sha256"] == "a" * 64 + assert round_trip["agreement_gate_reference"]["passed"] is True + assert round_trip["frozen_target_register"]["target_surface_sha256"] == "c" * 64 + + +def test_full_pool_support_is_identity_but_weights_are_refit() -> None: + frame, targets = _fixture_pool() + + outcome = calibrate_exact_k_ladder( + frame, + targets, + k=8, + pi_hi=0.95, + seed=99, + epochs=3, + l0_lambda=1e-8, + ) + + np.testing.assert_array_equal(outcome.support, np.arange(8)) + np.testing.assert_array_equal( + outcome.result.frame.table("household")["household_id"], + frame.table("household")["household_id"], + ) + assert outcome.selection_receipt == { + "k": 8, + "pi_hi": 0.95, + "seed": 99, + "certainty_count": 8, + "boundary_pool_size": 0, + "design": "full-pool", + } + assert not np.array_equal( + outcome.result.weights, + frame.weights_for("household").values, + ) + assert ( + outcome.refit_baseline_diagnostics["method"] + == "full_pool_original_frame_weights" + ) + + +def test_manifest_refuses_requested_realized_count_mismatch() -> None: + """The r1 k=8 receipt/frame.n=7 fault injection never reaches naming.""" + outcome = SimpleNamespace( + result=SimpleNamespace(frame=SimpleNamespace(n=lambda entity: 7)), + selection_receipt={ + "k": 8, + "pi_hi": 0.95, + "seed": 17, + "certainty_count": 8, + "boundary_pool_size": 0, + "design": "full-pool", + }, + refit_baseline_diagnostics={}, + ) + + with pytest.raises( + RuntimeError, + match=( + "ExactKRealizedCountMismatchError: requested/realized household " + "count mismatch: requested=8, realized=7" + ), + ): + exact_k_ladder_manifest_payload( + outcome, + k=8, + seed=17, + pool={}, + agreement_gate_reference={}, + frozen_target_register={}, + ) + + +def test_ladder_calibration_rejects_invalid_cardinality_before_selection() -> None: + frame, targets = _fixture_pool() + + with pytest.raises(ValueError, match="k=9 exceeds the pool size 8"): + calibrate_exact_k_ladder( + frame, + targets, + k=9, + pi_hi=0.95, + seed=0, + l0_lambda=1e-8, + ) + + with pytest.raises(ValueError, match="requires a positive finite l0_lambda"): + calibrate_exact_k_ladder( + frame, + targets, + k=4, + pi_hi=0.95, + seed=0, + l0_lambda=0.0, + ) diff --git a/packages/populace-build/tests/test_us_exact_k_ladder_e2e.py b/packages/populace-build/tests/test_us_exact_k_ladder_e2e.py new file mode 100644 index 00000000..095275b7 --- /dev/null +++ b/packages/populace-build/tests/test_us_exact_k_ladder_e2e.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from populace.build.us_runtime.exact_k_ladder import calibrate_exact_k_ladder +from populace.build.us_runtime.h5_io import ( + US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND, + US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND, + US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + load_simulation_ready_us_multispine_pool, + write_nullable_us_h5, +) +from populace.calibrate import TargetRegistry, TargetSpec +from populace.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _builder_module(): + root = Path(__file__).resolve().parents[3] + path = root / "tools" / "build_us_fiscal_refresh_release.py" + spec = importlib.util.spec_from_file_location( + "build_us_fiscal_refresh_release_exact_k_e2e", path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _pool_frame() -> Frame: + ids = np.arange(1, 9, dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + **{ + US_SCHEMA.membership_column(entity): ids + for entity in US_SCHEMA.group_entities + }, + } + ) + tables = { + "person": person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + } + tables["household"]["fixture_measure"] = np.asarray( + [0.0, 1.0, 2.0, 3.0, 6.0, 9.0, 12.0, 20.0] + ) + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.arange(1.0, 9.0), + WeightKind.IMPORTANCE, + ) + }, + ) + + +def _write_ready_pool(tmp_path: Path) -> Path: + run_id = "fixture-publication" + pool_path = tmp_path / "pool.h5" + diagnostics_path = tmp_path / "pool.agreement.json" + manifest_path = tmp_path / "pool.manifest.json" + agreement_gate = { + "passed": True, + "gates": { + "us_spine_agreement": { + "passed": True, + "failures": [], + "details": {"fixture": True}, + } + }, + } + write_nullable_us_h5( + _pool_frame(), + pool_path, + period=2024, + artifact_kind=US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + publication_run_id=run_id, + ) + diagnostics_path.write_text( + json.dumps( + { + "artifact_kind": (US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND), + "schema_version": US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + "simulation_ready": True, + "publication_run_id": run_id, + "agreement_gate": agreement_gate, + } + ), + encoding="utf-8", + ) + manifest_path.write_text( + json.dumps( + { + "artifact_kind": US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND, + "schema_version": US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + "status": "simulation_ready", + "simulation_ready": True, + "publication_run_id": run_id, + "period": 2024, + "stage_checkpoints": { + "agreement": { + "source": "always_fresh", + "cached": False, + "terminal_verdict_persisted": False, + } + }, + "agreement_gate": agreement_gate, + "provenance_counts": {"household": {"rows": 8}}, + "pool_h5": { + "path": str(pool_path.resolve()), + "sha256": _sha256(pool_path), + "size_bytes": pool_path.stat().st_size, + "artifact_kind": US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + "publication_run_id": run_id, + }, + "agreement_diagnostics": { + "path": str(diagnostics_path.resolve()), + "sha256": _sha256(diagnostics_path), + "publication_run_id": run_id, + }, + } + ), + encoding="utf-8", + ) + return manifest_path + + +@pytest.mark.parametrize( + ("ladder_point", "k", "expected_design"), + ( + ("N", 8, "full-pool"), + ("57,240 fixture analogue", 6, "sampford"), + ("20,000 fixture analogue", 4, "sampford"), + ), +) +def test_ready_pool_to_refit_and_release_manifests_for_each_ladder_point( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ladder_point: str, + k: int, + expected_design: str, +) -> None: + pytest.importorskip("tables") + builder = _builder_module() + pool_manifest_path = _write_ready_pool(tmp_path) + pool, pool_manifest, authenticated_pool_h5 = ( + load_simulation_ready_us_multispine_pool(pool_manifest_path) + ) + target = TargetSpec( + name="fixture_measure", + entity="household", + value=float( + pool.weights_for("household").values + @ pool.table("household")["fixture_measure"].to_numpy() + ) + * 0.9, + measure="fixture_measure", + period=2024, + source="fixture frozen register", + family="fixture", + ) + registry = TargetRegistry((target,), country="us") + outcome = calibrate_exact_k_ladder( + pool, + registry.to_target_set(), + k=k, + pi_hi=1.0, + seed=17, + epochs=3, + refit_epochs=3, + learning_rate=0.02, + max_weight_ratio=20.0, + l0_lambda=1e-8, + target_loss_weights=np.ones(1), + ) + + diagnostic = outcome.result.diagnostics[0] + incumbent_rows = { + diagnostic.name: { + "target": diagnostic.target, + "final_estimate": diagnostic.target * 11.0, + } + } + loss_basis = builder._fiscal_target_loss_basis(registry, np.ones(1)) + incumbent_gate = builder._exact_k_frozen_register_fit_gate( + outcome.result, + incumbent_rows, + target_registry=registry, + target_loss_weights=np.ones(1), + configured_loss_basis=loss_basis, + incumbent_loss_basis=loss_basis, + ) + assert incumbent_gate.passed + target_surface = builder.diagnostics_payload( + outcome.result, + target_registry=registry, + )["target_surface"] + args = argparse.Namespace( + exact_k=k, + seed=17, + pool_release_id="fixture-publication", + pool_manifest_sha256=_sha256(pool_manifest_path), + incumbent_diagnostics_sha256="f" * 64, + ) + ladder_receipt = builder._exact_k_ladder_manifest_payload( + args=args, + outcome=outcome, + pool_manifest=pool_manifest, + authenticated_pool_h5=authenticated_pool_h5, + ledger_artifact={ + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + }, + target_surface=target_surface, + target_loss_basis=loss_basis, + incumbent_diagnostics_sha256="f" * 64, + incumbent_fit_gate=incumbent_gate, + puf_tail_gate=builder.GateResult( + name="exact_k_puf_capital_gains_tail", + passed=True, + details={"status": "fixture_support_retained"}, + ), + ) + + release_id = f"populace-us-2024-k{k}-fixture" + release_dir = tmp_path / "release" / release_id + artifact_root = tmp_path / "artifacts" + release_dir.mkdir(parents=True) + artifact_root.mkdir() + (artifact_root / builder.DATASET_FILENAME).write_bytes(b"fixture h5") + (artifact_root / builder.CALIBRATION_FILENAME).write_bytes(b"fixture npz") + (release_dir / "calibration_diagnostics.json").write_text("{}") + (release_dir / "us_source_coverage.json").write_text("{}") + (release_dir / "us_ssi_take_up.json").write_text("{}") + monkeypatch.setattr( + builder, + "_runtime_versions", + lambda: { + "python": "3.14.0", + "populace-data": "0.1.0", + "policyengine-core": "3.26.11", + "policyengine-us": "1.752.2", + }, + ) + monkeypatch.setattr(builder, "_git_output", lambda *args: "a" * 40) + monkeypatch.setattr(builder, "_release_gate_failures", lambda *args, **kwargs: []) + builder._build_manifests( + release_id=release_id, + release_dir=release_dir, + artifact_root=artifact_root, + result=outcome.result, + registry=registry, + dropped={"dropped_target_names": []}, + target_profile_gate=builder.GateResult( + name="target_profile_coverage", + passed=True, + details={"requirements_checked": 1}, + ), + ledger_artifact={ + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + }, + default_dataset={ + "method": "full_pool_refit" if k == 8 else "exact_k_sampford_refit", + "n_candidate_households": 8, + "n_selected_households": k, + }, + exact_k_ladder=ladder_receipt, + ) + + build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) + release_manifest = json.loads((release_dir / "release_manifest.json").read_text()) + assert ladder_point + assert outcome.result.frame.n("household") == k + assert outcome.result.frame.weights_for("household").kind is WeightKind.CALIBRATED + assert ladder_receipt["k"] == k + assert ladder_receipt["seed"] == 17 + assert ladder_receipt["selection_receipt"] == outcome.selection_receipt + assert ladder_receipt["selection_receipt"]["design"] == expected_design + assert ladder_receipt["pool"] == { + "release_id": "fixture-publication", + "release_id_source": "pool_manifest.publication_run_id", + "manifest_sha256": _sha256(pool_manifest_path), + "publication_run_id": "fixture-publication", + "pool_h5_sha256": pool_manifest["pool_h5"]["sha256"], + "pool_h5_size_bytes": pool_manifest["pool_h5"]["size_bytes"], + "agreement_diagnostics_sha256": pool_manifest["agreement_diagnostics"][ + "sha256" + ], + } + assert ladder_receipt["agreement_gate_reference"] == { + "passed": True, + "publication_run_id": "fixture-publication", + "diagnostics_sha256": pool_manifest["agreement_diagnostics"]["sha256"], + "verdict": pool_manifest["agreement_gate"], + } + assert ladder_receipt["frozen_target_register"]["ledger_artifact"] == { + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + } + assert ( + ladder_receipt["frozen_target_register"]["target_surface_sha256"] + == target_surface["sha256"] + ) + assert ladder_receipt["frozen_target_register"]["target_loss_basis"] == loss_basis + assert ( + ladder_receipt["frozen_target_register"]["incumbent_diagnostics_sha256"] + == "f" * 64 + ) + assert ladder_receipt["frozen_target_register"]["incumbent_fit"]["passed"] + assert ladder_receipt["invariant_battery"] == { + "puf_capital_gains_tail": { + "passed": True, + "failures": [], + "details": {"status": "fixture_support_retained"}, + } + } + assert ladder_receipt["refit_baseline_diagnostics"]["method"] == ( + "full_pool_original_frame_weights" + if k == 8 + else "normalized_horvitz_thompson_w_over_q" + ) + assert ( + ladder_receipt["refit_baseline_diagnostics"]["source_weight_kind"] + == "importance" + ) + assert build_manifest["exact_k_ladder"] == ladder_receipt + assert release_manifest["build"]["exact_k_ladder"] == ladder_receipt + assert release_manifest["default_datasets"] == {"national": "populace_us_2024"} + assert ( + release_manifest["artifacts"]["populace_us_2024"]["path"] + == builder.DATASET_FILENAME + ) + if k == 8: + np.testing.assert_array_equal(outcome.support, np.arange(8)) + assert not np.array_equal( + outcome.result.weights, + pool.weights_for("household").values, + ) diff --git a/packages/populace-build/tests/test_us_exact_k_ladder_launcher.py b/packages/populace-build/tests/test_us_exact_k_ladder_launcher.py new file mode 100644 index 00000000..5f600b70 --- /dev/null +++ b/packages/populace-build/tests/test_us_exact_k_ladder_launcher.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import sys +from pathlib import Path + +import pytest + + +def _launcher_module(): + tools = Path(__file__).resolve().parents[3] / "tools" + if str(tools) not in sys.path: + sys.path.insert(0, str(tools)) + return importlib.import_module("build_us_exact_k_ladder_release") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _config_payload( + *, + pool_manifest_sha256: str = "a" * 64, + requested_k: str | int = "N", + release_id: str = "populace-us-2024-k8-fixture", +) -> dict[str, object]: + return { + "schema_version": 1, + "pool": { + "release_id": "fixture-publication", + "manifest_sha256": pool_manifest_sha256, + }, + "ladder": {"k": requested_k, "seed": 17, "pi_hi": 0.95}, + "targets": { + "ledger_facts": "ledger", + "ledger_facts_sha256": "b" * 64, + "ledger_manifest_sha256": "c" * 64, + "incumbent_diagnostics": "incumbent.json", + "incumbent_diagnostics_sha256": "d" * 64, + "target_surface_sha256": "e" * 64, + }, + "calibration": { + "epochs": 3, + "learning_rate": 0.02, + "max_weight_ratio": 20.0, + "l0_refit_lambda_share": 0.8, + "l2_lambda": 0.0, + "refit_l2_lambda": 0.0, + }, + "release": { + "id": release_id, + "repo_id": "policyengine/populace-us", + }, + } + + +def _write_config(tmp_path: Path, payload: dict[str, object]) -> Path: + path = tmp_path / "config.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_config_requires_explicit_seed_and_ratified_k(tmp_path: Path) -> None: + launcher = _launcher_module() + missing_seed = _config_payload() + del missing_seed["ladder"]["seed"] + path = _write_config(tmp_path, missing_seed) + + with pytest.raises(ValueError, match=r"missing=\['seed'\]"): + launcher._read_config(path) + + bad_k = _config_payload(requested_k=57_241) + path = _write_config(tmp_path, bad_k) + with pytest.raises(ValueError, match="exactly 'N', 57240, or 20000"): + launcher._read_config(path) + + unpinned_retry = _config_payload() + unpinned_retry["targets"]["ssi_take_up_prior_weight_basis"] = "ssi.json" + path = _write_config(tmp_path, unpinned_retry) + with pytest.raises(ValueError, match="and its SHA-256 pin"): + launcher._read_config(path) + + +def test_config_rejects_k_larger_than_manifest_pool( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config = launcher._read_config( + _write_config( + tmp_path, + _config_payload( + pool_manifest_sha256=_sha256(manifest), + requested_k=57_240, + release_id="populace-us-2024-k57240-fixture", + ), + ) + ) + monkeypatch.setattr( + launcher, + "load_simulation_ready_us_multispine_pool_manifest", + lambda _, **_kwargs: { + "publication_run_id": "fixture-publication", + "agreement_gate": {"passed": True}, + "provenance_counts": {"household": {"rows": 50_000}}, + }, + ) + + with pytest.raises(ValueError, match="k=57240 exceeds the pool size 50000"): + launcher._validate_pins_and_resolve_k( + config=config, + pool_manifest_path=manifest, + ) + + +def test_pool_manifest_sha_pin_is_checked_on_loaded_bytes(tmp_path: Path) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config = launcher._read_config( + _write_config(tmp_path, _config_payload(pool_manifest_sha256="a" * 64)) + ) + with pytest.raises(ValueError, match="Pool manifest SHA-256 mismatch"): + launcher._validate_pins_and_resolve_k( + config=config, + pool_manifest_path=manifest, + ) + + +def test_n_resolves_to_realized_pool_size_with_valid_pins( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + (tmp_path / "ledger").mkdir() + incumbent = tmp_path / "incumbent.json" + incumbent.write_text( + json.dumps({"target_surface": {"sha256": "e" * 64}}), + encoding="utf-8", + ) + payload = _config_payload(pool_manifest_sha256=_sha256(manifest)) + payload["targets"]["incumbent_diagnostics_sha256"] = _sha256(incumbent) + config = launcher._read_config(_write_config(tmp_path, payload)) + validated_manifest = { + "publication_run_id": "fixture-publication", + "agreement_gate": {"passed": True}, + "provenance_counts": {"household": {"rows": 8}}, + } + + def fake_load_manifest(_, *, expected_manifest_sha256): + assert expected_manifest_sha256 == config.pool_manifest_sha256 + return validated_manifest + + monkeypatch.setattr( + launcher, + "load_simulation_ready_us_multispine_pool_manifest", + fake_load_manifest, + ) + + k, observed_manifest = launcher._validate_pins_and_resolve_k( + config=config, + pool_manifest_path=manifest, + ) + + assert k == 8 + assert observed_manifest is validated_manifest + + +@pytest.mark.parametrize( + ("bad_pin", "message"), + ( + ("incumbent_sha", "Incumbent diagnostics SHA-256 mismatch"), + ("target_surface", "target-surface SHA-256 mismatch"), + ("ssi_prior_basis", "prior-weight basis SHA-256 mismatch"), + ), +) +def test_incumbent_and_target_surface_pins_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + bad_pin: str, + message: str, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + (tmp_path / "ledger").mkdir() + incumbent = tmp_path / "incumbent.json" + incumbent.write_text( + json.dumps({"target_surface": {"sha256": "e" * 64}}), + encoding="utf-8", + ) + payload = _config_payload(pool_manifest_sha256=_sha256(manifest)) + payload["targets"]["incumbent_diagnostics_sha256"] = _sha256(incumbent) + prior_basis = tmp_path / "ssi.json" + prior_basis.write_text("{}", encoding="utf-8") + if bad_pin == "incumbent_sha": + payload["targets"]["incumbent_diagnostics_sha256"] = "d" * 64 + elif bad_pin == "target_surface": + payload["targets"]["target_surface_sha256"] = "f" * 64 + else: + payload["targets"]["ssi_take_up_prior_weight_basis"] = "ssi.json" + payload["targets"]["ssi_take_up_prior_weight_basis_sha256"] = "1" * 64 + config = launcher._read_config(_write_config(tmp_path, payload)) + monkeypatch.setattr( + launcher, + "load_simulation_ready_us_multispine_pool_manifest", + lambda _, **_kwargs: { + "publication_run_id": "fixture-publication", + "agreement_gate": {"passed": True}, + "provenance_counts": {"household": {"rows": 8}}, + }, + ) + + with pytest.raises(ValueError, match=message): + launcher._validate_pins_and_resolve_k( + config=config, + pool_manifest_path=manifest, + ) + + +def test_launcher_arguments_are_accepted_by_the_house_builder_parser( + tmp_path: Path, +) -> None: + launcher = _launcher_module() + payload = _config_payload( + requested_k=20_000, + release_id="populace-us-2024-k20000-fixture", + ) + payload["targets"]["ssi_take_up_prior_weight_basis"] = "ssi.json" + payload["targets"]["ssi_take_up_prior_weight_basis_sha256"] = "1" * 64 + config = launcher._read_config(_write_config(tmp_path, payload)) + + argv = launcher._builder_argv( + config=config, + pool_manifest=tmp_path / "pool.manifest.json", + out=tmp_path / "out", + k=20_000, + ) + parsed = launcher.fiscal_release._parse_args(argv) + + assert parsed.exact_k == 20_000 + assert parsed.seed == 17 + assert parsed.pool_release_id == "fixture-publication" + assert parsed.release_id == "populace-us-2024-k20000-fixture" + assert parsed.no_staging is True + assert parsed.ssi_take_up_prior_weight_basis == tmp_path / "ssi.json" + assert parsed.ssi_take_up_prior_weight_basis_sha256 == "1" * 64 + + +def test_launcher_delegates_to_house_builder_and_never_publishes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config_path = _write_config( + tmp_path, + _config_payload(pool_manifest_sha256=_sha256(manifest)), + ) + monkeypatch.setattr( + launcher, + "_validate_pins_and_resolve_k", + lambda **_: (8, {}), + ) + captured: list[str] = [] + + def fake_builder(argv): + captured.extend(argv) + return None + + result = launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=tmp_path / "out", + release_builder=fake_builder, + ) + + assert captured[captured.index("--exact-k") + 1] == "N" + assert captured[captured.index("--seed") + 1] == "17" + assert "--no-staging" in captured + assert captured[captured.index("--pool-manifest-sha256") + 1] == _sha256(manifest) + assert result["automatic_publish"] is False + assert result["release_dir"] == str( + tmp_path / "out" / "releases" / "populace-us-2024-k8-fixture" + ) + assert result["artifact_root"] == str(tmp_path / "out" / "artifacts") + assert result["pointer_update"] is False + assert result["pointer_updates"]["production"]["pointer_update"] is False + assert result["pointer_updates"]["staging"]["pointer_update"] is False + assert result["publish_argv"][-3:] == [ + "--create-tag", + "--no-latest", + "--tag-only", + ] + assert "--artifact-root" in result["publish_argv"] + assert "--repo-id policyengine/populace-us" in result["publish_command"] + assert json.loads((tmp_path / "out" / "package_result.json").read_text()) == ( + result + ) + + +def test_pool_release_id_must_match_authenticated_manifest_identity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + (tmp_path / "ledger").mkdir() + incumbent = tmp_path / "incumbent.json" + incumbent.write_text( + json.dumps({"target_surface": {"sha256": "e" * 64}}), + encoding="utf-8", + ) + payload = _config_payload(pool_manifest_sha256=_sha256(manifest)) + payload["pool"]["release_id"] = "invented-pool-release" + payload["targets"]["incumbent_diagnostics_sha256"] = _sha256(incumbent) + config = launcher._read_config(_write_config(tmp_path, payload)) + monkeypatch.setattr( + launcher, + "load_simulation_ready_us_multispine_pool_manifest", + lambda _, **_kwargs: { + "publication_run_id": "fixture-publication", + "agreement_gate": {"passed": True}, + "provenance_counts": {"household": {"rows": 8}}, + }, + ) + + with pytest.raises( + ValueError, + match="PoolReleaseIdentityMismatchError: configured pool release id", + ): + launcher._validate_pins_and_resolve_k( + config=config, + pool_manifest_path=manifest, + ) + + +def test_staging_credentials_cannot_enable_pointer_writes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + payload = _config_payload( + requested_k=20_000, + release_id="populace-us-2024-k20000-fixture", + ) + config = launcher._read_config(_write_config(tmp_path, payload)) + monkeypatch.setenv("HF_TOKEN", "credentialed-fixture") + monkeypatch.setenv("POPULACE_STAGING_REPO_ID", "fixture/staging") + constructed = False + + class UnexpectedTelemetry: + def __init__(self, **kwargs): + nonlocal constructed + constructed = True + + monkeypatch.setattr( + launcher.fiscal_release, + "StagingTelemetry", + UnexpectedTelemetry, + ) + argv = launcher._builder_argv( + config=config, + pool_manifest=tmp_path / "pool.manifest.json", + out=tmp_path / "out", + k=20_000, + ) + parsed = launcher.fiscal_release._parse_args(argv) + + telemetry = launcher.fiscal_release._staging_telemetry( + parsed, + release_root=tmp_path / "out", + release_id=config.release_id, + ) + + assert parsed.no_staging is True + assert telemetry is None + assert constructed is False + assert not (tmp_path / "out" / "staging").exists() diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index f20f310c..c75f9b61 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -1,4 +1,5 @@ import builtins +import hashlib import importlib.util import json import sys @@ -1258,6 +1259,232 @@ def test_weeks_unemployed_source_override_parses(monkeypatch) -> None: assert args.asec_2023_weeks_unemployed_source == Path("asecpub23csv.zip") +def _exact_k_builder_argv(k: str, *, seed: int | None = 17) -> list[str]: + argv = [ + "--pool-manifest", + "pool.manifest.json", + "--pool-manifest-sha256", + "a" * 64, + "--pool-release-id", + "fixture-publication", + "--exact-k", + k, + "--exact-k-pi-hi", + "0.95", + "--ledger-facts", + "facts", + "--ledger-facts-sha256", + "b" * 64, + "--ledger-manifest-sha256", + "c" * 64, + "--incumbent-diagnostics", + "incumbent.json", + "--incumbent-diagnostics-sha256", + "d" * 64, + "--frozen-target-surface-sha256", + "e" * 64, + "--out", + "out", + "--release-id", + "populace-us-2024-k20000-fixture", + "--no-staging", + ] + if seed is not None: + argv.extend(("--seed", str(seed))) + return argv + + +def test_builder_exact_k_parser_enforces_charter_and_explicit_seed(capsys) -> None: + builder = _load_builder_module() + + with pytest.raises(SystemExit): + builder._parse_args(_exact_k_builder_argv("57241")) + assert "ExactKCharterError" in capsys.readouterr().err + + with pytest.raises(SystemExit): + builder._parse_args(_exact_k_builder_argv("20000", seed=None)) + assert "ExactKExplicitSeedError" in capsys.readouterr().err + + parsed = builder._parse_args(_exact_k_builder_argv("N")) + assert parsed.exact_k == "N" + assert parsed.seed == 17 + assert parsed.no_staging is True + + +def test_builder_exact_k_requires_pointer_suppression(capsys) -> None: + builder = _load_builder_module() + argv = _exact_k_builder_argv("20000") + argv.remove("--no-staging") + + with pytest.raises(SystemExit): + builder._parse_args(argv) + + assert "ExactKPointerSuppressionError" in capsys.readouterr().err + + +def test_builder_pool_release_identity_is_manifest_authenticated() -> None: + builder = _load_builder_module() + + assert ( + builder._assert_pool_release_identity( + "fixture-publication", + {"publication_run_id": "fixture-publication"}, + ) + == "fixture-publication" + ) + with pytest.raises( + ValueError, + match="PoolReleaseIdentityMismatchError: configured pool release id", + ): + builder._assert_pool_release_identity( + "invented-release", + {"publication_run_id": "fixture-publication"}, + ) + assert "_assert_pool_release_id_value" in builder.main.__code__.co_names + + +def test_builder_rejects_replaced_authenticated_pool_h5_at_first_consumer( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from populace.build.us_runtime.h5_io import ( + AuthenticatedPoolH5MismatchError, + ) + + builder = _load_builder_module() + pool_h5 = tmp_path / "pool.h5" + authenticated_bytes = b"a" * 32 + replacement_bytes = b"b" * 32 + pool_h5.write_bytes(authenticated_bytes) + out = tmp_path / "out" + argv = _exact_k_builder_argv("20000") + argv[argv.index("out")] = str(out) + + monkeypatch.setattr(builder, "_git_dirty", lambda: False) + monkeypatch.setattr( + builder, + "_refuse_certified_release_dir_reuse", + lambda path: None, + ) + monkeypatch.setattr( + builder, + "_load_verified_incumbent_diagnostics_payload", + lambda path, *, expected_sha256: ({}, expected_sha256), + ) + + def fake_load_pool(path, *, expected_manifest_sha256): + authenticated = builder.AuthenticatedPoolH5( + path=pool_h5.resolve(), + sha256=hashlib.sha256(authenticated_bytes).hexdigest(), + size_bytes=len(authenticated_bytes), + publication_run_id="fixture-publication", + manifest_sha256=expected_manifest_sha256, + ) + pool_h5.write_bytes(replacement_bytes) + return ( + SimpleNamespace(), + {"publication_run_id": "fixture-publication"}, + authenticated, + ) + + monkeypatch.setattr( + builder, + "load_simulation_ready_us_multispine_pool", + fake_load_pool, + ) + monkeypatch.setattr( + builder, + "_assert_pool_release_id_value", + lambda *args: pytest.fail("pool identity ran after an H5 divergence"), + ) + + with pytest.raises( + AuthenticatedPoolH5MismatchError, + match="builder base dataset identity", + ): + builder.main(argv) + + assert not out.exists() + + +def test_authenticated_pool_h5_consumers_use_one_returned_identity() -> None: + import ast + import inspect + + builder = _load_builder_module() + source = Path(builder.__file__).read_text() + tree = ast.parse(source) + forbidden_base_hashes = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_sha256" + and node.args + and isinstance(node.args[0], ast.Name) + and node.args[0].id == "base_h5" + ] + assert forbidden_base_hashes == [] + assert "shutil.copy2(base_h5" not in source + assert 'pool_h5.get("sha256")' not in source + + main_source = inspect.getsource(builder.main) + diagnostics_source = inspect.getsource( + builder._write_release_calibration_diagnostics + ) + receipt_source = inspect.getsource(builder._exact_k_ladder_manifest_payload) + assert ( + "authenticated_pool_h5.verified_digest(\n" + ' consumer="builder base dataset identity"' + ) in main_source + assert "base_dataset_sha256=base_dataset_sha256" in main_source + assert '"base_dataset_sha256": base_dataset_sha256' in diagnostics_source + assert '"manifest_sha256": authenticated_pool_h5.manifest_sha256' in receipt_source + assert '"pool_h5_sha256": authenticated_pool_h5.sha256' in receipt_source + assert '"pool_h5_size_bytes": authenticated_pool_h5.size_bytes' in receipt_source + + +def test_builder_reconciles_exact_k_count_before_any_release_write() -> None: + import inspect + + builder = _load_builder_module() + source = inspect.getsource(builder.main) + count_gate = source.index("assert_exact_k_realized_count(ladder_outcome") + + for later_write in ( + "_write_release_calibration_diagnostics(", + "release_engine.write_dataset(", + "_write_npz(", + "_build_manifests(", + ): + assert count_gate < source.index(later_write) + + +def test_legacy_cli_result_is_origin_main_three_key_fixture( + capsys, + tmp_path: Path, +) -> None: + builder = _load_builder_module() + release_dir = tmp_path / "releases" / "fixture-release" + artifact_root = tmp_path / "artifacts" + + returned = builder._print_build_result( + release_id="fixture-release", + release_dir=release_dir, + artifact_root=artifact_root, + ) + + assert returned is None + assert capsys.readouterr().out == ( + "{\n" + ' "release_id": "fixture-release",\n' + f' "release_dir": "{release_dir}",\n' + f' "artifact_root": "{artifact_root}"\n' + "}\n" + ) + assert builder.main.__annotations__["return"] in {None, "None"} + + def test_frozen_support_selection_is_followed_by_weeks_unemployed_regate() -> None: builder = _load_builder_module() source = Path(builder.__file__).read_text(encoding="utf-8") @@ -3201,7 +3428,6 @@ def fake_write_calibration_diagnostics(result, path, *, target_registry, build): captured["build"] = build return path - monkeypatch.setattr(builder, "_sha256", lambda path: "base-sha") monkeypatch.setattr( builder, "write_calibration_diagnostics", fake_write_calibration_diagnostics ) @@ -3219,7 +3445,7 @@ def fake_write_calibration_diagnostics(result, path, *, target_registry, build): result=result, release_dir=tmp_path, registry=registry, - base_h5=tmp_path / "base.h5", + base_dataset_sha256="base-sha", compilation={"dropped_target_names": []}, target_profile_gate=profile_gate, health_input_gate=health_gate, @@ -3299,7 +3525,7 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( result=result, release_dir=tmp_path, registry=registry, - base_h5=base_h5, + base_dataset_sha256=builder._sha256(base_h5), compilation={"dropped_target_names": []}, target_profile_gate=passing_gate, health_input_gate=passing_gate, @@ -3319,7 +3545,7 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( @pytest.mark.parametrize( "terminal_mode", - ["merge", "integrity", "retirement", "crash", "telemetry"], + ["merge", "integrity", "retirement", "crash", "telemetry", "puf_tail"], ) def test_main_writes_diagnostics_before_post_calibration_gate_failure( monkeypatch, tmp_path, terminal_mode @@ -3344,14 +3570,23 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( ``telemetry``: live telemetry raises while attaching the already-written calibration diagnostics; the exception becomes a batch line and every later terminal gate group still evaluates before the terminal raise. + ``puf_tail``: exact-k selection loses the original PUF capital-gains tail; + the failure is batched while diagnostics and final-weight evidence remain, + every later terminal group runs, and release artifacts stay suppressed. """ builder = _load_builder_module() - release_id = "populace-us-2024-gate-failure-test" + release_id = ( + "populace-us-2024-k2-gate-failure-test" + if terminal_mode == "puf_tail" + else "populace-us-2024-gate-failure-test" + ) base_h5 = tmp_path / "base.h5" + pool_manifest = tmp_path / "pool.manifest.json" weeks_source = tmp_path / "asecpub23csv.zip" facts = tmp_path / "facts.jsonl" out = tmp_path / "out" base_h5.write_bytes(b"h5") + pool_manifest.write_text("fixture", encoding="utf-8") facts.write_text("{}\n") target_spec = TargetSpec( name="amount", @@ -3370,6 +3605,8 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( l0_lambda=0.2, n_nonzero=2, frame=SimpleNamespace(n=lambda entity: 2), + weights=np.asarray([12.0, 35.0]), + initial_weights=np.asarray([1.0, 1.0]), weight_entity="household", selection=SimpleNamespace(n_nonzero=2, final_loss=1.5), ) @@ -3385,7 +3622,19 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( class FakeFrame: def n(self, entity): assert entity == "household" - return 4 + return 2 if terminal_mode == "puf_tail" else 4 + + def table(self, entity): + assert entity == "household" + size = self.n("household") + return pd.DataFrame({"household_id": np.arange(1, size + 1, dtype="int64")}) + + def weights_for(self, entity): + assert entity == "household" + return Weights( + np.ones(self.n("household"), dtype=np.float64), + WeightKind.IMPORTANCE, + ) class FakeExportFrame: def n(self, entity): @@ -3403,21 +3652,71 @@ def table(self, entity): assert entity == "household" return pd.DataFrame({"household_id": np.asarray([10, 20], dtype="int64")}) - argv = [ - "build_us_fiscal_refresh_release.py", - "--base-h5", - str(base_h5), - "--ledger-facts", - str(facts), - "--out", - str(out), - "--release-id", - release_id, - "--asec-2023-weeks-unemployed-source", - str(weeks_source), - "--no-target-frame-checkpoint", - ] - if terminal_mode != "telemetry": + if terminal_mode == "puf_tail": + loss_basis = builder._fiscal_target_loss_basis(registry, np.ones(1)) + incumbent = tmp_path / "incumbent.json" + incumbent.write_text( + json.dumps( + { + "target_surface": {"sha256": "e" * 64, "n_targets": 0}, + "build": {"target_loss_basis": loss_basis}, + "targets": [], + } + ), + encoding="utf-8", + ) + incumbent_sha256 = ( + __import__("hashlib").sha256(incumbent.read_bytes()).hexdigest() + ) + argv = [ + "build_us_fiscal_refresh_release.py", + "--pool-manifest", + str(pool_manifest), + "--pool-manifest-sha256", + "a" * 64, + "--pool-release-id", + "fixture-publication", + "--exact-k", + "N", + "--exact-k-pi-hi", + "0.95", + "--seed", + "17", + "--ledger-facts", + str(facts), + "--ledger-facts-sha256", + "b" * 64, + "--ledger-manifest-sha256", + "c" * 64, + "--incumbent-diagnostics", + str(incumbent), + "--incumbent-diagnostics-sha256", + incumbent_sha256, + "--frozen-target-surface-sha256", + "e" * 64, + "--out", + str(out), + "--release-id", + release_id, + "--no-target-frame-checkpoint", + "--no-staging", + ] + else: + argv = [ + "build_us_fiscal_refresh_release.py", + "--base-h5", + str(base_h5), + "--ledger-facts", + str(facts), + "--out", + str(out), + "--release-id", + release_id, + "--asec-2023-weeks-unemployed-source", + str(weeks_source), + "--no-target-frame-checkpoint", + ] + if terminal_mode not in {"telemetry", "puf_tail"}: argv.append("--no-staging") if terminal_mode == "crash": # Nonexistent incumbent: the degraded-mode guard must record the @@ -3429,10 +3728,18 @@ def table(self, entity): ] monkeypatch.setattr(sys, "argv", argv) monkeypatch.setattr(builder, "_git_dirty", lambda: False) + + def fake_sha256(path): + if terminal_mode == "puf_tail" and Path(path) == pool_manifest: + return "a" * 64 + if Path(path) == weeks_source: + return "weeks-source-sha" + return "base-sha" + monkeypatch.setattr( builder, "_sha256", - lambda path: "weeks-source-sha" if Path(path) == weeks_source else "base-sha", + fake_sha256, ) monkeypatch.setattr(builder, "_git_output", lambda *args: "commit") if terminal_mode == "telemetry": @@ -3442,6 +3749,8 @@ class LiveTelemetry: def stage(self, stage, **details): captured.setdefault("telemetry_events", []).append(("stage", stage)) + if stage == "weeks_unemployed_input": + captured["weeks_unemployed_telemetry"] = dict(details) def attach_artifact(self, name, path, **details): captured.setdefault("telemetry_events", []).append( @@ -3472,7 +3781,7 @@ def complete(self): "_staging_telemetry", lambda *args, **kwargs: live_telemetry, ) - if terminal_mode in {"integrity", "retirement", "telemetry"}: + if terminal_mode in {"integrity", "retirement", "telemetry", "puf_tail"}: monkeypatch.setattr( builder, "PolicyEngineUSEngine", @@ -3589,6 +3898,44 @@ def fake_load_frame(path): return FakeFrame() monkeypatch.setattr(builder, "_load_frame", fake_load_frame) + if terminal_mode == "puf_tail": + + def fake_load_pool(path, *, expected_manifest_sha256): + assert path == pool_manifest + assert expected_manifest_sha256 == "a" * 64 + return ( + FakeFrame(), + { + "publication_run_id": "fixture-publication", + "agreement_gate": {"passed": True, "gates": {}}, + "pool_h5": { + "path": str(base_h5), + "sha256": "1" * 64, + "size_bytes": base_h5.stat().st_size, + }, + "agreement_diagnostics": {"sha256": "2" * 64}, + }, + builder.AuthenticatedPoolH5( + path=base_h5.resolve(), + sha256=__import__("hashlib") + .sha256(base_h5.read_bytes()) + .hexdigest(), + size_bytes=base_h5.stat().st_size, + publication_run_id="fixture-publication", + manifest_sha256="a" * 64, + ), + ) + + monkeypatch.setattr( + builder, + "load_simulation_ready_us_multispine_pool", + fake_load_pool, + ) + monkeypatch.setattr( + builder, + "_person_population", + lambda frame: builder.US_BASE_PERSON_POPULATION_BENCHMARK, + ) def fake_load_weeks_unemployed_source(path, **kwargs): captured["source_stage_events"].append("load_weeks_source") @@ -4727,6 +5074,43 @@ def fake_write_calibration_diagnostics(result, path, *, target_registry, build): return path monkeypatch.setattr(builder, "calibrate_l0_refit", fake_calibrate_l0_refit) + if terminal_mode == "puf_tail": + ladder_outcome = SimpleNamespace( + result=result, + support=np.asarray([0, 1], dtype=np.int64), + selected_inclusion_probabilities=np.ones(2), + selection_receipt={ + "k": 2, + "pi_hi": 0.95, + "seed": 17, + "certainty_count": 2, + "boundary_pool_size": 0, + "design": "full-pool", + }, + refit_baseline_diagnostics={"method": "fixture-full-pool"}, + ) + monkeypatch.setattr( + builder, + "calibrate_exact_k_ladder", + lambda *args, **kwargs: ladder_outcome, + ) + monkeypatch.setattr( + builder, + "_exact_k_puf_tail_support_gate", + lambda frame, support: builder.GateResult( + name="exact_k_puf_capital_gains_tail", + passed=False, + failures=("fixture PUF own-tail donor missing",), + details={"status": "failed"}, + ), + ) + monkeypatch.setattr( + builder, + "diagnostics_payload", + lambda result, *, target_registry: { + "target_surface": {"sha256": "e" * 64, "n_targets": 0} + }, + ) def fake_l0_refit_weights(frame, refit_result): captured["export_frame_from_l0_refit"] = True @@ -4761,6 +5145,12 @@ def fake_final_medicaid_diagnostics( return {} monkeypatch.setattr(builder, "_with_l0_refit_weights", fake_l0_refit_weights) + if terminal_mode == "puf_tail": + monkeypatch.setattr( + builder, + "_with_calibrated_weights", + lambda frame, weights: FakeExportFrame(), + ) monkeypatch.setattr( builder, "us_ssi_take_up_diagnostics", @@ -4776,7 +5166,7 @@ def fake_ssi_delivery_gate(diagnostics, *, targets, enforcement_fences=None): # own early failure. Other modes retain the populace#547 delivery # cofailure and its written retry basis. captured.setdefault("ssi_event_order", []).append("delivery_gate") - passes = terminal_mode in {"integrity", "retirement"} + passes = terminal_mode in {"integrity", "retirement", "puf_tail"} return builder.GateResult( name="ssi_take_up_delivery", passed=passes, @@ -4857,7 +5247,13 @@ def fake_release_gate_failures(*args, **kwargs): # coverage/parity evaluation errors on the fake frame may append # further lines after them. message = str(exc) - if terminal_mode == "retirement": + if terminal_mode == "puf_tail": + assert message.startswith( + "Release gates failed: Exact-k PUF capital-gains tail failed: " + "fixture PUF own-tail donor missing" + ) + assert "SSI take-up delivery failed:" not in message + elif terminal_mode == "retirement": assert message.startswith( "Release gates failed: Retirement-distribution signal failed: " f"{retirement_missing_failure}" @@ -4904,7 +5300,13 @@ def fake_release_gate_failures(*args, **kwargs): written_diagnostics = json.loads( (release_dir / "calibration_diagnostics.json").read_text() ) - if terminal_mode == "retirement": + if terminal_mode == "puf_tail": + assert ( + "Exact-k PUF capital-gains tail failed: " + "fixture PUF own-tail donor missing" + in written_diagnostics["build"]["release_gates"]["failures"] + ) + elif terminal_mode == "retirement": assert ( "Retirement-distribution signal failed: " f"{retirement_missing_failure}" @@ -4992,14 +5394,33 @@ def fake_release_gate_failures(*args, **kwargs): assert not list(release_dir.glob("*manifest*")) if terminal_mode == "telemetry": assert captured["telemetry_crashed"] is True - if terminal_mode in {"integrity", "retirement", "telemetry"}: + assert captured["weeks_unemployed_telemetry"] == { + "message": ( + "Restored measured ASEC LKWEEKS before frozen-support " + "selection and target materialization." + ), + "source_path": str(weeks_source.resolve()), + "source_sha256": builder.ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, + "source_rows": 2, + } + if terminal_mode in {"integrity", "retirement", "telemetry", "puf_tail"}: assert captured["terminal_gate_events"] == [ "input_coverage", "input_mass_parity", "qrf_tail_concentration", ] if terminal_mode != "crash": - if terminal_mode == "retirement": + if terminal_mode == "puf_tail": + expected_gate_failures = [ + "Exact-k PUF capital-gains tail failed: " + "fixture PUF own-tail donor missing", + "Other health insurance signal failed on the export frame: " + "premiums signal flattened [cofailure-sentinel]", + "Exact-k frozen-register fit failed: Exact-k frozen-register " + "comparison has no complete candidate target rows.", + "ctc failed", + ] + elif terminal_mode == "retirement": expected_gate_failures = [ f"Retirement-distribution signal failed: {retirement_missing_failure}", "Other health insurance signal failed on the export frame: " @@ -5043,6 +5464,15 @@ def fake_release_gate_failures(*args, **kwargs): "ctc failed", ] assert captured["diagnostics"]["gate_failures"] == expected_gate_failures + if terminal_mode == "puf_tail": + assert captured["diagnostics"]["exact_k_ladder"]["invariant_battery"][ + "puf_capital_gains_tail" + ] == { + "passed": False, + "failures": ["fixture PUF own-tail donor missing"], + "details": {"status": "failed"}, + } + return else: # Corridor order: SSI delivery + basis note, health-input crash # guard, other-health gate failure, incumbent guard, release-gate @@ -5464,6 +5894,293 @@ def test_incumbent_diagnostics_must_match_current_target_surface(tmp_path) -> No ) +def test_exact_k_gate_requires_strict_weighted_loss_improvement() -> None: + builder = _load_builder_module() + specs = ( + TargetSpec( + name="fixture/one", + entity="household", + value=100.0, + measure="one", + period=builder.PERIOD, + source="fixture", + family="fixture_a", + ), + TargetSpec( + name="fixture/two", + entity="household", + value=200.0, + measure="two", + period=builder.PERIOD, + source="fixture", + family="fixture_b", + ), + ) + registry = TargetRegistry(specs, country="us") + names = tuple(builder._target_row_name(spec) for spec in specs) + loss_weights = np.asarray([1.0, 3.0]) + + def result(estimates: tuple[float, float]): + targets = np.asarray([100.0, 200.0]) + final = np.asarray(estimates) + return SimpleNamespace( + diagnostics=tuple( + SimpleNamespace(name=name, target=target, final_estimate=estimate) + for name, target, estimate in zip(names, targets, final, strict=True) + ), + final_loss=builder.relative_error_loss( + final, + targets, + target_loss_weights=loss_weights, + target_loss_cap=builder.US_FISCAL_TARGET_LOSS_CAP, + ), + ) + + incumbent = { + names[0]: {"target": 100.0, "final_estimate": 120.0}, + names[1]: {"target": 200.0, "final_estimate": 240.0}, + } + loss_basis = builder._fiscal_target_loss_basis(registry, loss_weights) + passing = builder._exact_k_frozen_register_fit_gate( + result((110.0, 220.0)), + incumbent, + target_registry=registry, + target_loss_weights=loss_weights, + configured_loss_basis=loss_basis, + incumbent_loss_basis=loss_basis, + ) + tied = builder._exact_k_frozen_register_fit_gate( + result((120.0, 240.0)), + incumbent, + target_registry=registry, + target_loss_weights=loss_weights, + configured_loss_basis=loss_basis, + incumbent_loss_basis=loss_basis, + ) + + assert passing.passed + assert passing.details["candidate_loss"] < passing.details["incumbent_loss"] + assert passing.details["strict_improvement_required"] is True + assert not tied.passed + assert "did not beat the incumbent" in tied.failures[0] + + +def test_exact_k_gate_rejects_incumbent_weight_swap_that_flips_verdict() -> None: + """The r1 [1, 10] versus [10, 1] loss-basis flip must fail closed.""" + builder = _load_builder_module() + specs = ( + TargetSpec( + name="fixture/one", + entity="household", + value=100.0, + measure="one", + period=builder.PERIOD, + source="fixture", + family="fixture_a", + ), + TargetSpec( + name="fixture/two", + entity="household", + value=100.0, + measure="two", + period=builder.PERIOD, + source="fixture", + family="fixture_b", + ), + ) + registry = TargetRegistry(specs, country="us") + names = tuple(builder._target_row_name(spec) for spec in specs) + configured_weights = np.asarray([1.0, 10.0]) + incumbent_weights = np.asarray([10.0, 1.0]) + targets = np.asarray([100.0, 100.0]) + candidate_estimates = np.asarray([120.0, 100.0]) + candidate = SimpleNamespace( + diagnostics=tuple( + SimpleNamespace(name=name, target=target, final_estimate=estimate) + for name, target, estimate in zip( + names, + targets, + candidate_estimates, + strict=True, + ) + ), + final_loss=builder.relative_error_loss( + candidate_estimates, + targets, + target_loss_weights=configured_weights, + target_loss_cap=builder.US_FISCAL_TARGET_LOSS_CAP, + ), + ) + incumbent = { + name: {"target": target, "final_estimate": 110.0} + for name, target in zip(names, targets, strict=True) + } + + gate = builder._exact_k_frozen_register_fit_gate( + candidate, + incumbent, + target_registry=registry, + target_loss_weights=configured_weights, + configured_loss_basis=builder._fiscal_target_loss_basis( + registry, + configured_weights, + ), + incumbent_loss_basis=builder._fiscal_target_loss_basis( + registry, + incumbent_weights, + ), + ) + + assert candidate.final_loss == pytest.approx(0.01818181818181818) + assert not gate.passed + assert gate.details["candidate_loss"] is None + assert gate.details["incumbent_loss"] is None + assert gate.failures[0].startswith("IncumbentLossBasisMismatchError:") + + +def test_exact_k_gate_rejects_different_recorded_loss_basis_metadata() -> None: + """A surface-identical incumbent cannot substitute weighting metadata.""" + builder = _load_builder_module() + spec = TargetSpec( + name="fixture/one", + entity="household", + value=100.0, + measure="one", + period=builder.PERIOD, + source="fixture", + family="fixture_a", + ) + registry = TargetRegistry((spec,), country="us") + name = builder._target_row_name(spec) + candidate = SimpleNamespace( + diagnostics=(SimpleNamespace(name=name, target=100.0, final_estimate=100.0),), + final_loss=0.0, + ) + configured_basis = builder._fiscal_target_loss_basis( + registry, + np.ones(1), + ) + incumbent_basis = { + **configured_basis, + "target_loss_weighting": "different_weighting_version", + "target_loss_family_multipliers": {"fixture_a": 999.0}, + "target_loss_cap": 0.01, + } + + gate = builder._exact_k_frozen_register_fit_gate( + candidate, + {name: {"target": 100.0, "final_estimate": 200.0}}, + target_registry=registry, + target_loss_weights=np.ones(1), + configured_loss_basis=configured_basis, + incumbent_loss_basis=incumbent_basis, + ) + + assert not gate.passed + assert gate.failures[0].startswith("IncumbentLossBasisMismatchError:") + + +def test_verified_incumbent_bytes_survive_post_verification_replacement( + tmp_path: Path, +) -> None: + """The r1 strong-to-weak replacement cannot change the scored incumbent.""" + builder = _load_builder_module() + spec = TargetSpec( + name="fixture/one", + entity="household", + value=100.0, + measure="one", + period=builder.PERIOD, + source="fixture", + family="fixture", + ) + registry = TargetRegistry((spec,), country="us") + name = builder._target_row_name(spec) + loss_weights = np.ones(1) + loss_basis = builder._fiscal_target_loss_basis(registry, loss_weights) + incumbent_path = tmp_path / "incumbent.json" + strong_payload = { + "target_surface": {"sha256": "a" * 64}, + "build": {"target_loss_basis": loss_basis}, + "targets": [ + {"name": name, "target": 100.0, "final_estimate": 105.0}, + ], + } + incumbent_path.write_text(json.dumps(strong_payload), encoding="utf-8") + expected_sha256 = ( + __import__("hashlib").sha256(incumbent_path.read_bytes()).hexdigest() + ) + + pinned_payload, observed_sha256 = ( + builder._load_verified_incumbent_diagnostics_payload( + incumbent_path, + expected_sha256=expected_sha256, + ) + ) + weak_payload = { + **strong_payload, + "targets": [ + {"name": name, "target": 100.0, "final_estimate": 200.0}, + ], + } + incumbent_path.write_text(json.dumps(weak_payload), encoding="utf-8") + candidate = SimpleNamespace( + diagnostics=(SimpleNamespace(name=name, target=100.0, final_estimate=110.0),), + final_loss=0.1, + ) + + gate = builder._exact_k_frozen_register_fit_gate( + candidate, + builder._diagnostics_by_target_name(pinned_payload, path=incumbent_path), + target_registry=registry, + target_loss_weights=loss_weights, + configured_loss_basis=loss_basis, + incumbent_loss_basis=pinned_payload["build"]["target_loss_basis"], + ) + + assert observed_sha256 == expected_sha256 + assert not gate.passed + assert gate.details["incumbent_loss"] == pytest.approx(0.05) + assert "did not beat the incumbent" in gate.failures[0] + + +def test_exact_k_frozen_register_gate_fails_closed_on_row_mismatch() -> None: + builder = _load_builder_module() + spec = TargetSpec( + name="fixture/one", + entity="household", + value=100.0, + measure="one", + period=builder.PERIOD, + source="fixture", + family="fixture", + ) + registry = TargetRegistry((spec,), country="us") + name = builder._target_row_name(spec) + result = SimpleNamespace( + diagnostics=(SimpleNamespace(name=name, target=100.0, final_estimate=100.0),), + final_loss=0.0, + ) + + gate = builder._exact_k_frozen_register_fit_gate( + result, + {}, + target_registry=registry, + target_loss_weights=np.ones(1), + configured_loss_basis=builder._fiscal_target_loss_basis( + registry, + np.ones(1), + ), + incumbent_loss_basis=builder._fiscal_target_loss_basis( + registry, + np.ones(1), + ), + ) + + assert not gate.passed + assert "do not equal" in gate.failures[0] + + def test_legacy_cd_provenance_requires_crosswalk_metadata() -> None: scorer = _load_scorer_module() @@ -7655,6 +8372,230 @@ def __len__(self): ) +def test_build_manifests_uses_loadable_paths_and_round_trips_exact_count_receipt( + monkeypatch, tmp_path +) -> None: + builder = _load_builder_module() + release_id = "populace-us-2024-k57240-fixture" + dataset_key = "populace_us_2024" + calibration_key = "populace_us_2024_calibration" + dataset_filename = builder.DATASET_FILENAME + calibration_filename = builder.CALIBRATION_FILENAME + release_dir = tmp_path / "release" / release_id + release_dir.mkdir(parents=True) + artifact_root = tmp_path / "artifacts" + artifact_root.mkdir() + (artifact_root / dataset_filename).write_bytes(b"h5") + (artifact_root / calibration_filename).write_bytes(b"npz") + (release_dir / "calibration_diagnostics.json").write_text("{}") + (release_dir / "us_source_coverage.json").write_text("{}") + (release_dir / "us_ssi_take_up.json").write_text("{}") + monkeypatch.setattr( + builder, + "_runtime_versions", + lambda: { + "python": "3.14.0", + "populace-data": "0.1.0", + "policyengine-core": "3.26.11", + "policyengine-us": "1.752.2", + }, + ) + monkeypatch.setattr(builder, "_git_output", lambda *args: "a" * 40) + monkeypatch.setattr( + builder, + "diagnostics_payload", + lambda result, target_registry: { + "initial_loss": 2.0, + "final_loss": 1.0, + "fraction_within_10pct": 1.0, + "target_surface": {"sha256": "b" * 64, "n_targets": 1}, + }, + ) + selection_receipt = { + "k": 57_240, + "pi_hi": 0.95, + "seed": 17, + "certainty_count": 3, + "boundary_pool_size": 100, + "design": "sampford", + } + ladder = { + "k": 57_240, + "seed": 17, + "selection_receipt": selection_receipt, + "refit_baseline_diagnostics": { + "method": "normalized_horvitz_thompson_w_over_q" + }, + "pool": { + "release_id": "fixture-pool", + "release_id_source": "pool_manifest.publication_run_id", + "manifest_sha256": "c" * 64, + "pool_h5_sha256": "d" * 64, + }, + "agreement_gate_reference": { + "passed": True, + "diagnostics_sha256": "e" * 64, + }, + "frozen_target_register": { + "target_surface_sha256": "b" * 64, + "incumbent_diagnostics_sha256": "f" * 64, + "incumbent_fit": { + "passed": True, + "failures": [], + "details": { + "candidate_loss": 0.1, + "incumbent_loss": 0.2, + }, + }, + }, + "invariant_battery": { + "puf_capital_gains_tail": { + "passed": True, + "failures": [], + "details": {"status": "retained"}, + } + }, + } + + builder._build_manifests( + dataset_key=dataset_key, + dataset_filename=dataset_filename, + calibration_key=calibration_key, + calibration_filename=calibration_filename, + exact_k_ladder=ladder, + **_minimal_manifest_kwargs(builder, release_id, release_dir, artifact_root), + ) + + build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) + release_manifest = json.loads((release_dir / "release_manifest.json").read_text()) + assert build_manifest["exact_k_ladder"] == ladder + assert ( + build_manifest["gates"]["exact_k_frozen_register_fit"] + == (ladder["frozen_target_register"]["incumbent_fit"]) + ) + assert ( + build_manifest["gates"]["exact_k_puf_capital_gains_tail"] + == (ladder["invariant_battery"]["puf_capital_gains_tail"]) + ) + assert release_manifest["build"]["exact_k_ladder"] == ladder + assert ( + release_manifest["build"]["exact_k_ladder"]["selection_receipt"] + == selection_receipt + ) + assert build_manifest["dataset"]["filename"] == dataset_filename + assert build_manifest["calibration"]["filename"] == calibration_filename + assert release_manifest["default_datasets"] == {"national": dataset_key} + assert release_manifest["artifacts"][dataset_key]["path"] == dataset_filename + assert ( + release_manifest["artifacts"][calibration_key]["path"] == calibration_filename + ) + + +def test_pool_owned_fiscal_transforms_are_guarded_for_prepared_pool_input() -> None: + """The pool is post-agreement input, so its owned producers run only legacy.""" + import ast + + from populace.build.us_runtime.multispine_pool import POOL_OPERATOR_CONTRACTS + + builder = _load_builder_module() + tree = ast.parse(Path(builder.__file__).read_text()) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "main" + ) + + def call_name(call: ast.Call) -> str | None: + return getattr(call.func, "id", None) or getattr(call.func, "attr", None) + + all_calls = { + name + for call in ast.walk(main_fn) + if isinstance(call, ast.Call) + if (name := call_name(call)) is not None + } + pool_owned_call_sites = { + (name, call.lineno) + for call in ast.walk(main_fn) + if isinstance(call, ast.Call) + if (name := call_name(call)) in POOL_OPERATOR_CONTRACTS + } + guarded_calls: set[str] = set() + guarded_call_sites: set[tuple[str, int]] = set() + for node in ast.walk(main_fn): + if ( + not isinstance(node, ast.If) + or ast.unparse(node.test) != "pool_frame is None" + ): + continue + guarded_calls.update( + name + for statement in node.body + for call in ast.walk(statement) + if isinstance(call, ast.Call) + if (name := call_name(call)) is not None + ) + guarded_call_sites.update( + (name, call.lineno) + for statement in node.body + for call in ast.walk(statement) + if isinstance(call, ast.Call) + if (name := call_name(call)) in POOL_OPERATOR_CONTRACTS + ) + + pool_owned_fiscal_calls = set(POOL_OPERATOR_CONTRACTS) & all_calls + assert pool_owned_fiscal_calls + assert pool_owned_fiscal_calls <= guarded_calls + assert pool_owned_call_sites == guarded_call_sites + assert { + "with_us_weeks_unemployed", + "with_us_qbi_input_reconciliation", + "with_us_childcare_inputs", + "with_us_energy_subsidy_input", + "with_us_retirement_contribution_inputs", + "with_us_immigration_inputs", + "with_us_take_up_inputs", + "with_us_hours_worked_inputs", + "with_us_relationship_inputs", + "with_us_medicare_take_up_input", + "with_us_retirement_distribution_inputs", + "with_us_eligibility_inputs", + "with_us_education_inputs", + "with_us_pregnancy_inputs", + "with_us_wic_claim_input", + } <= guarded_calls + assert "_with_snap_state_take_up_outputs" in all_calls + assert "_with_snap_state_take_up_outputs" not in guarded_calls + + +def test_exact_k_selection_batches_original_puf_tail_failure(monkeypatch) -> None: + builder = _load_builder_module() + marker = object() + monkeypatch.setattr( + builder, + "_exact_k_original_support_frame", + lambda frame, support: marker, + ) + + def fail_tail(frame, selected, *, require_present): + assert frame is marker + assert selected is marker + assert require_present is True + raise ValueError("fixture tail donor missing") + + monkeypatch.setattr( + builder, + "assert_puf_capital_gains_tail_survives_selection", + fail_tail, + ) + + gate = builder._exact_k_puf_tail_support_gate(marker, np.asarray([0])) + + assert not gate.passed + assert gate.failures == ("fixture tail donor missing",) + assert gate.details["status"] == "failed" + + def test_build_manifests_records_selection_source_provenance( monkeypatch, tmp_path ) -> None: diff --git a/packages/populace-build/tests/test_us_multispine_pool_h5_io.py b/packages/populace-build/tests/test_us_multispine_pool_h5_io.py new file mode 100644 index 00000000..524bf33e --- /dev/null +++ b/packages/populace-build/tests/test_us_multispine_pool_h5_io.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import populace.build.us_runtime.h5_io as h5_io +from populace.build.us_runtime.h5_io import ( + US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND, + US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND, + US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + AuthenticatedPoolH5MismatchError, + load_simulation_ready_us_multispine_pool, + write_nullable_us_h5, +) +from populace.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _pool_frame() -> Frame: + ids = np.asarray([10, 20, 30], dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + **{ + US_SCHEMA.membership_column(entity): ids + for entity in US_SCHEMA.group_entities + }, + "nullable_input": np.asarray([True, None, False], dtype=object), + } + ) + tables = { + "person": person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + } + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray([2.0, 3.0, 5.0]), + WeightKind.IMPORTANCE, + ) + }, + ) + + +def _write_ready_pool(tmp_path: Path) -> Path: + run_id = "fixture-publication" + pool_path = tmp_path / "pool.h5" + diagnostics_path = tmp_path / "pool.agreement.json" + manifest_path = tmp_path / "pool.manifest.json" + agreement_gate = { + "passed": True, + "gates": { + "us_spine_agreement": { + "passed": True, + "failures": [], + "details": {"fixture": True}, + } + }, + } + write_nullable_us_h5( + _pool_frame(), + pool_path, + period=2024, + artifact_kind=US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + publication_run_id=run_id, + ) + diagnostics_path.write_text( + json.dumps( + { + "artifact_kind": (US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND), + "schema_version": US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + "simulation_ready": True, + "publication_run_id": run_id, + "agreement_gate": agreement_gate, + } + ), + encoding="utf-8", + ) + manifest_path.write_text( + json.dumps( + { + "artifact_kind": US_MULTISPINE_POOL_MANIFEST_ARTIFACT_KIND, + "schema_version": US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, + "status": "simulation_ready", + "simulation_ready": True, + "publication_run_id": run_id, + "period": 2024, + "stage_checkpoints": { + "agreement": { + "source": "always_fresh", + "cached": False, + "terminal_verdict_persisted": False, + } + }, + "agreement_gate": agreement_gate, + "provenance_counts": {"household": {"rows": 3}}, + "pool_h5": { + "path": str(pool_path.resolve()), + "sha256": _sha256(pool_path), + "size_bytes": pool_path.stat().st_size, + "artifact_kind": US_MULTISPINE_POOL_H5_ARTIFACT_KIND, + "publication_run_id": run_id, + }, + "agreement_diagnostics": { + "path": str(diagnostics_path.resolve()), + "sha256": _sha256(diagnostics_path), + "publication_run_id": run_id, + }, + } + ), + encoding="utf-8", + ) + return manifest_path + + +def test_ready_pool_loader_preserves_importance_weights_and_nullable_inputs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + expected_manifest_sha256 = _sha256(manifest_path) + original_read_bytes = Path.read_bytes + manifest_reads = 0 + + def replace_after_pinned_read(path: Path) -> bytes: + nonlocal manifest_reads + raw = original_read_bytes(path) + if path == manifest_path: + manifest_reads += 1 + replacement = json.loads(raw) + replacement["publication_run_id"] = "replacement-publication" + path.write_text(json.dumps(replacement), encoding="utf-8") + return raw + + monkeypatch.setattr(Path, "read_bytes", replace_after_pinned_read) + + frame, manifest, authenticated_pool_h5 = load_simulation_ready_us_multispine_pool( + manifest_path, + expected_manifest_sha256=expected_manifest_sha256, + ) + + weights = frame.weights_for("household") + assert weights.kind is WeightKind.IMPORTANCE + np.testing.assert_array_equal(weights.values, [2.0, 3.0, 5.0]) + assert frame.table("person")["nullable_input"].tolist() == [True, None, False] + assert frame.n("household") == 3 + assert manifest["publication_run_id"] == "fixture-publication" + assert authenticated_pool_h5.path == Path(manifest["pool_h5"]["path"]) + assert authenticated_pool_h5.sha256 == manifest["pool_h5"]["sha256"] + assert authenticated_pool_h5.size_bytes == manifest["pool_h5"]["size_bytes"] + assert authenticated_pool_h5.publication_run_id == "fixture-publication" + assert authenticated_pool_h5.manifest_sha256 == expected_manifest_sha256 + assert manifest_reads == 1 + assert json.loads(manifest_path.read_text())["publication_run_id"] == ( + "replacement-publication" + ) + + +def test_ready_pool_loader_rejects_a_false_h5_size_receipt(tmp_path: Path) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["pool_h5"]["size_bytes"] += 1 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="pool_h5 size_bytes .* does not match"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_authenticated_pool_h5_copy_rejects_a_raced_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + _, _, authenticated_pool_h5 = load_simulation_ready_us_multispine_pool( + manifest_path + ) + replacement = bytearray(authenticated_pool_h5.path.read_bytes()) + replacement[0] ^= 1 + original_copy = h5_io._copy_file_bytes + + def replace_then_copy(source: Path, destination: Path) -> None: + source.write_bytes(replacement) + original_copy(source, destination) + + monkeypatch.setattr(h5_io, "_copy_file_bytes", replace_then_copy) + destination = tmp_path / "audit" / "base_pool.h5" + + with pytest.raises( + AuthenticatedPoolH5MismatchError, + match="builder final local-audit copy.*copied bytes", + ): + authenticated_pool_h5.copy_verified_to( + destination, + consumer="builder final local-audit copy", + ) + + assert not destination.exists() + assert not list(destination.parent.glob(".*.tmp")) + + +def test_ready_pool_loader_reconciles_manifest_and_h5_household_counts( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["provenance_counts"]["household"]["rows"] = 4 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="household row count 4.*H5 count 3"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_pool_loader_requires_explicitly_green_agreement_receipt( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["agreement_gate"]["passed"] = False + diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + diagnostics["agreement_gate"]["passed"] = False + diagnostics_path.write_text(json.dumps(diagnostics), encoding="utf-8") + manifest["agreement_diagnostics"]["sha256"] = _sha256(diagnostics_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="no passing agreement-gate verdict"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_pool_loader_binds_diagnostics_agreement_verdict( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + diagnostics["agreement_gate"]["gates"]["us_spine_agreement"]["details"] = { + "fixture": False + } + diagnostics_path.write_text(json.dumps(diagnostics), encoding="utf-8") + manifest["agreement_diagnostics"]["sha256"] = _sha256(diagnostics_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="verdict does not match"): + load_simulation_ready_us_multispine_pool(manifest_path) diff --git a/packages/populace-build/tests/test_us_spine_blindness.py b/packages/populace-build/tests/test_us_spine_blindness.py index d6fa1bfa..bc9e7bd9 100644 --- a/packages/populace-build/tests/test_us_spine_blindness.py +++ b/packages/populace-build/tests/test_us_spine_blindness.py @@ -186,6 +186,7 @@ "education_assistance_source.py", "eligibility_inputs.py", "engine_lifecycle.py", + "exact_k_ladder.py", # Calibration/selection seam; no source treatment. "fiscal_targets.py", "geography_ladder.py", "hours_worked.py", diff --git a/packages/populace-data/README.md b/packages/populace-data/README.md index 234737c9..ab1ae563 100644 --- a/packages/populace-data/README.md +++ b/packages/populace-data/README.md @@ -65,7 +65,7 @@ strengths and gaps are documented on the dataset card and the [populace.dev dashboard](https://populace.dev/dashboard). Historical incumbent benchmark comparisons live outside this package. -## Release Contract +## Release contract Published releases live under `releases//` in the Hub dataset repo. Each release must include `build_manifest.json`, `release_manifest.json`, and @@ -77,3 +77,10 @@ using PEP 440 specifiers. Use `latest.json` to discover the current release and its contract file paths; use the release id/tag in artifact revisions when loading an immutable release. +An exact-k ladder candidate published with +`--create-tag --no-latest --tag-only` changes neither the main branch nor +`latest.json`, so it does not replace the release selected by the default +`load("us")` resolution or its canonical root artifacts. Inspect it through its +explicit release id or Hub tag until a separate promotion updates `latest.json`. +The exact-k build also runs with `--no-staging`, so it does not create or update +a staging pointer. diff --git a/packages/populace-data/src/populace/data/publish_cli.py b/packages/populace-data/src/populace/data/publish_cli.py index 60a7c0a6..b4046ba1 100644 --- a/packages/populace-data/src/populace/data/publish_cli.py +++ b/packages/populace-data/src/populace/data/publish_cli.py @@ -64,7 +64,10 @@ def main(argv: list[str] | None = None) -> int: "--create-tag", action="store_true", default=True, - help="Create a Hugging Face tag for the release before updating latest.json.", + help=( + "Create a Hugging Face tag for the immutable release before any " + "main-branch update." + ), ) parser.add_argument( "--no-create-tag", @@ -83,7 +86,7 @@ def main(argv: list[str] | None = None) -> int: "--extra-file", action="append", default=[], - help="Additional release-dir file to upload before latest.json.", + help="Additional release-dir file to include in the immutable release.", ) parser.add_argument( "--updated-at", @@ -97,6 +100,15 @@ def main(argv: list[str] | None = None) -> int: "immutable tag, but never touch latest.json (the default pointer)." ), ) + parser.add_argument( + "--tag-only", + action="store_true", + help=( + "Publish only the immutable tag revision, without any main-branch " + "commit. Requires --no-latest and tag creation; used by exact-k " + "ladder candidates." + ), + ) parser.add_argument( "--allow-incomplete-reform-validation", action="store_true", @@ -108,6 +120,11 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) + if args.tag_only and not args.no_latest: + parser.error("--tag-only requires --no-latest.") + if args.tag_only and not args.create_tag: + parser.error("--tag-only requires tag creation; remove --no-create-tag.") + if not args.allow_incomplete_reform_validation and _reform_validation_skipped( Path(args.release_dir) ): @@ -139,6 +156,7 @@ def main(argv: list[str] | None = None) -> int: extra_files=tuple(args.extra_file), updated_at=args.updated_at, update_latest=not args.no_latest, + tag_only=args.tag_only, ) print(json.dumps(pointer, indent=2)) diff --git a/packages/populace-data/src/populace/data/release.py b/packages/populace-data/src/populace/data/release.py index 7597bbfb..89eaaaa7 100644 --- a/packages/populace-data/src/populace/data/release.py +++ b/packages/populace-data/src/populace/data/release.py @@ -12,8 +12,9 @@ - :func:`publish_release` is the producer: it validates the local release directory against the :mod:`release contract ` (a release that fails the contract refuses to publish), uploads its - files, and uploads ``latest.json`` **last** — so a reader never sees the - pointer before the release it points at. + files, and either stops at an immutable tag (the exact-k candidate lane) or + uploads ``latest.json`` **last** — so a reader never sees the pointer before + the release it points at. - :func:`latest_release` is the consumer: it downloads ``latest.json`` and returns the typed pointer, the one-call answer to "which release is current?" for dashboards and scorers. @@ -123,17 +124,19 @@ def publish_release( extra_files: tuple[str, ...] = (), updated_at: str | None = None, update_latest: bool = True, + tag_only: bool = False, notify: bool = True, ) -> dict: - """Upload a release directory and point ``latest.json`` at it. + """Publish a release directory and optionally point ``latest.json`` at it. The order is the guarantee: the release contract is validated first (an invalid release never reaches the Hub), then an immutable branch commit is - created with every release file and artifact and tagged. Only after that - certificate exists does one atomic main-branch commit update the release - copies, mutable root conveniences, and ``latest.json`` (the final - operation). Backends without the branch and atomic-commit surface are - refused before any remote mutation. + created with every release file and artifact and tagged. A tag-only publish + stops there. Otherwise, only after that certificate exists does one atomic + main-branch commit update the release copies, mutable root conveniences, + and, for standard publication, ``latest.json`` (the final operation). + Backends without the branch and atomic-commit surface are refused before + any remote mutation. Args: release_dir: Local ``releases/`` directory. @@ -154,6 +157,15 @@ def publish_release( extra_files: Additional filenames in ``release_dir`` to upload beyond the contract files (e.g. a diagnostics artifact). updated_at: Pointer timestamp; defaults to now (UTC). + update_latest: Update the production ``latest.json`` pointer after the + immutable release tag is created. ``False`` preserves the legacy + non-default publication behavior: release copies and root artifacts + are still committed to main, but the pointer is not. + tag_only: Publish only the immutable tagged revision, with no main-branch + commit. Requires ``update_latest=False`` and ``create_tag=True``. + This is the exact-k candidate lane; it is separate from legacy + ``update_latest=False`` publication so existing non-default releases + retain their main-branch copies. notify: Post a best-effort Slack release alert once ``latest.json`` is live (no-op unless the country ``SLACK_WEBHOOK_POPULACE_*`` env var is set; never fatal). Coupling the alert to the promotion here means @@ -163,7 +175,8 @@ def publish_release( ``False`` to suppress it (tests, dry-runs, re-publishes). Returns: - The ``latest.json`` payload that was published. + The release's ``latest.json`` payload. It is uploaded only when + ``update_latest=True``. Raises: ReleaseContractError: If the release directory violates the @@ -184,6 +197,16 @@ def publish_release( "never update latest.json. Pass update_latest=False " "(publish CLI: --no-latest)." ) + if tag_only and update_latest: + raise ValueError( + "tag_only=True requires update_latest=False; a tag-only publication " + "cannot update latest.json." + ) + if tag_only and not create_tag: + raise ValueError( + "tag_only=True requires create_tag=True; deleting the staging branch " + "without a tag would leave no published revision." + ) artifact_root = Path(artifact_root) if artifact_root is not None else None if role == NATIONAL_DEFAULT_DATASET_ROLE: @@ -263,6 +286,7 @@ def publish_release( payload=payload, create_tag=create_tag, update_latest=update_latest, + tag_only=tag_only, ) # The pointer is live: announce it. Best-effort and coupled to the promotion # so every publish path alerts; warn (don't fail) if the webhook is unset. @@ -348,6 +372,7 @@ def _publish_atomic( payload: dict, create_tag: bool, update_latest: bool = True, + tag_only: bool = False, ) -> None: staging_branch = f"release-staging/{release_id}" main_revision = _repo_revision(api, repo_id=repo_id) @@ -388,6 +413,12 @@ def _publish_atomic( branch=staging_branch, repo_type="dataset", ) + if tag_only: + # The release-id tag points directly at immutable_revision, so deleting + # the temporary branch does not make the candidate unreachable. Exact-k + # candidates deliberately stop here: neither canonical root artifacts + # nor release-directory copies are written to main. + return if update_latest: message = f"Update latest release to {release_id}" pointer = json.dumps(payload, indent=1).encode() diff --git a/packages/populace-data/tests/test_publish_guard.py b/packages/populace-data/tests/test_publish_guard.py index 14bcc083..8addab2d 100644 --- a/packages/populace-data/tests/test_publish_guard.py +++ b/packages/populace-data/tests/test_publish_guard.py @@ -1,6 +1,8 @@ import json from pathlib import Path +import pytest + from populace.data.publish_cli import _reform_validation_skipped, main @@ -43,7 +45,9 @@ def _stub_publish(monkeypatch): return cli -def test_publish_warns_when_build_manifest_has_no_staging(tmp_path, capsys, monkeypatch): +def test_publish_warns_when_build_manifest_has_no_staging( + tmp_path, capsys, monkeypatch +): (tmp_path / "build_manifest.json").write_text( json.dumps({"build_id": "x", "staging": None}) ) @@ -63,3 +67,53 @@ def test_publish_silent_when_staging_recorded(tmp_path, capsys, monkeypatch): rc = cli.main([str(tmp_path)]) assert rc == 0 assert "no staging telemetry" not in capsys.readouterr().err + + +def test_tag_only_cli_forwards_no_main_publication_mode(tmp_path, monkeypatch): + import populace.data.publish_cli as cli + + captured: dict = {} + + def fake_publish(*args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + return {"release_id": "r", "updated_at": None} + + monkeypatch.setattr(cli, "publish_release", fake_publish) + + rc = cli.main([str(tmp_path), "--no-latest", "--tag-only"]) + + assert rc == 0 + assert captured["kwargs"]["update_latest"] is False + assert captured["kwargs"]["tag_only"] is True + assert captured["kwargs"]["create_tag"] is True + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + (["--tag-only"], "--tag-only requires --no-latest"), + ( + ["--no-latest", "--tag-only", "--no-create-tag"], + "--tag-only requires tag creation", + ), + ], +) +def test_tag_only_cli_rejects_unsafe_flag_combinations_before_publish( + tmp_path, capsys, monkeypatch, arguments, message +): + import populace.data.publish_cli as cli + + called = False + + def unexpected_publish(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr(cli, "publish_release", unexpected_publish) + + with pytest.raises(SystemExit, match="2"): + cli.main([str(tmp_path), *arguments]) + + assert message in capsys.readouterr().err + assert called is False diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py index a31a47fa..2a9ad39f 100644 --- a/packages/populace-data/tests/test_release.py +++ b/packages/populace-data/tests/test_release.py @@ -748,6 +748,117 @@ def test_publish_no_latest_never_touches_pointer( assert final_commit["revision"] == "main" assert LATEST_POINTER_PATH not in final_commit["paths"] assert final_commit["message"] == f"Publish non-default release {RELEASE_ID}" + assert { + "populace_us_2024.h5", + "populace_us_2024_calibration.npz", + }.issubset(final_commit["paths"]) + + +def test_exact_k_tag_only_publish_never_mutates_main( + hub: FakeHub, release_dir: Path, artifact_root: Path +) -> None: + main_before = hub._refs["main"] + + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + updated_at="2026-06-11T13:53:15+00:00", + update_latest=False, + tag_only=True, + ) + + assert [event for event, _ in hub.events] == [ + "create_branch", + "create_commit", + "create_tag", + "delete_branch", + ] + assert hub._refs["main"] == main_before + + canonical_root_paths = { + "populace_us_2024.h5", + "populace_us_2024_calibration.npz", + } + main_commit_paths = { + path + for event, receipt in hub.events + if event == "create_commit" and receipt["revision"] == "main" + for path in receipt["paths"] + } + canonical_root_mutation_present = bool(canonical_root_paths & main_commit_paths) + assert canonical_root_mutation_present is False + assert canonical_root_paths.isdisjoint(main_commit_paths) + + immutable = hub.events[1][1] + assert canonical_root_paths.issubset(immutable["paths"]) + assert { + f"releases/{RELEASE_ID}/{filename}" + for filename in required_release_files(RELEASE_ID) + }.issubset(immutable["paths"]) + assert hub.tags == [{"tag": RELEASE_ID, "revision": immutable["commit"]}] + + tagged_manifest = Path( + hub.hf_hub_download( + repo_id="policyengine/populace-us", + filename=f"releases/{RELEASE_ID}/release_manifest.json", + repo_type="dataset", + revision=RELEASE_ID, + ) + ) + tagged_h5 = Path( + hub.hf_hub_download( + repo_id="policyengine/populace-us", + filename="populace_us_2024.h5", + repo_type="dataset", + revision=RELEASE_ID, + ) + ) + assert ( + tagged_manifest.read_bytes() + == (release_dir / "release_manifest.json").read_bytes() + ) + assert ( + tagged_h5.read_bytes() == (artifact_root / "populace_us_2024.h5").read_bytes() + ) + with pytest.raises(FileNotFoundError, match="populace_us_2024.h5@main"): + hub.hf_hub_download( + repo_id="policyengine/populace-us", + filename="populace_us_2024.h5", + repo_type="dataset", + ) + + +@pytest.mark.parametrize( + ("publish_options", "message"), + [ + ({"update_latest": True}, "requires update_latest=False"), + ( + {"update_latest": False, "create_tag": False}, + "requires create_tag=True", + ), + ], +) +def test_tag_only_rejects_unsafe_modes_before_remote_mutation( + hub: FakeHub, + release_dir: Path, + artifact_root: Path, + publish_options: dict, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + tag_only=True, + **publish_options, + ) + + assert hub.events == [] + assert hub.uploads == [] def test_publish_commits_immutable_release_before_root_and_pointer( diff --git a/tools/build_us_exact_k_ladder_release.py b/tools/build_us_exact_k_ladder_release.py new file mode 100644 index 00000000..9052edc7 --- /dev/null +++ b/tools/build_us_exact_k_ladder_release.py @@ -0,0 +1,626 @@ +"""Launch one immutable US exact-record-count ladder release. + +The launcher owns only strict configuration, artifact pins, ladder-point +resolution, and the non-publishing package receipt. Target materialization, +checkpointing, calibration gates, diagnostics, and release manifests remain in +``build_us_fiscal_refresh_release.py`` so the ladder cannot drift from the +incumbent's frozen-register release path. + +Example:: + + uv run python tools/build_us_exact_k_ladder_release.py \ + --pool-manifest /artifacts/pool.manifest.json \ + --config configs/us_exact_k_57240.json \ + --out build/us-exact-k + +Schema-v1 configuration (paths are resolved relative to the config file):: + + { + "schema_version": 1, + "pool": {"release_id": "...", "manifest_sha256": ""}, + "ladder": {"k": 57240, "seed": 17, "pi_hi": 0.95}, + "targets": { + "ledger_facts": "...", "ledger_facts_sha256": "", + "ledger_manifest_sha256": "", + "incumbent_diagnostics": "...", + "incumbent_diagnostics_sha256": "", + "target_surface_sha256": "" + }, + "calibration": { + "epochs": 256, "learning_rate": 0.02, "max_weight_ratio": 20, + "l0_refit_lambda_share": 0.8, "l2_lambda": 0, + "refit_l2_lambda": 0 + }, + "release": {"id": "populace-us-2024-k57240-...", + "repo_id": "policyengine/populace-us"} + } + +``ladder.k`` accepts exactly ``"N"``, ``57240``, or ``20000``. ``targets`` +may additionally carry the paired +``ssi_take_up_prior_weight_basis`` and +``ssi_take_up_prior_weight_basis_sha256`` fields for the house one-retry +delivery-gate protocol. +""" + +# The sibling house builder is importable only after its tools directory is +# installed below. +# ruff: noqa: E402,I001 + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shlex +import sys +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +_TOOLS_DIR = Path(__file__).resolve().parent +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +import build_us_fiscal_refresh_release as fiscal_release +from populace.build.us_runtime.h5_io import ( + load_simulation_ready_us_multispine_pool_manifest, +) + +CONFIG_SCHEMA_VERSION = 1 +RATIFIED_SPARSE_K = fiscal_release.RATIFIED_EXACT_K_COUNTS +US_RELEASE_REPO_ID = "policyengine/populace-us" +_LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") +_RELEASE_ID = re.compile(r"[A-Za-z0-9-]+") + + +@dataclass(frozen=True) +class LadderReleaseConfig: + """Validated, path-resolved launcher configuration.""" + + pool_release_id: str + pool_manifest_sha256: str + requested_k: str | int + seed: int + pi_hi: float + ledger_facts: Path + ledger_facts_sha256: str + ledger_manifest_sha256: str + incumbent_diagnostics: Path + incumbent_diagnostics_sha256: str + target_surface_sha256: str + ssi_take_up_prior_weight_basis: Path | None + ssi_take_up_prior_weight_basis_sha256: str | None + epochs: int + learning_rate: float + max_weight_ratio: float + l0_refit_lambda_share: float + l2_lambda: float + refit_l2_lambda: float + release_id: str + repo_id: str + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--pool-manifest", + type=Path, + required=True, + help="Simulation-ready manifest produced by build_us_multispine_pool.py.", + ) + parser.add_argument( + "--config", + type=Path, + required=True, + help="Strict schema-v1 JSON release configuration.", + ) + parser.add_argument("--out", type=Path, required=True) + return parser.parse_args(argv) + + +def _read_config(path: Path) -> LadderReleaseConfig: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Ladder config {path} is not valid JSON: {exc}.") from exc + root = _object(payload, label=f"ladder config {path}") + _keys( + root, + required={ + "schema_version", + "pool", + "ladder", + "targets", + "calibration", + "release", + }, + label=f"ladder config {path}", + ) + if ( + isinstance(root["schema_version"], bool) + or root["schema_version"] != CONFIG_SCHEMA_VERSION + ): + raise ValueError( + f"Ladder config {path} schema_version must be " + f"{CONFIG_SCHEMA_VERSION}, got {root['schema_version']!r}." + ) + + pool = _object(root["pool"], label="pool") + _keys(pool, required={"release_id", "manifest_sha256"}, label="pool") + ladder = _object(root["ladder"], label="ladder") + _keys(ladder, required={"k", "seed", "pi_hi"}, label="ladder") + targets = _object(root["targets"], label="targets") + _keys( + targets, + required={ + "ledger_facts", + "ledger_facts_sha256", + "ledger_manifest_sha256", + "incumbent_diagnostics", + "incumbent_diagnostics_sha256", + "target_surface_sha256", + }, + optional={ + "ssi_take_up_prior_weight_basis", + "ssi_take_up_prior_weight_basis_sha256", + }, + label="targets", + ) + calibration = _object(root["calibration"], label="calibration") + _keys( + calibration, + required={ + "epochs", + "learning_rate", + "max_weight_ratio", + "l0_refit_lambda_share", + "l2_lambda", + "refit_l2_lambda", + }, + label="calibration", + ) + release = _object(root["release"], label="release") + _keys(release, required={"id", "repo_id"}, label="release") + + requested_k = ladder["k"] + if requested_k != "N" and ( + isinstance(requested_k, bool) + or not isinstance(requested_k, int) + or requested_k not in RATIFIED_SPARSE_K + ): + raise ValueError( + f"ladder.k must be exactly 'N', 57240, or 20000; got {requested_k!r}." + ) + seed = _nonnegative_int(ladder["seed"], label="ladder.seed") + pi_hi = _finite_number(ladder["pi_hi"], label="ladder.pi_hi") + if not 0.0 <= pi_hi <= 1.0: + raise ValueError(f"ladder.pi_hi must be in [0, 1], got {pi_hi!r}.") + + config_dir = path.resolve().parent + ledger_facts = _resolve_path(targets["ledger_facts"], config_dir, "ledger_facts") + incumbent = _resolve_path( + targets["incumbent_diagnostics"], + config_dir, + "incumbent_diagnostics", + ) + prior_basis_value = targets.get("ssi_take_up_prior_weight_basis") + prior_basis_sha_value = targets.get("ssi_take_up_prior_weight_basis_sha256") + if (prior_basis_value is None) != (prior_basis_sha_value is None): + raise ValueError( + "targets.ssi_take_up_prior_weight_basis and its SHA-256 pin must " + "be provided together." + ) + prior_basis = ( + None + if prior_basis_value is None + else _resolve_path( + prior_basis_value, + config_dir, + "ssi_take_up_prior_weight_basis", + ) + ) + prior_basis_sha = ( + None + if prior_basis_sha_value is None + else _sha256_value( + prior_basis_sha_value, + label="targets.ssi_take_up_prior_weight_basis_sha256", + ) + ) + epochs = _positive_int(calibration["epochs"], label="calibration.epochs") + learning_rate = _positive_number( + calibration["learning_rate"], label="calibration.learning_rate" + ) + max_weight_ratio = _finite_number( + calibration["max_weight_ratio"], label="calibration.max_weight_ratio" + ) + if max_weight_ratio < 1.0: + raise ValueError("calibration.max_weight_ratio must be at least 1.") + l0_share = _positive_number( + calibration["l0_refit_lambda_share"], + label="calibration.l0_refit_lambda_share", + ) + l2_lambda = _nonnegative_number( + calibration["l2_lambda"], label="calibration.l2_lambda" + ) + refit_l2_lambda = _nonnegative_number( + calibration["refit_l2_lambda"], + label="calibration.refit_l2_lambda", + ) + + pool_release_id = _nonempty_string(pool["release_id"], label="pool.release_id") + release_id = _nonempty_string(release["id"], label="release.id") + if _RELEASE_ID.fullmatch(release_id) is None: + raise ValueError( + "release.id may contain only ASCII letters, digits, and hyphens." + ) + repo_id = _nonempty_string(release["repo_id"], label="release.repo_id") + if repo_id != US_RELEASE_REPO_ID: + raise ValueError( + f"release.repo_id must be {US_RELEASE_REPO_ID!r}, got {repo_id!r}." + ) + + return LadderReleaseConfig( + pool_release_id=pool_release_id, + pool_manifest_sha256=_sha256_value( + pool["manifest_sha256"], label="pool.manifest_sha256" + ), + requested_k=requested_k, + seed=seed, + pi_hi=pi_hi, + ledger_facts=ledger_facts, + ledger_facts_sha256=_sha256_value( + targets["ledger_facts_sha256"], label="targets.ledger_facts_sha256" + ), + ledger_manifest_sha256=_sha256_value( + targets["ledger_manifest_sha256"], + label="targets.ledger_manifest_sha256", + ), + incumbent_diagnostics=incumbent, + incumbent_diagnostics_sha256=_sha256_value( + targets["incumbent_diagnostics_sha256"], + label="targets.incumbent_diagnostics_sha256", + ), + target_surface_sha256=_sha256_value( + targets["target_surface_sha256"], + label="targets.target_surface_sha256", + ), + ssi_take_up_prior_weight_basis=prior_basis, + ssi_take_up_prior_weight_basis_sha256=prior_basis_sha, + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + l0_refit_lambda_share=l0_share, + l2_lambda=l2_lambda, + refit_l2_lambda=refit_l2_lambda, + release_id=release_id, + repo_id=repo_id, + ) + + +def _validate_pins_and_resolve_k( + *, + config: LadderReleaseConfig, + pool_manifest_path: Path, +) -> tuple[int, Mapping[str, object]]: + pool_manifest = load_simulation_ready_us_multispine_pool_manifest( + pool_manifest_path, + expected_manifest_sha256=config.pool_manifest_sha256, + ) + fiscal_release._assert_pool_release_identity( + config.pool_release_id, + pool_manifest, + ) + agreement_gate = _object( + pool_manifest.get("agreement_gate"), label="pool manifest agreement_gate" + ) + if agreement_gate.get("passed") is not True: + raise ValueError("Pool manifest has no passing agreement-gate verdict.") + counts = _object( + pool_manifest.get("provenance_counts"), + label="pool manifest provenance_counts", + ) + household_counts = _object( + counts.get("household"), + label="pool manifest provenance_counts.household", + ) + pool_size = _positive_int( + household_counts.get("rows"), + label="pool manifest provenance_counts.household.rows", + ) + k = pool_size if config.requested_k == "N" else int(config.requested_k) + if k > pool_size: + raise ValueError( + f"k={k} exceeds the pool size {pool_size}; ladder selection never " + "clamps the requested cardinality." + ) + expected_prefix = f"populace-us-2024-k{k}-" + if not config.release_id.startswith(expected_prefix): + raise ValueError( + f"release.id must start with {expected_prefix!r} for resolved k={k}; " + f"got {config.release_id!r}." + ) + + if not config.ledger_facts.exists(): + raise FileNotFoundError( + f"targets.ledger_facts does not exist: {config.ledger_facts}" + ) + if not config.incumbent_diagnostics.is_file(): + raise FileNotFoundError( + "targets.incumbent_diagnostics is not a file: " + f"{config.incumbent_diagnostics}" + ) + incumbent_bytes = config.incumbent_diagnostics.read_bytes() + observed_incumbent_sha256 = hashlib.sha256(incumbent_bytes).hexdigest() + if observed_incumbent_sha256 != config.incumbent_diagnostics_sha256: + raise ValueError( + "Incumbent diagnostics SHA-256 mismatch: got " + f"{observed_incumbent_sha256}, expected " + f"{config.incumbent_diagnostics_sha256}." + ) + incumbent = _object( + json.loads(incumbent_bytes), + label="incumbent diagnostics", + ) + target_surface = _object( + incumbent.get("target_surface"), + label="incumbent diagnostics target_surface", + ) + if target_surface.get("sha256") != config.target_surface_sha256: + raise ValueError( + "Incumbent diagnostics target-surface SHA-256 mismatch: got " + f"{target_surface.get('sha256')!r}, expected " + f"{config.target_surface_sha256!r}." + ) + if config.ssi_take_up_prior_weight_basis is not None: + if not config.ssi_take_up_prior_weight_basis.is_file(): + raise FileNotFoundError( + "targets.ssi_take_up_prior_weight_basis is not a file: " + f"{config.ssi_take_up_prior_weight_basis}" + ) + observed_prior_basis_sha256 = _sha256(config.ssi_take_up_prior_weight_basis) + if observed_prior_basis_sha256 != ( + config.ssi_take_up_prior_weight_basis_sha256 + ): + raise ValueError( + "SSI take-up prior-weight basis SHA-256 mismatch: got " + f"{observed_prior_basis_sha256}, expected " + f"{config.ssi_take_up_prior_weight_basis_sha256}." + ) + return k, pool_manifest + + +def _builder_argv( + *, + config: LadderReleaseConfig, + pool_manifest: Path, + out: Path, + k: str | int, +) -> list[str]: + argv = [ + "--pool-manifest", + str(pool_manifest), + "--pool-manifest-sha256", + config.pool_manifest_sha256, + "--pool-release-id", + config.pool_release_id, + "--exact-k", + str(k), + "--exact-k-pi-hi", + str(config.pi_hi), + "--ledger-facts", + str(config.ledger_facts), + "--ledger-facts-sha256", + config.ledger_facts_sha256, + "--ledger-manifest-sha256", + config.ledger_manifest_sha256, + "--incumbent-diagnostics", + str(config.incumbent_diagnostics), + "--incumbent-diagnostics-sha256", + config.incumbent_diagnostics_sha256, + "--frozen-target-surface-sha256", + config.target_surface_sha256, + "--out", + str(out), + "--release-id", + config.release_id, + "--seed", + str(config.seed), + "--epochs", + str(config.epochs), + "--learning-rate", + str(config.learning_rate), + "--max-weight-ratio", + str(config.max_weight_ratio), + "--l0-refit-lambda-share", + str(config.l0_refit_lambda_share), + "--l2-lambda", + str(config.l2_lambda), + "--refit-l2-lambda", + str(config.refit_l2_lambda), + "--no-staging", + ] + if config.ssi_take_up_prior_weight_basis is not None: + argv.extend( + [ + "--ssi-take-up-prior-weight-basis", + str(config.ssi_take_up_prior_weight_basis), + "--ssi-take-up-prior-weight-basis-sha256", + str(config.ssi_take_up_prior_weight_basis_sha256), + ] + ) + return argv + + +def launch( + *, + pool_manifest: Path, + config_path: Path, + out: Path, + release_builder: Callable[[Sequence[str] | None], object] = fiscal_release.main, +) -> dict[str, object]: + """Validate pins, run the house release path, and write a publish receipt.""" + + config = _read_config(config_path) + resolved_pool_manifest = pool_manifest.resolve() + resolved_out = out.resolve() + k, _ = _validate_pins_and_resolve_k( + config=config, + pool_manifest_path=resolved_pool_manifest, + ) + release_builder( + _builder_argv( + config=config, + pool_manifest=resolved_pool_manifest, + out=resolved_out, + k=config.requested_k, + ) + ) + build = { + "release_id": config.release_id, + "release_dir": str(resolved_out / "releases" / config.release_id), + "artifact_root": str(resolved_out / "artifacts"), + } + + publish_argv = [ + "tools/publish_release.sh", + build["release_dir"], + "--repo-id", + config.repo_id, + "--artifact-root", + build["artifact_root"], + "--create-tag", + "--no-latest", + "--tag-only", + ] + result: dict[str, object] = { + **build, + "k": k, + "seed": config.seed, + "automatic_publish": False, + "pointer_update": False, + "pointer_updates": { + "production": { + "repo_id": config.repo_id, + "pointer_update": False, + }, + "staging": { + "repo_id": os.environ.get( + "POPULACE_STAGING_REPO_ID", + "policyengine/populace-us-staging", + ), + "pointer_update": False, + }, + }, + "publish_argv": publish_argv, + "publish_command": shlex.join(publish_argv), + } + resolved_out.mkdir(parents=True, exist_ok=True) + package_result = resolved_out / "package_result.json" + package_result.write_text( + json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return result + + +def _object(value: object, *, label: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be a JSON object.") + return value + + +def _keys( + value: Mapping[str, object], + *, + required: set[str], + optional: set[str] | None = None, + label: str, +) -> None: + optional = optional or set() + missing = sorted(required - set(value)) + unknown = sorted(set(value) - required - optional) + if missing or unknown: + raise ValueError( + f"{label} keys do not match schema; missing={missing}, unknown={unknown}." + ) + + +def _nonempty_string(value: object, *, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label} must be a non-empty string.") + return value + + +def _sha256_value(value: object, *, label: str) -> str: + parsed = _nonempty_string(value, label=label) + if _LOWERCASE_SHA256.fullmatch(parsed) is None: + raise ValueError(f"{label} must be a 64-character lowercase SHA-256.") + return parsed + + +def _nonnegative_int(value: object, *, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{label} must be a non-negative integer, got {value!r}.") + return value + + +def _positive_int(value: object, *, label: str) -> int: + parsed = _nonnegative_int(value, label=label) + if parsed == 0: + raise ValueError(f"{label} must be positive.") + return parsed + + +def _finite_number(value: object, *, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"{label} must be a finite number, got {value!r}.") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"{label} must be a finite number, got {value!r}.") + return parsed + + +def _positive_number(value: object, *, label: str) -> float: + parsed = _finite_number(value, label=label) + if parsed <= 0.0: + raise ValueError(f"{label} must be positive, got {parsed!r}.") + return parsed + + +def _nonnegative_number(value: object, *, label: str) -> float: + parsed = _finite_number(value, label=label) + if parsed < 0.0: + raise ValueError(f"{label} must be non-negative, got {parsed!r}.") + return parsed + + +def _resolve_path(value: object, base: Path, label: str) -> Path: + raw = _nonempty_string(value, label=f"targets.{label}") + path = Path(raw) + return path.resolve() if path.is_absolute() else (base / path).resolve() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(argv: Sequence[str] | None = None) -> dict[str, object]: + args = _parse_args(argv) + return launch( + pool_manifest=args.pool_manifest, + config_path=args.config, + out=args.out, + ) + + +if __name__ == "__main__": + main() diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 33f0ee57..127eb217 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -215,9 +215,19 @@ write_demographics, ) from populace.build.us_runtime.engine_lifecycle import release_engine_simulation +from populace.build.us_runtime.exact_k_ladder import ( + ExactKLadderCalibration, + assert_exact_k_realized_count, + calibrate_exact_k_ladder, + exact_k_ladder_manifest_payload, +) from populace.build.us_runtime.fiscal_targets import ( SSA_SSI_AGE_BAND_RECIPIENTS_TARGET_ROLE, ) +from populace.build.us_runtime.h5_io import ( + AuthenticatedPoolH5, + load_simulation_ready_us_multispine_pool, +) from populace.build.us_runtime.input_mass import us_input_mass_totals from populace.build.us_runtime.l0_refit_export import ( attach_l0_refit_entity_weights, @@ -258,6 +268,7 @@ TargetSpec, calibrate, calibrate_l0_refit, + relative_error_loss, ) from populace.calibrate.diagnostics import ( diagnostics_payload, @@ -334,6 +345,17 @@ ) US_FISCAL_TARGET_VALUE_WEIGHT_POWER = 0.5 US_FISCAL_TARGET_LOSS_CAP = 1.0 +RATIFIED_EXACT_K_COUNTS = frozenset({57_240, 20_000}) + + +class IncumbentLossBasisMismatchError(RuntimeError): + """The pinned incumbent was scored on a different fiscal-loss basis.""" + + +class PoolReleaseIdentityMismatchError(ValueError): + """The configured pool identity differs from its authenticated manifest.""" + + # Bumped 1 -> 2 for #217: the per-reform income-tax cache key now depends only on # the inputs that actually determine per-household reform estimates and no longer # includes build_commit / seed / target_registry_version. Old (v1) coarse-key @@ -749,13 +771,85 @@ def _automatic_gc_suspended(): ) -def _parse_args() -> argparse.Namespace: +def _parse_ratified_exact_k(value: str) -> str | int: + if value == "N": + return value + if value in {str(k) for k in RATIFIED_EXACT_K_COUNTS}: + return int(value) + raise argparse.ArgumentTypeError( + "ExactKCharterError: --exact-k must be exactly N, 57240, or 20000; " + f"got {value!r}." + ) + + +def _assert_pool_release_identity( + configured_release_id: str, + pool_manifest: Mapping[str, object], +) -> str: + return _assert_pool_release_id_value( + configured_release_id, + pool_manifest.get("publication_run_id"), + ) + + +def _assert_pool_release_id_value( + configured_release_id: str, + publication_run_id: object, +) -> str: + if ( + not isinstance(publication_run_id, str) + or not publication_run_id + or configured_release_id != publication_run_id + ): + raise PoolReleaseIdentityMismatchError( + "PoolReleaseIdentityMismatchError: configured pool release id " + f"{configured_release_id!r} does not match authenticated manifest " + f"publication_run_id {publication_run_id!r}." + ) + return publication_run_id + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( "--base-h5", type=Path, help="Existing Populace US H5 to recalibrate. Defaults to HF latest.", ) + parser.add_argument( + "--pool-manifest", + type=Path, + help=( + "Simulation-ready build_us_multispine_pool.py manifest. The " + "manifest, rather than a bare H5, is the readiness authority. " + "Mutually exclusive with --base-h5." + ), + ) + parser.add_argument( + "--pool-manifest-sha256", + help=( + "Expected SHA-256 of --pool-manifest. Required for an exact-k " + "ladder release so the artifact-store envelope is pinned." + ), + ) + parser.add_argument( + "--pool-release-id", + help=("Authenticated publication_run_id of the pool artifact envelope."), + ) + parser.add_argument( + "--exact-k", + type=_parse_ratified_exact_k, + help=( + "Ratified ladder point: N, 57240, or 20000 households. N resolves " + "to the authenticated pool size and uses identity support with an " + "ordinary full-pool refit." + ), + ) + parser.add_argument( + "--exact-k-pi-hi", + type=float, + help="Certainty-unit threshold for --exact-k selection.", + ) parser.add_argument( "--ledger-facts", type=Path, @@ -845,6 +939,20 @@ def _parse_args() -> argparse.Namespace: "still pass if they improve on this incumbent row by row." ), ) + parser.add_argument( + "--incumbent-diagnostics-sha256", + help=( + "Expected SHA-256 of --incumbent-diagnostics. Required for an " + "exact-k ladder release." + ), + ) + parser.add_argument( + "--frozen-target-surface-sha256", + help=( + "Expected target-surface SHA-256 embedded in the pinned " + "incumbent diagnostics. Required for an exact-k ladder release." + ), + ) parser.add_argument( "--input-mass-reference-h5", type=Path, @@ -1112,7 +1220,7 @@ def _parse_args() -> argparse.Namespace: "drifting rebuild and is not yet wired into calibration." ), ) - parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--seed", type=int) parser.add_argument( "--asec-2023-weeks-unemployed-source", type=Path, @@ -1376,7 +1484,7 @@ def _parse_args() -> argparse.Namespace: default=30.0, help="Minimum seconds between progress uploads to the staging repo.", ) - args = parser.parse_args() + args = parser.parse_args(argv) if ( args.include_congressional_district_targets and args.congressional_district_vintage_crosswalk is None @@ -1398,6 +1506,90 @@ def _parse_args() -> argparse.Namespace: "--refit-l2-lambda requires the sparse L0+refit default dataset; " "--dense-default-dataset has no refit stage (use --l2-lambda)." ) + ladder_values = ( + args.exact_k, + args.exact_k_pi_hi, + args.pool_manifest, + args.pool_manifest_sha256, + args.pool_release_id, + ) + if any(value is not None for value in ladder_values): + if any(value is None for value in ladder_values): + parser.error( + "--exact-k, --exact-k-pi-hi, --pool-manifest, " + "--pool-manifest-sha256, and --pool-release-id must be " + "provided together." + ) + if args.base_h5 is not None: + parser.error("--pool-manifest is mutually exclusive with --base-h5.") + if args.seed is None or args.seed < 0: + parser.error( + "ExactKExplicitSeedError: --exact-k requires an explicit " + "non-negative --seed." + ) + if not args.no_staging: + parser.error( + "ExactKPointerSuppressionError: --exact-k requires --no-staging." + ) + if not math.isfinite(args.exact_k_pi_hi) or not ( + 0.0 <= args.exact_k_pi_hi <= 1.0 + ): + parser.error("--exact-k-pi-hi must be finite and in [0, 1].") + if len(args.pool_manifest_sha256) != 64 or any( + character not in "0123456789abcdef" + for character in args.pool_manifest_sha256 + ): + parser.error( + "--pool-manifest-sha256 must be exactly 64 lowercase " + "hexadecimal characters." + ) + if not args.pool_release_id.strip(): + parser.error("--pool-release-id must be non-empty.") + if args.incumbent_diagnostics is None: + parser.error( + "--exact-k requires --incumbent-diagnostics so every ladder " + "point is judged against the incumbent on the frozen target " + "register." + ) + for flag, value in ( + ( + "--incumbent-diagnostics-sha256", + args.incumbent_diagnostics_sha256, + ), + ( + "--frozen-target-surface-sha256", + args.frozen_target_surface_sha256, + ), + ): + if ( + value is None + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + parser.error( + f"{flag} must be exactly 64 lowercase hexadecimal characters." + ) + if args.ledger_facts_sha256 is None or args.ledger_manifest_sha256 is None: + parser.error( + "--exact-k requires both --ledger-facts-sha256 and " + "--ledger-manifest-sha256 to pin the frozen target register." + ) + if args.dense_default_dataset: + parser.error( + "--exact-k owns the full-pool identity arm; do not combine it " + "with --dense-default-dataset." + ) + if ( + args.selection_source_h5 is not None + or args.selection_source_manifest is not None + ): + parser.error( + "--exact-k operates on the complete multispine pool and cannot " + "be combined with a frozen selection source." + ) + elif args.seed is None: + # Preserve the pre-exact-k default for every legacy lane. + args.seed = 0 multipliers: dict[str, float] = {} for entry in args.target_family_loss_multiplier: family, separator, raw_value = entry.partition("=") @@ -1440,6 +1632,27 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _legacy_base_h5_sha256(path: Path) -> str: + """Hash only the non-pool base path selected by main's legacy branch.""" + + return _sha256(path) + + +def _copy_base_h5_for_local_audit( + source: Path, + destination: Path, + *, + authenticated_pool_h5: AuthenticatedPoolH5 | None, +) -> Path: + if authenticated_pool_h5 is not None: + return authenticated_pool_h5.copy_verified_to( + destination, + consumer="builder final local-audit copy", + ) + shutil.copy2(source, destination) + return destination + + def _runtime_versions() -> dict[str, str]: packages = ( "populace-build", @@ -2247,11 +2460,28 @@ def _resolve_selection_source(args): def _assert_cd_vintage_support_matches( h5_path: Path, crosswalk_metadata: Mapping[str, object] | None, + *, + authenticated_pool_h5: AuthenticatedPoolH5 | None = None, ) -> None: if crosswalk_metadata is None: return expected_sha256 = str(crosswalk_metadata.get("sha256") or "") - support_provenance = _read_cd_vintage_support_provenance(h5_path) + if authenticated_pool_h5 is not None: + authenticated_pool_h5.verified_digest( + consumer="congressional-district support preflight before H5 read" + ) + try: + support_provenance = _read_cd_vintage_support_provenance(h5_path) + except Exception: + if authenticated_pool_h5 is not None: + authenticated_pool_h5.verified_digest( + consumer="congressional-district support preflight failed H5 read" + ) + raise + if authenticated_pool_h5 is not None: + authenticated_pool_h5.verified_digest( + consumer="congressional-district support preflight after H5 read" + ) actual_sha256 = support_provenance.get( CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR ) @@ -2361,6 +2591,29 @@ def _load_incumbent_diagnostics_payload(path: Path | None) -> dict[str, object]: return payload +def _load_verified_incumbent_diagnostics_payload( + path: Path, + *, + expected_sha256: str, +) -> tuple[dict[str, object], str]: + """Load one incumbent from the exact bytes authenticated for scoring.""" + + raw = path.read_bytes() + observed_sha256 = hashlib.sha256(raw).hexdigest() + if observed_sha256 != expected_sha256: + raise ValueError( + "Incumbent diagnostics SHA-256 mismatch for " + f"{path}: got {observed_sha256}, expected {expected_sha256}." + ) + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError( + f"{path} is not a Populace calibration_diagnostics.json file: " + "expected a JSON object." + ) + return payload, observed_sha256 + + def _diagnostics_by_target_name( payload: Mapping[str, object], *, @@ -5502,6 +5755,77 @@ def _fiscal_target_loss_weights( return weights / weights.mean() +def _fiscal_target_loss_basis( + registry: TargetRegistry, + target_loss_weights: np.ndarray, + family_multipliers: Mapping[str, float] | None = None, +) -> dict[str, object]: + """Content-address the complete loss basis without changing target surface.""" + + weights = np.asarray(target_loss_weights, dtype=np.float64) + if weights.shape != (len(registry.specs),): + raise ValueError( + "Fiscal target loss basis vector shape does not match the compiled " + f"registry: got {weights.shape}, expected {(len(registry.specs),)}." + ) + if not np.isfinite(weights).all() or (weights <= 0.0).any(): + raise ValueError( + "Fiscal target loss basis weights must be finite and positive." + ) + loss_vector = [ + { + "row_name": _target_row_name(spec), + "weight_hex": float(weight).hex(), + } + for spec, weight in zip(registry.specs, weights, strict=True) + ] + loss_vector_sha256 = hashlib.sha256( + json.dumps( + loss_vector, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + return { + "schema_version": 1, + "target_loss_weighting": US_FISCAL_TARGET_LOSS_WEIGHTING, + "target_loss_family_multipliers": { + family: float(multiplier) + for family, multiplier in sorted((family_multipliers or {}).items()) + }, + "target_loss_cap": US_FISCAL_TARGET_LOSS_CAP, + "n_targets": len(loss_vector), + "loss_vector_sha256": loss_vector_sha256, + } + + +def _incumbent_target_loss_basis( + payload: Mapping[str, object], +) -> Mapping[str, object] | None: + build = payload.get("build") + if not isinstance(build, Mapping): + return None + basis = build.get("target_loss_basis") + return basis if isinstance(basis, Mapping) else None + + +def _assert_incumbent_loss_basis_matches( + configured: Mapping[str, object], + incumbent: Mapping[str, object] | None, +) -> None: + if incumbent is None: + raise IncumbentLossBasisMismatchError( + "pinned incumbent diagnostics have no build.target_loss_basis; " + "rescore the incumbent on the frozen register before release." + ) + if dict(incumbent) != dict(configured): + raise IncumbentLossBasisMismatchError( + "pinned incumbent target-loss basis differs from the configured " + f"basis: configured={dict(configured)!r}, incumbent={dict(incumbent)!r}." + ) + + def _ssi_take_up_band_targets_from_registry(target_specs: tuple) -> dict[str, float]: """SSA age-band recipient counts as compiled into the calibration registry. @@ -6280,6 +6604,169 @@ def _incumbent_relative_error( return (final_estimate - target_value) / target_value +def _exact_k_frozen_register_fit_gate( + result, + incumbent_diagnostics: Mapping[str, Mapping[str, object]], + *, + target_registry: TargetRegistry, + target_loss_weights: np.ndarray, + configured_loss_basis: Mapping[str, object], + incumbent_loss_basis: Mapping[str, object] | None, +) -> GateResult: + """Require an exact-k candidate to beat the incumbent on one register. + + The comparison re-scores both artifacts from their per-target rows with + the same capped, concept-budget-weighted loss used by the release solve. + Target-surface fingerprint equality is checked before this gate is called; + exact row-set, value, and loss-vector checks here make that binding + executable rather than trusting summary scalars from the incumbent file. + """ + + diagnostics = tuple(getattr(result, "diagnostics", ()) or ()) + names = [str(getattr(row, "name", "")) for row in diagnostics] + failures: list[str] = [] + try: + _assert_incumbent_loss_basis_matches( + configured_loss_basis, + incumbent_loss_basis, + ) + except IncumbentLossBasisMismatchError as error: + failures.append(f"{type(error).__name__}: {error}") + if not names or any(not name for name in names): + failures.append( + "Exact-k frozen-register comparison has no complete candidate target rows." + ) + if len(names) != len(set(names)): + failures.append( + "Exact-k frozen-register comparison found duplicate candidate target names." + ) + + incumbent_names = set(incumbent_diagnostics) + candidate_names = set(names) + missing = sorted(candidate_names - incumbent_names) + extra = sorted(incumbent_names - candidate_names) + if missing or extra: + failures.append( + "Exact-k incumbent target rows do not equal the frozen candidate " + f"register (missing={missing[:5]}, extra={extra[:5]})." + ) + + weights_by_name = { + _target_row_name(spec): float(weight) + for spec, weight in zip( + target_registry.specs, + np.asarray(target_loss_weights, dtype=np.float64), + strict=True, + ) + } + missing_weights = sorted(candidate_names - set(weights_by_name)) + if missing_weights: + failures.append( + "Exact-k frozen-register comparison has no loss weight for target " + f"row(s) {missing_weights[:5]}." + ) + + candidate_targets: list[float] = [] + candidate_estimates: list[float] = [] + incumbent_estimates: list[float] = [] + aligned_weights: list[float] = [] + for diagnostic in diagnostics: + name = str(getattr(diagnostic, "name", "")) + incumbent = incumbent_diagnostics.get(name) + target = getattr(diagnostic, "target", None) + estimate = getattr(diagnostic, "final_estimate", None) + incumbent_target = None if incumbent is None else incumbent.get("target") + incumbent_estimate = ( + None if incumbent is None else incumbent.get("final_estimate") + ) + values = (target, estimate, incumbent_target, incumbent_estimate) + if not all( + isinstance(value, int | float) and math.isfinite(float(value)) + for value in values + ): + failures.append( + "Exact-k frozen-register comparison has non-finite target or " + f"estimate data for {name!r}." + ) + continue + if not math.isclose( + float(target), + float(incumbent_target), + rel_tol=1e-9, + abs_tol=1e-6, + ): + failures.append( + "Exact-k incumbent target value changed for " + f"{name!r}: candidate={target!r}, incumbent={incumbent_target!r}." + ) + continue + weight = weights_by_name.get(name) + if weight is None: + continue + candidate_targets.append(float(target)) + candidate_estimates.append(float(estimate)) + incumbent_estimates.append(float(incumbent_estimate)) + aligned_weights.append(weight) + + candidate_loss: float | None = None + incumbent_loss: float | None = None + if not failures: + target_vector = np.asarray(candidate_targets, dtype=np.float64) + weights = np.asarray(aligned_weights, dtype=np.float64) + candidate_loss = relative_error_loss( + np.asarray(candidate_estimates, dtype=np.float64), + target_vector, + target_loss_weights=weights, + target_loss_cap=US_FISCAL_TARGET_LOSS_CAP, + ) + incumbent_loss = relative_error_loss( + np.asarray(incumbent_estimates, dtype=np.float64), + target_vector, + target_loss_weights=weights, + target_loss_cap=US_FISCAL_TARGET_LOSS_CAP, + ) + reported_loss = float(getattr(result, "final_loss", math.nan)) + if not math.isclose( + candidate_loss, + reported_loss, + rel_tol=1e-6, + abs_tol=1e-12, + ): + failures.append( + "Exact-k frozen-register candidate re-score does not match the " + f"solver loss: rescored={candidate_loss}, reported={reported_loss}." + ) + elif not candidate_loss < incumbent_loss: + failures.append( + "Exact-k candidate did not beat the incumbent on the frozen " + f"target register: candidate_loss={candidate_loss}, " + f"incumbent_loss={incumbent_loss}." + ) + + return GateResult( + name="exact_k_frozen_register_fit", + passed=not failures, + failures=tuple(failures), + details={ + "metric": "capped_concept_budget_weighted_mean_absolute_relative_error", + "target_loss_cap": US_FISCAL_TARGET_LOSS_CAP, + "configured_loss_basis": dict(configured_loss_basis), + "incumbent_loss_basis": ( + dict(incumbent_loss_basis) if incumbent_loss_basis is not None else None + ), + "n_targets": len(names), + "candidate_loss": candidate_loss, + "incumbent_loss": incumbent_loss, + "strict_improvement_required": True, + "improvement": ( + None + if candidate_loss is None or incumbent_loss is None + else incumbent_loss - candidate_loss + ), + }, + ) + + def _incumbent_critical_target_payload( incumbent_diagnostics: Mapping[str, Mapping[str, object]], ) -> dict[str, dict[str, float]]: @@ -6371,7 +6858,7 @@ def _write_release_calibration_diagnostics( result, release_dir: Path, registry: TargetRegistry, - base_h5: Path, + base_dataset_sha256: str, compilation: Mapping[str, object], target_profile_gate: GateResult, health_input_gate: GateResult | None, @@ -6391,11 +6878,14 @@ def _write_release_calibration_diagnostics( selection_source: Mapping[str, object] | None = None, default_dataset: Mapping[str, object] | None = None, incumbent_diagnostics_path: Path | None = None, + incumbent_diagnostics_sha256: str | None = None, incumbent_diagnostics: Mapping[str, Mapping[str, object]] | None = None, degenerate_input_gate: GateResult | None = None, ecps_parity_gate: GateResult | None = None, validation_input_coverage_gate: GateResult | None = None, target_loss_family_multipliers: Mapping[str, float] | None = None, + target_loss_basis: Mapping[str, object] | None = None, + exact_k_ladder: Mapping[str, object] | None = None, ) -> None: """Write calibration diagnostics even when hard release gates fail.""" failures = list(gate_failures) @@ -6403,7 +6893,11 @@ def _write_release_calibration_diagnostics( incumbent_payload = ( { "path": str(incumbent_diagnostics_path), - "sha256": _sha256(incumbent_diagnostics_path), + "sha256": ( + incumbent_diagnostics_sha256 + if incumbent_diagnostics_sha256 is not None + else _sha256(incumbent_diagnostics_path) + ), "critical_targets": _incumbent_critical_target_payload(incumbent_rows), } if incumbent_diagnostics_path is not None @@ -6414,7 +6908,7 @@ def _write_release_calibration_diagnostics( release_dir / "calibration_diagnostics.json", target_registry=registry, build={ - "base_dataset_sha256": _sha256(base_h5), + "base_dataset_sha256": base_dataset_sha256, "target_compilation": compilation, "target_loss_weighting": US_FISCAL_TARGET_LOSS_WEIGHTING, "target_loss_family_multipliers": ( @@ -6423,6 +6917,11 @@ def _write_release_calibration_diagnostics( else None ), "target_loss_cap": US_FISCAL_TARGET_LOSS_CAP, + **( + {"target_loss_basis": dict(target_loss_basis)} + if target_loss_basis is not None + else {} + ), "target_profile_coverage": { "passed": target_profile_gate.passed, "failures": list(target_profile_gate.failures), @@ -6551,6 +7050,11 @@ def _write_release_calibration_diagnostics( "default_dataset": ( dict(default_dataset) if default_dataset is not None else None ), + **( + {"exact_k_ladder": dict(exact_k_ladder)} + if exact_k_ladder is not None + else {} + ), "timing": dict(timing or {}), "release_gates": { "passed": not failures, @@ -6769,9 +7273,14 @@ def _build_manifests( medicaid_enrollment_substitutions: Sequence[Mapping[str, object]] = (), staging: Mapping[str, object] | None = None, ledger_artifact: Mapping[str, object] | None = None, + dataset_key: str = "populace_us_2024", + dataset_filename: str = DATASET_FILENAME, + calibration_key: str = "populace_us_2024_calibration", + calibration_filename: str = CALIBRATION_FILENAME, + exact_k_ladder: Mapping[str, object] | None = None, ) -> None: - dataset_path = artifact_root / DATASET_FILENAME - calibration_path = artifact_root / CALIBRATION_FILENAME + dataset_path = artifact_root / dataset_filename + calibration_path = artifact_root / calibration_filename diagnostics_path = release_dir / "calibration_diagnostics.json" coverage_path = release_dir / "us_source_coverage.json" dataset_sha = _sha256(dataset_path) @@ -6828,13 +7337,18 @@ def _build_manifests( "runtime": runtime, "timing": timing_payload, "ledger_artifact": dict(ledger_artifact) if ledger_artifact else None, + **( + {"exact_k_ladder": dict(exact_k_ladder)} + if exact_k_ladder is not None + else {} + ), "dataset": { - "filename": DATASET_FILENAME, + "filename": dataset_filename, "sha256": dataset_sha, "default": default_dataset_payload, }, "calibration": { - "filename": CALIBRATION_FILENAME, + "filename": calibration_filename, "sha256": calibration_sha, "warm_start": warm_start_payload, "selection_source": selection_source_payload, @@ -6863,6 +7377,18 @@ def _build_manifests( "final_loss": diag["final_loss"], "fraction_within_10pct": diag["fraction_within_10pct"], }, + **( + { + "exact_k_frozen_register_fit": dict( + exact_k_ladder["frozen_target_register"]["incumbent_fit"] + ), + "exact_k_puf_capital_gains_tail": dict( + exact_k_ladder["invariant_battery"]["puf_capital_gains_tail"] + ), + } + if exact_k_ladder is not None + else {} + ), "target_compilation": dropped, "target_profile_coverage": { "passed": target_profile_gate.passed, @@ -7009,7 +7535,7 @@ def _build_manifests( "name": "populace-data", "version": runtime["populace-data"], }, - "default_datasets": {"national": "populace_us_2024"}, + "default_datasets": {"national": dataset_key}, "build": { "build_id": release_id, "built_at": built_at, @@ -7023,6 +7549,11 @@ def _build_manifests( }, "timing": timing_payload, "ledger_artifact": dict(ledger_artifact) if ledger_artifact else None, + **( + {"exact_k_ladder": dict(exact_k_ladder)} + if exact_k_ladder is not None + else {} + ), "warm_start_calibration": warm_start_payload, "selection_source": selection_source_payload, "default_dataset": default_dataset_payload, @@ -7136,14 +7667,14 @@ def _build_manifests( } ], "artifacts": { - "populace_us_2024": _artifact_entry( - DATASET_FILENAME, + dataset_key: _artifact_entry( + dataset_filename, dataset_sha, kind="microdata", revision=release_id, ), - "populace_us_2024_calibration": _artifact_entry( - CALIBRATION_FILENAME, + calibration_key: _artifact_entry( + calibration_filename, calibration_sha, kind="calibration", revision=release_id, @@ -7328,6 +7859,159 @@ def _assert_us_release_id(release_id: str) -> None: ) +def _assert_exact_k_release_id(release_id: str, k: int) -> None: + expected_prefix = f"populace-us-{PERIOD}-k{k}-" + if not release_id.startswith(expected_prefix): + raise ValueError( + "US exact-k ladder release ids must start with " + f"{expected_prefix!r}; got {release_id!r}." + ) + if any(not (character.isalnum() or character == "-") for character in release_id): + raise ValueError( + "US exact-k ladder release ids may contain only ASCII letters, " + f"digits, and hyphens; got {release_id!r}." + ) + + +def _exact_k_ladder_manifest_payload( + *, + args: argparse.Namespace, + outcome: ExactKLadderCalibration, + pool_manifest: Mapping[str, object], + authenticated_pool_h5: AuthenticatedPoolH5, + ledger_artifact: Mapping[str, object], + target_surface: Mapping[str, object], + target_loss_basis: Mapping[str, object], + incumbent_diagnostics_sha256: str, + incumbent_fit_gate: GateResult, + puf_tail_gate: GateResult, +) -> dict[str, object]: + """Build the one receipt block shared by diagnostics and both manifests.""" + + agreement_diagnostics = pool_manifest.get("agreement_diagnostics") + agreement_gate = pool_manifest.get("agreement_gate") + if not all( + isinstance(value, Mapping) for value in (agreement_diagnostics, agreement_gate) + ): + raise RuntimeError("Validated pool manifest lost a required receipt block.") + if agreement_gate.get("passed") is not True: + raise RuntimeError("Validated pool manifest lost its passing agreement gate.") + pool_release_id = _assert_pool_release_id_value( + args.pool_release_id, + authenticated_pool_h5.publication_run_id, + ) + payload = exact_k_ladder_manifest_payload( + outcome, + k=int(args.exact_k), + seed=int(args.seed), + pool={ + "release_id": pool_release_id, + "release_id_source": "pool_manifest.publication_run_id", + "manifest_sha256": authenticated_pool_h5.manifest_sha256, + "publication_run_id": authenticated_pool_h5.publication_run_id, + "pool_h5_sha256": authenticated_pool_h5.sha256, + "pool_h5_size_bytes": authenticated_pool_h5.size_bytes, + "agreement_diagnostics_sha256": agreement_diagnostics.get("sha256"), + }, + agreement_gate_reference={ + "passed": True, + "publication_run_id": authenticated_pool_h5.publication_run_id, + "diagnostics_sha256": agreement_diagnostics.get("sha256"), + "verdict": dict(agreement_gate), + }, + frozen_target_register={ + "ledger_artifact": dict(ledger_artifact), + "target_surface_sha256": target_surface.get("sha256"), + "target_loss_basis": dict(target_loss_basis), + "incumbent_diagnostics_sha256": incumbent_diagnostics_sha256, + "incumbent_fit": { + "passed": incumbent_fit_gate.passed, + "failures": list(incumbent_fit_gate.failures), + "details": dict(incumbent_fit_gate.details), + }, + }, + ) + payload["invariant_battery"] = { + "puf_capital_gains_tail": { + "passed": puf_tail_gate.passed, + "failures": list(puf_tail_gate.failures), + "details": dict(puf_tail_gate.details), + } + } + return payload + + +def _exact_k_original_support_frame( + frame: Frame, + support: np.ndarray, +) -> Frame: + """Subset an original frame by household positions without changing weights.""" + + household_ids = frame.table("household")[ + frame.schema.id_column("household") + ].to_numpy()[np.asarray(support, dtype=np.int64)] + person_households = frame.table("person")[ + frame.schema.membership_column("household") + ] + return frame.select(person_households.isin(household_ids).to_numpy()) + + +def _exact_k_puf_tail_support_gate( + frame: Frame, + support: np.ndarray, +) -> GateResult: + """Batch a post-selection PUF-tail miss without discarding solve evidence.""" + + try: + receipt = assert_puf_capital_gains_tail_survives_selection( + frame, + _exact_k_original_support_frame(frame, support), + require_present=True, + ) + except ValueError as error: + return GateResult( + name="exact_k_puf_capital_gains_tail", + passed=False, + failures=(str(error),), + details={ + "status": "failed", + "error_type": f"{type(error).__module__}.{type(error).__qualname__}", + }, + ) + return GateResult( + name="exact_k_puf_capital_gains_tail", + passed=True, + details={key: value for key, value in receipt.items() if key != "passed"}, + ) + + +def _assert_exact_k_original_pool_alignment( + frame: Frame, + *, + household_ids: np.ndarray, + household_weights: np.ndarray, +) -> None: + """Fail if downstream preparation changed the pool rows or design weights.""" + + observed_weights = frame.weights_for("household") + observed_ids = frame.table("household")["household_id"].to_numpy() + if observed_weights.kind is not WeightKind.IMPORTANCE: + raise RuntimeError( + "Exact-k target preparation changed the original pool weight kind: " + f"got {observed_weights.kind.value!r}, expected 'importance'." + ) + if not np.array_equal(observed_ids, household_ids): + raise RuntimeError( + "Exact-k target preparation changed or reordered the original pool " + "household support." + ) + if not np.array_equal(observed_weights.values, household_weights): + raise RuntimeError( + "Exact-k target preparation changed the original pool weights before " + "selection; the HT-with-q baseline would no longer be frame-original." + ) + + def _staging_telemetry( args: argparse.Namespace, *, @@ -7399,8 +8083,28 @@ def attach_artifact( self._call("attach_artifact", name, path, **details) -def main() -> None: - args = _parse_args() +def _print_build_result( + *, + release_id: str, + release_dir: Path, + artifact_root: Path, +) -> None: + """Print the legacy three-key builder result without widening its API.""" + + print( + json.dumps( + { + "release_id": release_id, + "release_dir": str(release_dir), + "artifact_root": str(artifact_root), + }, + indent=2, + ) + ) + + +def main(argv: Sequence[str] | None = None) -> None: + args = _parse_args(argv) if _git_dirty(): raise SystemExit("Refusing to build a release from a dirty git worktree.") build_started = time.perf_counter() @@ -7416,17 +8120,77 @@ def main() -> None: _refuse_certified_release_dir_reuse( args.out.resolve() / "releases" / args.release_id ) - base_h5 = args.base_h5 or _download_base_h5() - base_dataset_sha256 = _sha256(base_h5) + pinned_incumbent_payload: dict[str, object] | None = None + pinned_incumbent_sha256: str | None = None + if args.exact_k is not None: + pinned_incumbent_payload, pinned_incumbent_sha256 = ( + _load_verified_incumbent_diagnostics_payload( + args.incumbent_diagnostics, + expected_sha256=args.incumbent_diagnostics_sha256, + ) + ) + pool_frame: Frame | None = None + pool_original_household_ids: np.ndarray | None = None + pool_original_household_weights: np.ndarray | None = None + pool_manifest_payload: dict[str, object] | None = None + authenticated_pool_h5: AuthenticatedPoolH5 | None = None + if args.pool_manifest is not None: + pool_frame, pool_manifest_payload, authenticated_pool_h5 = ( + load_simulation_ready_us_multispine_pool( + args.pool_manifest, + expected_manifest_sha256=args.pool_manifest_sha256, + ) + ) + base_dataset_sha256 = authenticated_pool_h5.verified_digest( + consumer="builder base dataset identity" + ) + _assert_pool_release_id_value( + args.pool_release_id, + authenticated_pool_h5.publication_run_id, + ) + if args.exact_k == "N": + args.exact_k = int(pool_frame.n("household")) + pool_original_household_ids = pool_frame.table("household")[ + "household_id" + ].to_numpy(copy=True) + pool_original_household_weights = pool_frame.weights_for( + "household" + ).values.copy() + if args.exact_k > pool_frame.n("household"): + raise ValueError( + f"k={args.exact_k} exceeds the pool size " + f"{pool_frame.n('household')}; ladder selection never clamps " + "the requested cardinality." + ) + base_h5 = authenticated_pool_h5.path + else: + base_h5 = args.base_h5 or _download_base_h5() + base_dataset_sha256 = _legacy_base_h5_sha256(base_h5) digest = base_dataset_sha256[:7] build_timestamp = datetime.now(UTC) full_commit = _git_output("rev-parse", "HEAD") commit = _git_output("rev-parse", "--short=12", "HEAD") - release_id = ( - args.release_id - or f"populace-us-2024-{digest}-{commit}-{build_timestamp:%Y%m%dT%H%M%SZ}" + release_id = args.release_id or ( + f"populace-us-2024-k{args.exact_k}-{digest}-{commit}-" + f"{build_timestamp:%Y%m%dT%H%M%SZ}" + if args.exact_k is not None + else (f"populace-us-2024-{digest}-{commit}-{build_timestamp:%Y%m%dT%H%M%SZ}") ) _assert_us_release_id(release_id) + if args.exact_k is not None: + _assert_exact_k_release_id(release_id, args.exact_k) + # The immutable release id is the dataset's exact-count name. Keep the + # files at the registry's canonical paths so a later *manual* pointer + # flip remains loadable by populace-data. + dataset_key = "populace_us_2024" + dataset_filename = DATASET_FILENAME + calibration_key = "populace_us_2024_calibration" + calibration_filename = CALIBRATION_FILENAME + else: + dataset_key = "populace_us_2024" + dataset_filename = DATASET_FILENAME + calibration_key = "populace_us_2024_calibration" + calibration_filename = CALIBRATION_FILENAME congressional_district_vintage_crosswalk = ( load_congressional_district_vintage_crosswalk( args.congressional_district_vintage_crosswalk @@ -7443,7 +8207,9 @@ def main() -> None: else None ) _assert_cd_vintage_support_matches( - base_h5, congressional_district_vintage_crosswalk_metadata + base_h5, + congressional_district_vintage_crosswalk_metadata, + authenticated_pool_h5=authenticated_pool_h5, ) # Preflight (before the expensive calibration): every provision-critical # input leaf the reform-validation configs depend on must be produced by a @@ -7615,7 +8381,10 @@ def main() -> None: if telemetry is not None: telemetry.stage("load_base_frame", message="Loading base population H5.") - base_frame = _load_frame(base_h5) + if pool_frame is None: + base_frame = _load_frame(base_h5) + else: + base_frame = pool_frame capital_gains_tail_presence = assert_puf_capital_gains_tail_survives_selection( base_frame, base_frame, @@ -7627,20 +8396,39 @@ def main() -> None: message="Verified the materialized PUF capital-gains own-tail.", **capital_gains_tail_presence, ) - weeks_unemployed_source_path = ( - args.asec_2023_weeks_unemployed_source - if args.asec_2023_weeks_unemployed_source is not None - else fetch_asec_2023_weeks_unemployed_source() - ) - weeks_unemployed_source = load_asec_2023_weeks_unemployed_source( - weeks_unemployed_source_path - ) - base_frame = with_us_weeks_unemployed( - base_frame, - seed=args.seed, - time_period=PERIOD, - asec_2023_source=weeks_unemployed_source, - ) + if pool_frame is None: + weeks_unemployed_source_path = ( + args.asec_2023_weeks_unemployed_source + if args.asec_2023_weeks_unemployed_source is not None + else fetch_asec_2023_weeks_unemployed_source() + ) + weeks_unemployed_source = load_asec_2023_weeks_unemployed_source( + weeks_unemployed_source_path + ) + base_frame = with_us_weeks_unemployed( + base_frame, + seed=args.seed, + time_period=PERIOD, + asec_2023_source=weeks_unemployed_source, + ) + weeks_unemployed_source_receipt = { + "source_path": str(Path(weeks_unemployed_source_path).resolve()), + "source_sha256": ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, + "source_rows": len(weeks_unemployed_source), + } + weeks_unemployed_message = ( + "Restored measured ASEC LKWEEKS before frozen-support selection " + "and target materialization." + ) + else: + weeks_unemployed_source_receipt = { + "source": "validated_multispine_pool", + "pool_publication_run_id": authenticated_pool_h5.publication_run_id, + } + weeks_unemployed_message = ( + "Verified measured ASEC LKWEEKS before selection and target " + "materialization." + ) weeks_unemployed_gate = us_weeks_unemployed_signal_gate(base_frame) if not weeks_unemployed_gate.passed: if telemetry is not None: @@ -7661,13 +8449,8 @@ def main() -> None: if telemetry is not None: telemetry.stage( "weeks_unemployed_input", - message=( - "Restored measured ASEC LKWEEKS before frozen-support " - "selection and target materialization." - ), - source_path=str(Path(weeks_unemployed_source_path).resolve()), - source_sha256=ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, - source_rows=len(weeks_unemployed_source), + message=weeks_unemployed_message, + **weeks_unemployed_source_receipt, ) # Capture direct ASEC reporter lineage on the FULL clone-aware support. # Frozen-support recovery may retain only a PUF clone; deriving anchors @@ -7739,7 +8522,26 @@ def main() -> None: ) ) - base_frame, base_population_repair = _with_base_population_mass_repair(base_frame) + if pool_frame is None: + base_frame, base_population_repair = _with_base_population_mass_repair( + base_frame + ) + else: + pool_population = _person_population(base_frame) + base_population_repair = { + "method": "preserve_validated_multispine_pool_weights", + "applied": False, + "reason": ( + "Exact-k selection and HT-with-q refit retain the validated " + "pool artifact's original importance-weight baseline." + ), + "initial_population": pool_population, + "benchmark": US_BASE_PERSON_POPULATION_BENCHMARK, + "factor": 1.0, + "initial_relative_error": _base_population_relative_error(pool_population), + "repaired_population": pool_population, + "repaired_relative_error": _base_population_relative_error(pool_population), + } base_frame, social_security_component_repair = ( _with_social_security_component_value_repair(base_frame, target_specs) ) @@ -7794,7 +8596,8 @@ def main() -> None: for failure in base_population_gate.failures ) ) - base_frame = with_us_qbi_input_reconciliation(base_frame) + if pool_frame is None: + base_frame = with_us_qbi_input_reconciliation(base_frame) qbi_inputs_gate = us_qbi_inputs_signal_gate(base_frame) if not qbi_inputs_gate.passed: if telemetry is not None: @@ -7965,12 +8768,13 @@ def main() -> None: for failure in capital_gain_details_gate.failures ) ) - base_frame = with_us_childcare_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - allow_existing_without_source=True, - ) + if pool_frame is None: + base_frame = with_us_childcare_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + allow_existing_without_source=True, + ) childcare_gate = us_childcare_signal_gate(base_frame) if not childcare_gate.passed: if telemetry is not None: @@ -7988,12 +8792,13 @@ def main() -> None: for failure in childcare_gate.failures ) ) - base_frame = with_us_energy_subsidy_input( - base_frame, - seed=args.seed, - time_period=PERIOD, - allow_existing_without_source=True, - ) + if pool_frame is None: + base_frame = with_us_energy_subsidy_input( + base_frame, + seed=args.seed, + time_period=PERIOD, + allow_existing_without_source=True, + ) energy_subsidy_gate = us_energy_subsidy_signal_gate(base_frame) if not energy_subsidy_gate.passed: if telemetry is not None: @@ -8067,11 +8872,12 @@ def main() -> None: "retirement_contribution_inputs", message=("Verifying ASEC-sourced desired retirement-contribution inputs."), ) - base_frame = with_us_retirement_contribution_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_retirement_contribution_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) retirement_contributions_gate = us_retirement_contributions_signal_gate(base_frame) if not retirement_contributions_gate.passed: if telemetry is not None: @@ -8094,11 +8900,12 @@ def main() -> None: "immigration_inputs", message="Deriving SSN card type and immigration status inputs.", ) - base_frame = with_us_immigration_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_immigration_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) immigration_gate = us_immigration_composition_gate(base_frame) if not immigration_gate.passed: if telemetry is not None: @@ -8121,11 +8928,12 @@ def main() -> None: "take_up_inputs", message="Seeding TANF and EITC take-up from administrative rates.", ) - base_frame = with_us_take_up_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_take_up_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) take_up_gate = us_take_up_signal_gate(base_frame) if not take_up_gate.passed: if telemetry is not None: @@ -8147,11 +8955,12 @@ def main() -> None: "hours_worked_inputs", message="Deriving hours-worked inputs from ASEC reported hours.", ) - base_frame = with_us_hours_worked_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_hours_worked_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) hours_worked_gate = us_hours_worked_signal_gate(base_frame) if not hours_worked_gate.passed: if telemetry is not None: @@ -8201,11 +9010,12 @@ def main() -> None: "relationship_inputs", message=("Deriving household-head and marital-status inputs from ASEC."), ) - base_frame = with_us_relationship_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_relationship_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) relationship_inputs_gate = us_relationship_inputs_signal_gate(base_frame) if not relationship_inputs_gate.passed: if telemetry is not None: @@ -8228,11 +9038,12 @@ def main() -> None: "medicare_take_up_input", message="Deriving measured Medicare enrollment from ASEC MCARE.", ) - base_frame = with_us_medicare_take_up_input( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_medicare_take_up_input( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) medicare_take_up_gate = us_medicare_take_up_signal_gate(base_frame) if not medicare_take_up_gate.passed: if telemetry is not None: @@ -8303,11 +9114,12 @@ def main() -> None: "Carrying measured ASEC retirement distributions by account type." ), ) - base_frame = with_us_retirement_distribution_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_retirement_distribution_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) retirement_distributions_gate = us_retirement_distributions_signal_gate(base_frame) early_terminal_gate_failures: list[str] = [] if not retirement_distributions_gate.passed: @@ -8337,11 +9149,12 @@ def main() -> None: "eligibility_inputs", message="Deriving SNAP eligibility and exemption inputs from ASEC.", ) - base_frame = with_us_eligibility_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_eligibility_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) eligibility_inputs_gate = us_eligibility_inputs_signal_gate(base_frame) if not eligibility_inputs_gate.passed: if telemetry is not None: @@ -8367,11 +9180,12 @@ def main() -> None: "factual inputs from PUF qualified tuition." ), ) - base_frame = with_us_education_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_education_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) education_inputs_gate = us_education_inputs_signal_gate(base_frame) if not education_inputs_gate.passed: if telemetry is not None: @@ -8394,11 +9208,12 @@ def main() -> None: "pregnancy_inputs", message="Seeding pregnancy among women 15-44 at the national rate.", ) - base_frame = with_us_pregnancy_inputs( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_pregnancy_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) pregnancy_gate = us_pregnancy_signal_gate(base_frame) if not pregnancy_gate.passed: if telemetry is not None: @@ -8421,11 +9236,12 @@ def main() -> None: "wic_claim_input", message="Assigning WIC claims from USDA FNS category coverage rates.", ) - base_frame = with_us_wic_claim_input( - base_frame, - seed=args.seed, - time_period=PERIOD, - ) + if pool_frame is None: + base_frame = with_us_wic_claim_input( + base_frame, + seed=args.seed, + time_period=PERIOD, + ) wic_claim_gate = us_wic_claim_signal_gate(base_frame) if not wic_claim_gate.passed: if telemetry is not None: @@ -9237,24 +10053,53 @@ def main() -> None: expected_initial_weights=target_frame.resolve_weights("household").values, ) candidate_households = int(target_frame.n("household")) + if args.exact_k is not None and args.exact_k > candidate_households: + raise ValueError( + f"k={args.exact_k} exceeds the pool size {candidate_households}; " + "ladder selection never clamps the requested cardinality." + ) + full_pool_calibration = bool( + args.dense_default_dataset + or (args.exact_k is not None and args.exact_k == candidate_households) + ) l0_refit_lambda = ( None - if args.dense_default_dataset + if full_pool_calibration else args.l0_refit_lambda_share / float(candidate_households) ) target_loss_weights = _fiscal_target_loss_weights( registry, args.target_family_loss_multipliers ) + target_loss_basis = ( + _fiscal_target_loss_basis( + registry, + target_loss_weights, + args.target_family_loss_multipliers, + ) + if args.exact_k is not None + else None + ) if telemetry is not None: telemetry.stage( "calibrating", message=( "Calibrating dense household weights." - if args.dense_default_dataset - else "Selecting sparse L0 support and refitting household weights." + if full_pool_calibration + else ( + "Selecting an exact-k Sampford support and refitting " + "household weights." + if args.exact_k is not None + else ( + "Selecting sparse L0 support and refitting household weights." + ) + ) ), default_dataset_method=( - "dense_no_l0" if args.dense_default_dataset else "l0_refit" + ("full_pool_refit" if args.exact_k is not None else "dense_no_l0") + if full_pool_calibration + else ( + "exact_k_sampford_refit" if args.exact_k is not None else "l0_refit" + ) ), epochs=args.epochs, learning_rate=args.learning_rate, @@ -9265,15 +10110,13 @@ def main() -> None: n_targets=len(registry), n_candidate_households=candidate_households, l0_refit_lambda_share=( - None - if args.dense_default_dataset - else float(args.l0_refit_lambda_share) + None if full_pool_calibration else float(args.l0_refit_lambda_share) ), l0_lambda=l0_refit_lambda, l2_lambda=float(args.l2_lambda), refit_l2_lambda=( None - if args.dense_default_dataset + if full_pool_calibration else float( args.l2_lambda if args.refit_l2_lambda is None @@ -9288,7 +10131,102 @@ def main() -> None: target_compilation_seconds=timing["target_compilation_seconds"], ) calibration_started = time.perf_counter() - if args.dense_default_dataset: + ladder_outcome = None + exact_k_puf_tail_gate: GateResult | None = None + if args.exact_k is not None: + if ( + pool_original_household_ids is None + or pool_original_household_weights is None + ): + raise RuntimeError("Exact-k calibration lost its original pool baseline.") + _assert_exact_k_original_pool_alignment( + target_frame, + household_ids=pool_original_household_ids, + household_weights=pool_original_household_weights, + ) + if l0_refit_lambda is None: + # The full-pool branch does not use L0, but the shared function's + # ignored value stays finite for a single, explicit call shape. + ladder_l0_lambda = float(args.l0_refit_lambda_share) / float( + candidate_households + ) + else: + ladder_l0_lambda = float(l0_refit_lambda) + ladder_outcome = calibrate_exact_k_ladder( + target_frame, + registry.to_target_set(), + k=args.exact_k, + pi_hi=args.exact_k_pi_hi, + seed=args.seed, + epochs=args.epochs, + refit_epochs=args.epochs, + learning_rate=args.learning_rate, + max_weight_ratio=args.max_weight_ratio, + mass="conserve", + l0_lambda=ladder_l0_lambda, + l2_lambda=args.l2_lambda, + refit_l2_lambda=args.refit_l2_lambda, + target_loss_weights=target_loss_weights, + target_loss_cap=US_FISCAL_TARGET_LOSS_CAP, + warm_start_weights=warm_start_weights, + progress_callback=( + telemetry.calibration_progress if telemetry is not None else None + ), + ) + assert_exact_k_realized_count(ladder_outcome, args.exact_k) + result = ladder_outcome.result + exact_k_puf_tail_gate = _exact_k_puf_tail_support_gate( + target_frame, + ladder_outcome.support, + ) + early_terminal_gate_failures.extend( + f"Exact-k PUF capital-gains tail failed: {failure}" + for failure in exact_k_puf_tail_gate.failures + ) + default_dataset = { + "method": ( + "full_pool_refit" + if args.exact_k == candidate_households + else "exact_k_sampford_refit" + ), + "sparse": args.exact_k < candidate_households, + "n_candidate_households": candidate_households, + "n_selected_households": int(args.exact_k), + "n_exported_households": int(result.frame.n("household")), + "l0_lambda_share": ( + None + if args.exact_k == candidate_households + else float(args.l0_refit_lambda_share) + ), + "l0_lambda": ( + None + if args.exact_k == candidate_households + else float(result.l0_lambda) + ), + "selection_epochs": ( + 0 if args.exact_k == candidate_households else int(args.epochs) + ), + "refit_epochs": int(args.epochs), + "selection_l2_lambda": ( + None if args.exact_k == candidate_households else float(args.l2_lambda) + ), + "refit_l2_lambda": float( + args.l2_lambda if args.refit_l2_lambda is None else args.refit_l2_lambda + ), + "selection_final_loss": ( + None + if args.exact_k == candidate_households + else _finite_or_none(result.selection.final_loss) + ), + "refit_initial_loss": _finite_or_none(result.initial_loss), + "refit_final_loss": _finite_or_none(result.final_loss), + "puf_capital_gains_tail_retention": { + "passed": exact_k_puf_tail_gate.passed, + "failures": list(exact_k_puf_tail_gate.failures), + "details": dict(exact_k_puf_tail_gate.details), + }, + } + elif args.dense_default_dataset: result = calibrate( target_frame, registry.to_target_set(), @@ -9369,7 +10307,7 @@ def main() -> None: # the frozen flags for the published diagnostics, and let the gap to the # SSA band counts ship in the scorecard as calibration's residual on the # #470 registry targets — like every other program's take-up miss. - if args.dense_default_dataset: + if full_pool_calibration: export_frame = _with_calibrated_weights( base_frame, np.asarray(result.weights, dtype=np.float64), @@ -9547,16 +10485,30 @@ def main() -> None: # diagnostics writer re-hashes any non-None incumbent path, which would # replay the exact I/O failure the guard just caught (populace#547, # confirm round 2 finding 2). + current_target_surface: Mapping[str, object] | None = None incumbent_diagnostics_path: Path | None = args.incumbent_diagnostics + incumbent_loss_basis: Mapping[str, object] | None = None try: - incumbent_payload = _load_incumbent_diagnostics_payload( - args.incumbent_diagnostics + incumbent_payload = ( + pinned_incumbent_payload + if pinned_incumbent_payload is not None + else _load_incumbent_diagnostics_payload(args.incumbent_diagnostics) ) if args.incumbent_diagnostics is not None: current_target_surface = diagnostics_payload( result, target_registry=registry, )["target_surface"] + if ( + args.exact_k is not None + and current_target_surface.get("sha256") + != args.frozen_target_surface_sha256 + ): + raise ValueError( + "Exact-k target surface does not match the frozen register: " + f"got {current_target_surface.get('sha256')}, expected " + f"{args.frozen_target_surface_sha256}." + ) _assert_incumbent_target_surface_matches( current_target_surface, incumbent_payload, @@ -9570,6 +10522,7 @@ def main() -> None: if args.incumbent_diagnostics is not None else {} ) + incumbent_loss_basis = _incumbent_target_loss_basis(incumbent_payload) except Exception as error: # Degraded-mode guard (populace#547): with earlier terminal failures # pending, an incumbent load/validation crash must record a line and @@ -9579,11 +10532,49 @@ def main() -> None: raise incumbent_diagnostics = {} incumbent_diagnostics_path = None + incumbent_loss_basis = None early_terminal_gate_failures.append( "Incumbent diagnostics could not be loaded/validated in " f"degraded mode; recorded instead of masking earlier failures: " f"{error}" ) + exact_k_ladder_provenance: Mapping[str, object] | None = None + exact_k_incumbent_fit_gate: GateResult | None = None + if args.exact_k is not None: + if ( + ladder_outcome is None + or pool_manifest_payload is None + or authenticated_pool_h5 is None + or exact_k_puf_tail_gate is None + ): + raise RuntimeError( + "Exact-k calibration lost its pool or selection receipt." + ) + if current_target_surface is None: + current_target_surface = diagnostics_payload( + result, + target_registry=registry, + )["target_surface"] + exact_k_incumbent_fit_gate = _exact_k_frozen_register_fit_gate( + result, + incumbent_diagnostics, + target_registry=registry, + target_loss_weights=target_loss_weights, + configured_loss_basis=target_loss_basis, + incumbent_loss_basis=incumbent_loss_basis, + ) + exact_k_ladder_provenance = _exact_k_ladder_manifest_payload( + args=args, + outcome=ladder_outcome, + pool_manifest=pool_manifest_payload, + authenticated_pool_h5=authenticated_pool_h5, + ledger_artifact=ledger_artifact.provenance(), + target_surface=current_target_surface, + target_loss_basis=target_loss_basis, + incumbent_diagnostics_sha256=pinned_incumbent_sha256, + incumbent_fit_gate=exact_k_incumbent_fit_gate, + puf_tail_gate=exact_k_puf_tail_gate, + ) enforced_input_mass_reference_gate = ( None if args.allow_input_mass_drift else input_mass_reference_gate ) @@ -9624,12 +10615,24 @@ def main() -> None: # mode guard lines) ride the same list as every other gate group, so the # diagnostics artifact records them and the terminal batch aborts on # them (populace#547). - gate_failures = [*early_terminal_gate_failures, *gate_failures] + exact_k_fit_failures = ( + [] + if exact_k_incumbent_fit_gate is None + else [ + f"Exact-k frozen-register fit failed: {failure}" + for failure in exact_k_incumbent_fit_gate.failures + ] + ) + gate_failures = [ + *early_terminal_gate_failures, + *exact_k_fit_failures, + *gate_failures, + ] _write_release_calibration_diagnostics( result=result, release_dir=release_dir, registry=registry, - base_h5=base_h5, + base_dataset_sha256=base_dataset_sha256, compilation=compilation, target_profile_gate=target_profile_gate, health_input_gate=health_input_gate, @@ -9651,12 +10654,15 @@ def main() -> None: gate_failures=gate_failures, timing=timing, incumbent_diagnostics_path=incumbent_diagnostics_path, + incumbent_diagnostics_sha256=pinned_incumbent_sha256, incumbent_diagnostics=incumbent_diagnostics, default_dataset=default_dataset, degenerate_input_gate=degenerate_input_gate, ecps_parity_gate=ecps_parity_gate, validation_input_coverage_gate=validation_input_coverage_gate, target_loss_family_multipliers=args.target_family_loss_multipliers, + target_loss_basis=target_loss_basis, + exact_k_ladder=exact_k_ladder_provenance, ) # Terminal-gate batching: evaluate EVERY terminal gate # group and raise once with the full failure list, instead of aborting at @@ -9980,7 +10986,7 @@ def main() -> None: release_dir / FINAL_HOUSEHOLD_WEIGHTS_METADATA_FILENAME, ): stale_evidence.unlink(missing_ok=True) - dataset_path = artifact_root / DATASET_FILENAME + dataset_path = artifact_root / dataset_filename # The export H5 write: everything below (reform smoke, take-up contract, # release manifest sha) reads THIS file, and it must be written only after # the batched pre-export raise so a gate-failed run never produces it. @@ -10061,7 +11067,7 @@ def main() -> None: if telemetry is not None: telemetry.stage("write_calibration_npz", message="Writing calibration NPZ.") - calibration_path = artifact_root / CALIBRATION_FILENAME + calibration_path = artifact_root / calibration_filename _write_npz(calibration_path, result=result, registry=registry) if not args.skip_reform_validation: @@ -10236,6 +11242,11 @@ def main() -> None: if telemetry is not None else None ), + dataset_key=dataset_key, + dataset_filename=dataset_filename, + calibration_key=calibration_key, + calibration_filename=calibration_filename, + exact_k_ladder=exact_k_ladder_provenance, ) if telemetry is not None: telemetry.attach_artifact("build_manifest", release_dir / "build_manifest.json") @@ -10246,16 +11257,15 @@ def main() -> None: telemetry.complete() # Keep a copy of the exact base artifact beside diagnostics for local audit. - shutil.copy2(base_h5, release_root / f"base_{base_h5.name}") - print( - json.dumps( - { - "release_id": release_id, - "release_dir": str(release_dir), - "artifact_root": str(artifact_root), - }, - indent=2, - ) + _copy_base_h5_for_local_audit( + base_h5, + release_root / f"base_{base_h5.name}", + authenticated_pool_h5=authenticated_pool_h5, + ) + _print_build_result( + release_id=release_id, + release_dir=release_dir, + artifact_root=artifact_root, ) diff --git a/tools/score_us_fiscal_targets.py b/tools/score_us_fiscal_targets.py index 0a8651da..93c1eb57 100644 --- a/tools/score_us_fiscal_targets.py +++ b/tools/score_us_fiscal_targets.py @@ -507,10 +507,11 @@ def score_frame( target_materialization_cache_dir=target_materialization_cache_dir, target_materialization_cache_context=target_materialization_cache_context, ) + target_loss_weights = release._fiscal_target_loss_weights(registry) result = score_targets( target_frame, registry.to_target_set(), - target_loss_weights=release._fiscal_target_loss_weights(registry), + target_loss_weights=target_loss_weights, target_loss_cap=release.US_FISCAL_TARGET_LOSS_CAP, options={ "mass": "existing_weights", @@ -637,7 +638,12 @@ def main() -> None: "base_dataset_sha256": release._sha256(h5), "target_compilation": compilation, "target_loss_weighting": release.US_FISCAL_TARGET_LOSS_WEIGHTING, + "target_loss_family_multipliers": None, "target_loss_cap": release.US_FISCAL_TARGET_LOSS_CAP, + "target_loss_basis": release._fiscal_target_loss_basis( + registry, + release._fiscal_target_loss_weights(registry), + ), "gates": gates, }, )