diff --git a/packages/populace-build/README.md b/packages/populace-build/README.md index 786cf6d7..267056b5 100644 --- a/packages/populace-build/README.md +++ b/packages/populace-build/README.md @@ -77,6 +77,24 @@ It accepts already-pooled or already-cloned household pools, so the compact UK artifact can remain the fast national default while a separate `local` variant scales up with pooled FRS years, cloned records, and L0 budget control. +Build the row-wise local-geography H5 from a compact Populace UK H5 with: + +```bash +uv run --project packages/populace-build --extra uk python \ + tools/build_uk_rowwise_dataset.py \ + --input-h5 /path/to/populace_uk_2023.h5 \ + --out /tmp/populace-uk-rowwise \ + --n-clones 2 \ + --constituency-codes /path/to/constituencies_2024.csv \ + --la-codes /path/to/local_authorities_2021.csv +``` + +If `--crosswalk` is omitted, the driver builds +`uk_official_geography_crosswalk.csv.gz` from public ONS, NRS, NISRA, and +postcode sources. It writes the cloned row-wise H5, a geography coverage CSV, +and `rowwise_build_manifest.json` with input/output hashes, row counts, target +coverage, weight preservation, and weakest local-support diagnostics. + ## US plan status `populace.build.us` declares the US build: stage order, donor graph with diff --git a/packages/populace-build/tests/test_uk_rowwise_build_driver.py b/packages/populace-build/tests/test_uk_rowwise_build_driver.py new file mode 100644 index 00000000..41ede826 --- /dev/null +++ b/packages/populace-build/tests/test_uk_rowwise_build_driver.py @@ -0,0 +1,423 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd +import pytest + + +def _load_builder_module(): + root = Path(__file__).resolve().parents[3] + path = root / "tools" / "build_uk_rowwise_dataset.py" + spec = importlib.util.spec_from_file_location("build_uk_rowwise_dataset", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _write_toy_h5( + path: Path, + *, + regions: tuple[str, str] = ("LONDON", "WALES"), + time_period: str = "2023", +) -> None: + with pd.HDFStore(path) as store: + store.put( + "household", + pd.DataFrame( + { + "household_id": [1, 2], + "household_weight": [10.0, 20.0], + "region": list(regions), + } + ), + format="table", + data_columns=True, + ) + store.put( + "person", + pd.DataFrame( + { + "person_id": [1001, 2001, 2002], + "person_household_id": [1, 2, 2], + "person_benunit_id": [101, 201, 201], + } + ), + format="table", + data_columns=True, + ) + store.put( + "benunit", + pd.DataFrame({"benunit_id": [101, 201]}), + format="table", + data_columns=True, + ) + store.put( + "time_period", + pd.Series([time_period]), + format="table", + data_columns=True, + ) + + +def _crosswalk_frame() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "oa_code": "E0001", + "lsoa_code": "E0101", + "msoa_code": "E0201", + "la_code": "E06000063", + "constituency_code": "E14000001", + "region_code": "E12000007", + "country": "England", + "population": 100, + }, + { + "oa_code": "W0001", + "lsoa_code": "W0101", + "msoa_code": "W0201", + "la_code": "W06000001", + "constituency_code": "W07000041", + "region_code": "W99999999", + "country": "Wales", + "population": 80, + }, + { + "oa_code": "S0001", + "lsoa_code": "S0101", + "msoa_code": "S0201", + "la_code": "S12000033", + "constituency_code": "S14000001", + "region_code": "S99999999", + "country": "Scotland", + "population": 90, + }, + { + "oa_code": "N20000001", + "lsoa_code": "N20000001", + "msoa_code": "N21000001", + "la_code": "N09000001", + "constituency_code": "N05000001", + "region_code": "N99999999", + "country": "Northern Ireland", + "population": 70, + }, + ] + ) + + +def test_build_uk_rowwise_dataset_writes_manifest_and_outputs(monkeypatch, tmp_path): + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2023.h5" + crosswalk_path = tmp_path / "crosswalk.csv.gz" + constituency_codes = tmp_path / "constituencies.csv" + la_codes = tmp_path / "local_authorities.csv" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5) + _crosswalk_frame().to_csv(crosswalk_path, index=False) + pd.DataFrame({"code": ["E14000001", "W07000041", "S14000001", "N05000001"]}).to_csv( + constituency_codes, index=False + ) + pd.DataFrame({"code": ["E06000063", "W06000001", "S12000033", "N09000001"]}).to_csv( + la_codes, index=False + ) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--out", + str(output_dir), + "--crosswalk", + str(crosswalk_path), + "--constituency-codes", + str(constituency_codes), + "--la-codes", + str(la_codes), + "--n-clones", + "2", + "--allow-missing-country", + "--allow-constituency-collisions", + ], + ) + + assert builder.main() == 0 + + output_h5 = output_dir / "populace_uk_2023_rowwise.h5" + manifest_path = output_dir / builder.MANIFEST_FILENAME + coverage_path = output_dir / builder.COVERAGE_FILENAME + assert output_h5.exists() + assert manifest_path.exists() + assert coverage_path.exists() + manifest = json.loads(manifest_path.read_text()) + assert manifest["build_kind"] == "uk_rowwise_local_geography_dataset" + assert manifest["parameters"]["n_clones"] == 2 + assert manifest["parameters"]["source_year"] == 2023 + assert manifest["parameters"]["require_all_countries"] is False + assert manifest["base_dataset"]["household_weight_sum"] == pytest.approx(30.0) + assert manifest["rowwise_dataset"]["household_weight_sum"] == pytest.approx(30.0) + assert manifest["rowwise_dataset"]["household_weight_delta"] == pytest.approx(0.0) + assert manifest["rowwise_dataset"]["missing_geography_rows"] == 0 + assert manifest["rowwise_dataset"]["assigned_constituencies"] == 2 + assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 2 + assert manifest["coverage"][0]["covered_areas"] == 4 + assert manifest["outputs"]["crosswalk"] is None + with pd.HDFStore(output_h5, mode="r") as store: + assert store["household"].shape[0] == 4 + assert store["person"].shape[0] == 6 + assert store["benunit"].shape[0] == 4 + + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--out", + str(output_dir), + "--crosswalk", + str(crosswalk_path), + "--n-clones", + "1", + "--allow-missing-country", + ], + ) + + assert builder.main() == 0 + assert not coverage_path.exists() + manifest = json.loads(manifest_path.read_text()) + assert manifest["coverage"] == [] + assert manifest["outputs"]["coverage_summary"] is None + + +def test_build_uk_rowwise_dataset_rejects_target_csv_without_code(tmp_path): + builder = _load_builder_module() + bad_codes = tmp_path / "bad.csv" + bad_codes.write_text("name\nAldershot\n") + + with pytest.raises(ValueError, match="code"): + builder._read_code_csv(bad_codes) + + +def test_build_uk_rowwise_dataset_counts_blank_geography(monkeypatch, tmp_path): + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2023.h5" + crosswalk_path = tmp_path / "england_only_crosswalk.csv.gz" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5, regions=("LONDON", "SCOTLAND")) + _crosswalk_frame().iloc[:1].to_csv(crosswalk_path, index=False) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--out", + str(output_dir), + "--crosswalk", + str(crosswalk_path), + "--n-clones", + "1", + "--allow-missing-country", + ], + ) + + assert builder.main() == 0 + + manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) + assert manifest["rowwise_dataset"]["missing_geography_rows"] == 1 + assert manifest["rowwise_dataset"]["assigned_constituencies"] == 1 + assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 1 + assert ( + manifest["rowwise_dataset"]["duplicate_source_household_constituency_pairs"] + == 0 + ) + + +def test_build_uk_rowwise_dataset_infers_source_year_from_h5(monkeypatch, tmp_path): + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2024.h5" + crosswalk_path = tmp_path / "crosswalk.csv.gz" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5, time_period="2024") + _crosswalk_frame().to_csv(crosswalk_path, index=False) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--out", + str(output_dir), + "--crosswalk", + str(crosswalk_path), + "--n-clones", + "1", + "--allow-missing-country", + "--allow-constituency-collisions", + ], + ) + + assert builder.main() == 0 + + manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) + assert manifest["parameters"]["source_year"] == 2024 + assert manifest["rowwise_dataset"]["time_period"] == "2024" + assert (output_dir / "populace_uk_2024_rowwise.h5").exists() + with pd.HDFStore(output_dir / "populace_uk_2024_rowwise.h5", mode="r") as store: + household = store["household"] + assert household["source_year"].unique().tolist() == [2024] + assert household["source_household_key"].tolist() == [ + "2024:1", + "2024:2", + ] + + +@pytest.mark.parametrize( + "dataset_filename", + [ + "../escaped.h5", + "/tmp/escaped.h5", + "rowwise_build_manifest.json", + "geography_coverage_summary.csv", + "uk_official_geography_crosswalk.csv.gz", + ], +) +def test_dataset_output_path_rejects_paths_and_reserved_names( + dataset_filename, tmp_path +): + builder = _load_builder_module() + + with pytest.raises(ValueError, match="dataset-filename"): + builder._dataset_output_path( + tmp_path, + dataset_filename=dataset_filename, + source_year=2023, + ) + + +def test_validate_output_paths_rejects_crosswalk_collision(tmp_path): + builder = _load_builder_module() + crosswalk = tmp_path / "rowwise.h5" + args = type( + "Args", + (), + { + "out": tmp_path, + "crosswalk": crosswalk, + }, + ) + + with pytest.raises(ValueError, match="differ"): + builder._validate_output_paths( + input_h5=tmp_path / "source.h5", + output_h5=crosswalk, + args=args, + ) + + +@pytest.mark.parametrize( + "sidecar_name", + ["rowwise_build_manifest.json", "geography_coverage_summary.csv"], +) +def test_validate_output_paths_rejects_supplied_crosswalk_sidecar_collision( + sidecar_name, + tmp_path, +): + builder = _load_builder_module() + sidecar_path = tmp_path / sidecar_name + args = type( + "Args", + (), + { + "out": tmp_path, + "crosswalk": sidecar_path, + }, + ) + + with pytest.raises(ValueError, match="crosswalk.*sidecars"): + builder._validate_output_paths( + input_h5=tmp_path / "source.h5", + output_h5=tmp_path / "rowwise.h5", + args=args, + ) + + +def test_load_or_build_crosswalk_unlinks_stale_generated_sidecar(tmp_path): + builder = _load_builder_module() + output_dir = tmp_path / "out" + output_dir.mkdir() + supplied_crosswalk = tmp_path / "supplied_crosswalk.csv.gz" + stale_generated_crosswalk = output_dir / builder.CROSSWALK_FILENAME + _crosswalk_frame().to_csv(supplied_crosswalk, index=False) + stale_generated_crosswalk.write_text("stale") + args = type( + "Args", + (), + { + "out": output_dir, + "crosswalk": supplied_crosswalk, + }, + ) + + source = builder._load_or_build_crosswalk(args) + + assert source.generated is False + assert source.path == supplied_crosswalk.resolve() + assert not stale_generated_crosswalk.exists() + + +def test_load_or_build_crosswalk_keeps_supplied_generated_path(tmp_path): + builder = _load_builder_module() + output_dir = tmp_path / "out" + output_dir.mkdir() + supplied_crosswalk = output_dir / builder.CROSSWALK_FILENAME + _crosswalk_frame().to_csv(supplied_crosswalk, index=False) + args = type( + "Args", + (), + { + "out": output_dir, + "crosswalk": supplied_crosswalk, + }, + ) + + source = builder._load_or_build_crosswalk(args) + + assert source.generated is False + assert source.path == supplied_crosswalk.resolve() + assert supplied_crosswalk.exists() + + +def test_build_uk_rowwise_dataset_rejects_overwriting_input(monkeypatch, tmp_path): + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2023_rowwise.h5" + _write_toy_h5(input_h5) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--out", + str(tmp_path), + ], + ) + + with pytest.raises(ValueError, match="must differ"): + builder.main() diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py new file mode 100644 index 00000000..f97118bb --- /dev/null +++ b/tools/build_uk_rowwise_dataset.py @@ -0,0 +1,415 @@ +"""Build a Populace UK row-wise local-geography dataset. + +This is the narrow build driver for the UK local replacement path. It starts +from an existing compact Populace UK single-year H5, builds or loads the +official-source geography crosswalk, clones the entity tables, assigns each +household a finest available geography row, and writes diagnostics that prove +coverage and weight preservation. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pandas as pd + +from populace.build.uk import ( + build_official_uk_geography_crosswalk, + clone_uk_dataset_with_rowwise_geography, + geography_coverage_summary, + validate_geography_coverage, + write_geography_crosswalk, +) + +CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" +DATASET_FILENAME_TEMPLATE = "populace_uk_{source_year}_rowwise.h5" +MANIFEST_FILENAME = "rowwise_build_manifest.json" +COVERAGE_FILENAME = "geography_coverage_summary.csv" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--input-h5", + type=Path, + required=True, + help="Compact Populace UK single-year H5 to clone.", + ) + parser.add_argument( + "--out", + type=Path, + required=True, + help="Output directory for the row-wise H5 and diagnostics.", + ) + parser.add_argument( + "--crosswalk", + type=Path, + help=( + "Optional existing official geography crosswalk CSV/CSV.GZ. If omitted, " + "the driver downloads public source tables and builds one." + ), + ) + parser.add_argument( + "--constituency-codes", + type=Path, + help="Optional CSV containing a `code` column for constituency coverage checks.", + ) + parser.add_argument( + "--la-codes", + type=Path, + help="Optional CSV containing a `code` column for local-authority coverage checks.", + ) + parser.add_argument("--n-clones", type=int, default=2) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--source-year", + type=int, + help="Source year for cloned household lineage. Defaults to the input H5 time_period.", + ) + parser.add_argument( + "--dataset-filename", + help=( + f"Output H5 filename within --out. Defaults to {DATASET_FILENAME_TEMPLATE}." + ), + ) + parser.add_argument( + "--allow-missing-country", + action="store_true", + help="Do not require all UK countries to appear in the input H5.", + ) + parser.add_argument( + "--allow-blank-constituency", + action="store_true", + help="Allow blank constituency codes in the crosswalk.", + ) + parser.add_argument( + "--allow-cross-region-assignment", + action="store_true", + help="Allow households to draw geography from any UK region in their country.", + ) + parser.add_argument( + "--allow-constituency-collisions", + action="store_true", + help="Allow the same source household to be assigned to the same constituency across clones.", + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + input_h5 = args.input_h5.resolve() + base_summary = _h5_summary(input_h5) + source_year = _source_year(args.source_year, base_summary=base_summary) + output_h5 = _dataset_output_path( + args.out, + dataset_filename=args.dataset_filename, + source_year=source_year, + ) + _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) + crosswalk_source = _load_or_build_crosswalk(args) + crosswalk = crosswalk_source.frame + crosswalk_path = crosswalk_source.path + area_codes_by_type = _area_codes_by_type(args) + coverage = _validate_optional_coverage(crosswalk, area_codes_by_type) + + result = clone_uk_dataset_with_rowwise_geography( + input_h5, + crosswalk, + output_path=output_h5, + n_clones=args.n_clones, + seed=args.seed, + source_year=source_year, + require_all_countries=not args.allow_missing_country, + require_constituency=not args.allow_blank_constituency, + constrain_to_region=not args.allow_cross_region_assignment, + avoid_constituency_collisions=not args.allow_constituency_collisions, + ) + rowwise_summary = _rowwise_summary(result, base_summary=base_summary) + coverage_path = args.out / COVERAGE_FILENAME + coverage_artifact = None + if not coverage.empty: + coverage.to_csv(coverage_path, index=False) + coverage_artifact = _artifact_info(coverage_path) + else: + coverage_path.unlink(missing_ok=True) + + manifest = { + "schema_version": 1, + "build_kind": "uk_rowwise_local_geography_dataset", + "created_at": datetime.now(UTC).isoformat(), + "git_commit": _git_commit(), + "parameters": { + "n_clones": args.n_clones, + "seed": args.seed, + "source_year": source_year, + "require_all_countries": not args.allow_missing_country, + "require_constituency": not args.allow_blank_constituency, + "constrain_to_region": not args.allow_cross_region_assignment, + "avoid_constituency_collisions": not args.allow_constituency_collisions, + }, + "inputs": { + "dataset": _artifact_info(input_h5), + "crosswalk": _artifact_info(crosswalk_path), + }, + "outputs": { + "dataset": _artifact_info(output_h5), + "crosswalk": ( + _artifact_info(crosswalk_path) if crosswalk_source.generated else None + ), + "coverage_summary": coverage_artifact, + }, + "base_dataset": base_summary, + "rowwise_dataset": rowwise_summary, + "coverage": coverage.to_dict("records") if not coverage.empty else [], + } + manifest_path = args.out / MANIFEST_FILENAME + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, indent=2, sort_keys=True)) + return 0 + + +class CrosswalkSource: + def __init__(self, frame: pd.DataFrame, path: Path, *, generated: bool) -> None: + self.frame = frame + self.path = path + self.generated = generated + + +def _dataset_output_path( + out_dir: Path, + *, + dataset_filename: str | None, + source_year: int, +) -> Path: + filename = dataset_filename or DATASET_FILENAME_TEMPLATE.format( + source_year=source_year + ) + path = Path(filename) + if path.is_absolute() or path.name != filename or path.name in {"", ".", ".."}: + raise ValueError("--dataset-filename must be a filename, not a path.") + reserved = {CROSSWALK_FILENAME, MANIFEST_FILENAME, COVERAGE_FILENAME} + if path.name in reserved: + raise ValueError( + f"--dataset-filename must not use reserved name {path.name!r}." + ) + return out_dir / path.name + + +def _validate_output_paths( + *, + input_h5: Path, + output_h5: Path, + args: argparse.Namespace, +) -> None: + output_sidecars = { + (args.out / MANIFEST_FILENAME).resolve(), + (args.out / COVERAGE_FILENAME).resolve(), + } + generated_crosswalk_path = (args.out / CROSSWALK_FILENAME).resolve() + reserved_paths = { + input_h5, + *output_sidecars, + } + if args.crosswalk is None: + reserved_paths.add(generated_crosswalk_path) + else: + crosswalk_path = args.crosswalk.resolve() + if crosswalk_path in output_sidecars: + raise ValueError("--crosswalk path must differ from output sidecars.") + reserved_paths.add(crosswalk_path) + if output_h5.resolve() in reserved_paths: + raise ValueError("Output H5 path must differ from inputs and sidecars.") + + +def _source_year(cli_source_year: int | None, *, base_summary: dict[str, Any]) -> int: + if cli_source_year is not None: + return cli_source_year + time_period = base_summary.get("time_period") + if time_period is None: + raise ValueError( + "Could not infer source year from input H5 time_period; pass --source-year." + ) + try: + return int(str(time_period)[:4]) + except ValueError as exc: + raise ValueError( + "Could not infer source year from input H5 time_period; pass --source-year." + ) from exc + + +def _load_or_build_crosswalk(args: argparse.Namespace) -> CrosswalkSource: + if args.crosswalk is not None: + path = args.crosswalk.resolve() + generated_crosswalk_path = args.out / CROSSWALK_FILENAME + if path != generated_crosswalk_path.resolve(): + generated_crosswalk_path.unlink(missing_ok=True) + return CrosswalkSource(_read_crosswalk(path), path, generated=False) + crosswalk = build_official_uk_geography_crosswalk() + path = args.out / CROSSWALK_FILENAME + write_geography_crosswalk(crosswalk, path) + return CrosswalkSource(crosswalk, path, generated=True) + + +def _read_crosswalk(path: Path) -> pd.DataFrame: + return pd.read_csv( + path, + dtype={ + "oa_code": str, + "lsoa_code": str, + "msoa_code": str, + "la_code": str, + "constituency_code": str, + "region_code": str, + "country": str, + }, + ) + + +def _area_codes_by_type(args: argparse.Namespace) -> dict[str, list[str]]: + area_codes: dict[str, list[str]] = {} + if args.constituency_codes is not None: + area_codes["constituency"] = _read_code_csv(args.constituency_codes) + if args.la_codes is not None: + area_codes["la"] = _read_code_csv(args.la_codes) + return area_codes + + +def _read_code_csv(path: Path) -> list[str]: + frame = pd.read_csv(path, dtype=str) + if "code" not in frame.columns: + raise ValueError(f"{path} must include a `code` column.") + return frame["code"].dropna().astype(str).str.strip().tolist() + + +def _validate_optional_coverage( + crosswalk: pd.DataFrame, + area_codes_by_type: dict[str, list[str]], +) -> pd.DataFrame: + if not area_codes_by_type: + return pd.DataFrame() + validate_geography_coverage( + crosswalk, + required_countries=["England", "Wales", "Scotland", "Northern Ireland"], + area_codes_by_type=area_codes_by_type, + ) + return geography_coverage_summary(crosswalk, area_codes_by_type) + + +def _h5_summary(path: Path) -> dict[str, Any]: + with pd.HDFStore(path, mode="r") as store: + household = store["household"] + return { + "path": str(path), + "tables": {key.strip("/"): list(store[key].shape) for key in store.keys()}, + "household_weight_sum": float(household["household_weight"].sum()), + "time_period": str(store["time_period"].iloc[0]), + } + + +def _rowwise_summary(result, *, base_summary: dict[str, Any]) -> dict[str, Any]: + household = result.household + geo_columns = ( + "oa_code", + "lsoa_code", + "msoa_code", + "la_code_oa", + "constituency_code_oa", + "region_code_oa", + ) + missing_geography = household[list(geo_columns)].isna().any(axis=1) + for column in geo_columns: + missing_geography |= household[column].fillna("").astype(str).str.strip().eq("") + assigned_constituencies = household.loc[ + _nonblank_string_mask(household["constituency_code_oa"]), + "constituency_code_oa", + ] + assigned_las = household.loc[ + _nonblank_string_mask(household["la_code_oa"]), + "la_code_oa", + ] + by_constituency = assigned_constituencies.groupby(assigned_constituencies).size() + by_la = assigned_las.groupby(assigned_las).size() + weight_sum = float(household["household_weight"].sum()) + constituency_rows = _area_row_summary(by_constituency) + la_rows = _area_row_summary(by_la) + return { + "tables": { + "person": list(result.person.shape), + "benunit": list(result.benunit.shape), + "household": list(household.shape), + }, + "time_period": result.time_period, + "n_clones": result.n_clones, + "id_multiplier": result.id_multiplier, + "household_weight_sum": weight_sum, + "household_weight_delta": weight_sum - base_summary["household_weight_sum"], + "missing_geography_rows": int(missing_geography.sum()), + "assigned_constituencies": int(by_constituency.size), + "assigned_local_authorities": int(by_la.size), + "min_household_rows_by_constituency": constituency_rows["min"], + "min_household_rows_by_local_authority": la_rows["min"], + "median_household_rows_by_constituency": constituency_rows["median"], + "median_household_rows_by_local_authority": la_rows["median"], + "duplicate_source_household_constituency_pairs": ( + _duplicate_source_household_constituency_pairs(household) + ), + } + + +def _nonblank_string_mask(values: pd.Series) -> pd.Series: + return values.notna() & values.astype(str).str.strip().ne("") + + +def _area_row_summary(counts: pd.Series) -> dict[str, int | float]: + if counts.empty: + return {"min": 0, "median": 0.0} + return {"min": int(counts.min()), "median": float(counts.median())} + + +def _duplicate_source_household_constituency_pairs(household: pd.DataFrame) -> int: + if "source_household_id" not in household.columns: + return 0 + assigned = household[_nonblank_string_mask(household["constituency_code_oa"])] + return int( + assigned.duplicated(["source_household_id", "constituency_code_oa"]).sum() + ) + + +def _artifact_info(path: Path) -> dict[str, Any]: + return { + "path": str(path), + "sha256": _sha256(path), + "bytes": path.stat().st_size, + } + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _git_commit() -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return result.stdout.strip() + + +if __name__ == "__main__": + raise SystemExit(main())