Update UK reproducibility bundle

.github/workflows/push.yaml

@@ -81,6 +81,8 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18.x
+      - name: Set up Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
       - name: Build HTML Assets
         run: make docs
       - name: Upload artifact

changelog.d/update-policyengine-uk-28814.changed.md

@@ -0,0 +1 @@
+Updated the bundled UK release to policyengine-uk 2.88.14 and policyengine-uk-data 1.55.5.

pyproject.toml

@@ -41,11 +41,11 @@ graph = [
     "networkx>=3.0",
 ]
 uk = [
-    "policyengine_core>=3.25.0",
-    "policyengine-uk==2.88.0",
+    "policyengine_core>=3.26.0",
+    "policyengine-uk==2.88.14",
 ]
 us = [
-    "policyengine_core>=3.25.0",
+    "policyengine_core>=3.26.0",
     "policyengine-us==1.687.0",
 ]
 dev = [
@@ -59,8 +59,8 @@ dev = [
     "plotly>=5.0.0",
     "pytest-asyncio>=0.26.0",
     "ruff>=0.9.0",
-    "policyengine_core>=3.25.0",
-    "policyengine-uk==2.88.0",
+    "policyengine_core>=3.26.0",
+    "policyengine-uk==2.88.14",
     "policyengine-us==1.687.0",
     "towncrier>=24.8.0",
     "mypy>=1.11.0",

src/policyengine/data/release_manifests/uk.json

@@ -5,29 +5,30 @@
   "policyengine_version": "4.3.1",
   "model_package": {
     "name": "policyengine-uk",
-    "version": "2.88.0",
-    "sha256": "46a3ba443b43ec810c5efaccd4645edb63c8dc90ef5acf9b0cdf5ace86b9334d",
-    "wheel_url": "https://files.pythonhosted.org/packages/23/7e/8a2a42eac1da63730a865964aa17e7fd4420ce4db4c80001c1b5ca6011e8/policyengine_uk-2.88.0-py3-none-any.whl"
+    "version": "2.88.14",
+    "sha256": "ed10005ba7d0c973c0966ebbf7672853fb3caaa0456b8bf485fb13f8c323d975",
+    "wheel_url": "https://files.pythonhosted.org/packages/a3/fc/276fb639a46bda35523329d8968bcc4089fde9e97fab82722c0ec853c6cc/policyengine_uk-2.88.14-py3-none-any.whl"
   },
   "data_package": {
     "name": "policyengine-uk-data",
-    "version": "1.40.4",
+    "version": "1.55.5",
     "repo_id": "policyengine/policyengine-uk-data-private"
   },
   "certified_data_artifact": {
     "data_package": {
       "name": "policyengine-uk-data",
-      "version": "1.40.4"
+      "version": "1.55.5"
     },
-    "build_id": "policyengine-uk-data-1.40.4",
+    "build_id": "policyengine-uk-data-1.55.5",
     "dataset": "enhanced_frs_2023_24",
-    "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+    "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
   },
   "certification": {
-    "compatibility_basis": "exact_build_model_version",
-    "data_build_id": "policyengine-uk-data-1.40.4",
-    "built_with_model_version": "2.88.0",
-    "certified_for_model_version": "2.88.0",
+    "compatibility_basis": "matching_data_build_fingerprint",
+    "data_build_id": "policyengine-uk-data-1.55.5",
+    "built_with_model_version": "2.88.14",
+    "certified_for_model_version": "2.88.14",
+    "data_build_fingerprint": "sha256:dff5c5bb976ce254fa965ecfce6a0d84859fe1629a714ae28a79b3075522a0ae",
     "certified_by": "policyengine.py bundled manifest"
   },
   "default_dataset": "enhanced_frs_2023_24",

src/policyengine/data/release_manifests/us.json

@@ -12,7 +12,9 @@
   "data_package": {
     "name": "policyengine-us-data",
     "version": "1.78.2",
-    "repo_id": "policyengine/policyengine-us-data"
+    "repo_id": "policyengine/policyengine-us-data",
+    "release_manifest_path": "releases/1.78.2/release_manifest.json",
+    "release_manifest_revision": "2f1ea16b152cd9db4a4d2f1aad4d42e7484d5999"
   },
   "certified_data_artifact": {
     "data_package": {

src/policyengine/provenance/manifest.py

@@ -31,6 +31,7 @@ class DataPackageVersion(PackageVersion):
     repo_id: str
     repo_type: str = "model"
     release_manifest_path: str = "release_manifest.json"
+    release_manifest_revision: Optional[str] = None
 
 
 class CompatibleModelPackage(BaseModel):
@@ -178,9 +179,10 @@ def https_dataset_uri(repo_id: str, path_in_repo: str, revision: str) -> str:
 
 def https_release_manifest_uri(data_package: "DataPackageVersion") -> str:
     """Return a dereferenceable HTTPS URI for a data release manifest."""
+    revision = data_package.release_manifest_revision or data_package.version
     return (
         f"https://huggingface.co/{data_package.repo_id}/resolve/"
-        f"{data_package.version}/{data_package.release_manifest_path}"
+        f"{revision}/{data_package.release_manifest_path}"
     )
 
 

tests/test_models.py

@@ -29,12 +29,12 @@ def test_has_release_manifest_metadata(self):
         assert uk_latest.release_manifest is not None
         assert uk_latest.release_manifest.country_id == "uk"
         assert uk_latest.model_package.name == "policyengine-uk"
-        assert uk_latest.model_package.version == "2.88.0"
+        assert uk_latest.model_package.version == "2.88.14"
         assert uk_latest.data_package.name == "policyengine-uk-data"
-        assert uk_latest.data_package.version == "1.40.4"
+        assert uk_latest.data_package.version == "1.55.5"
         assert (
             uk_latest.default_dataset_uri
-            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
         )
 
     def test_has_hundreds_of_parameters(self):

tests/test_release_manifests.py

@@ -19,6 +19,7 @@
     dataset_logical_name,
     get_data_release_manifest,
     get_release_manifest,
+    https_release_manifest_uri,
     resolve_dataset_reference,
     resolve_managed_dataset_reference,
 )
@@ -65,6 +66,14 @@ def test__given_us_manifest__then_has_pinned_model_and_data_packages(self):
         assert manifest.data_package.name == "policyengine-us-data"
         assert manifest.data_package.version == "1.78.2"
         assert manifest.data_package.repo_id == "policyengine/policyengine-us-data"
+        assert (
+            manifest.data_package.release_manifest_path
+            == "releases/1.78.2/release_manifest.json"
+        )
+        assert (
+            manifest.data_package.release_manifest_revision
+            == "2f1ea16b152cd9db4a4d2f1aad4d42e7484d5999"
+        )
         assert manifest.certified_data_artifact is not None
         assert (
             manifest.certified_data_artifact.build_id == "policyengine-us-data-1.78.2"
@@ -83,21 +92,25 @@ def test__given_uk_manifest__then_has_pinned_model_and_data_packages(self):
         assert manifest.country_id == "uk"
         assert manifest.policyengine_version == POLICYENGINE_VERSION
         assert manifest.model_package.name == "policyengine-uk"
-        assert manifest.model_package.version == "2.88.0"
+        assert manifest.model_package.version == "2.88.14"
         assert manifest.data_package.name == "policyengine-uk-data"
-        assert manifest.data_package.version == "1.40.4"
+        assert manifest.data_package.version == "1.55.5"
         assert (
             manifest.data_package.repo_id == "policyengine/policyengine-uk-data-private"
         )
         assert manifest.certified_data_artifact is not None
         assert (
-            manifest.certified_data_artifact.build_id == "policyengine-uk-data-1.40.4"
+            manifest.certified_data_artifact.build_id == "policyengine-uk-data-1.55.5"
         )
         assert manifest.certified_data_artifact.dataset == "enhanced_frs_2023_24"
         assert manifest.certification is not None
-        assert manifest.certification.data_build_id == "policyengine-uk-data-1.40.4"
-        assert manifest.certification.built_with_model_version == "2.88.0"
-        assert manifest.certification.certified_for_model_version == "2.88.0"
+        assert manifest.certification.data_build_id == "policyengine-uk-data-1.55.5"
+        assert manifest.certification.built_with_model_version == "2.88.14"
+        assert manifest.certification.certified_for_model_version == "2.88.14"
+        assert (
+            manifest.certification.data_build_fingerprint
+            == "sha256:dff5c5bb976ce254fa965ecfce6a0d84859fe1629a714ae28a79b3075522a0ae"
+        )
 
     def test__given_us_dataset_name__then_resolves_to_versioned_hf_url(self):
         resolved = resolve_dataset_reference("us", "enhanced_cps_2024")
@@ -112,7 +125,7 @@ def test__given_uk_dataset_name__then_resolves_to_versioned_hf_url(self):
 
         assert (
             resolved
-            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
         )
 
     def test__given_explicit_url__then_resolution_is_noop(self):
@@ -162,28 +175,28 @@ def test__given_country__then_can_fetch_data_release_manifest(self):
             "schema_version": 1,
             "data_package": {
                 "name": "policyengine-us-data",
-                "version": "1.73.0",
+                "version": "1.78.2",
             },
             "build": {
-                "build_id": "policyengine-us-data-1.73.0",
+                "build_id": "policyengine-us-data-1.78.2",
                 "built_at": "2026-04-10T12:00:00Z",
                 "built_with_model_package": {
                     "name": "policyengine-us",
-                    "version": "1.602.0",
+                    "version": "1.687.0",
                     "git_sha": "deadbeef",
                     "data_build_fingerprint": "sha256:fingerprint",
                 },
             },
             "compatible_model_packages": [
-                {"name": "policyengine-us", "specifier": "==1.602.0"}
+                {"name": "policyengine-us", "specifier": "==1.687.0"}
             ],
             "default_datasets": {"national": "enhanced_cps_2024"},
             "artifacts": {
                 "enhanced_cps_2024": {
                     "kind": "microdata",
                     "path": "enhanced_cps_2024.h5",
                     "repo_id": "policyengine/policyengine-us-data",
-                    "revision": "1.73.0",
+                    "revision": "1.78.2",
                     "sha256": "abc",
                     "size_bytes": 123,
                 }
@@ -200,15 +213,29 @@ def test__given_country__then_can_fetch_data_release_manifest(self):
         assert manifest.data_package.name == "policyengine-us-data"
         assert manifest.default_datasets["national"] == "enhanced_cps_2024"
         assert manifest.build is not None
-        assert manifest.build.build_id == "policyengine-us-data-1.73.0"
+        assert manifest.build.build_id == "policyengine-us-data-1.78.2"
         assert manifest.build.built_at == "2026-04-10T12:00:00Z"
         assert manifest.build.built_with_model_package is not None
-        assert manifest.build.built_with_model_package.version == "1.602.0"
+        assert manifest.build.built_with_model_package.version == "1.687.0"
         assert (
             manifest.artifacts["enhanced_cps_2024"].uri
-            == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.73.0"
+            == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.78.2"
+        )
+        mock_get.assert_called_once()
+        assert mock_get.call_args.args[0] == (
+            "https://huggingface.co/policyengine/policyengine-us-data/resolve/"
+            "2f1ea16b152cd9db4a4d2f1aad4d42e7484d5999/"
+            "releases/1.78.2/release_manifest.json"
+        )
+
+    def test__given_explicit_manifest_revision__then_builds_manifest_url(self):
+        manifest = get_release_manifest("us")
+
+        assert https_release_manifest_uri(manifest.data_package) == (
+            "https://huggingface.co/policyengine/policyengine-us-data/resolve/"
+            "2f1ea16b152cd9db4a4d2f1aad4d42e7484d5999/"
+            "releases/1.78.2/release_manifest.json"
         )
-        assert mock_get.call_count == 1
 
     def test__given_missing_data_release_manifest__then_fetch_raises_unavailable(self):
         get_data_release_manifest.cache_clear()
@@ -483,9 +510,9 @@ def test__given_manifest_certification__then_release_bundle_exposes_it(self):
         assert bundle["bundle_id"] == f"uk-{POLICYENGINE_VERSION}"
         assert bundle["default_dataset"] == "enhanced_frs_2023_24"
         assert bundle["default_dataset_uri"] == manifest.default_dataset_uri
-        assert bundle["certified_data_build_id"] == "policyengine-uk-data-1.40.4"
-        assert bundle["data_build_model_version"] == "2.88.0"
-        assert bundle["compatibility_basis"] == "exact_build_model_version"
+        assert bundle["certified_data_build_id"] == "policyengine-uk-data-1.55.5"
+        assert bundle["data_build_model_version"] == "2.88.14"
+        assert bundle["compatibility_basis"] == "matching_data_build_fingerprint"
         assert bundle["certified_by"] == "policyengine.py bundled manifest"
 
     def test__given_runtime_certification__then_release_bundle_prefers_runtime_value(
@@ -565,19 +592,19 @@ def test__given_uk_managed_dataset_name__then_resolves_within_bundle(self):
         else:
             assert dataset == (
                 "hf://policyengine/policyengine-uk-data-private/"
-                "enhanced_frs_2023_24.h5@1.40.4"
+                "enhanced_frs_2023_24.h5@1.55.5"
             )
         assert (
             microsim.policyengine_bundle["policyengine_version"] == POLICYENGINE_VERSION
         )
         assert microsim.policyengine_bundle["runtime_dataset"] == "enhanced_frs_2023_24"
         assert microsim.policyengine_bundle["runtime_dataset_uri"] == (
-            "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+            "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
         )
         dataset_source = microsim.policyengine_bundle["runtime_dataset_source"]
         assert (
             dataset_source
-            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
             or str(dataset_source).endswith(
                 "policyengine_uk_data/storage/enhanced_frs_2023_24.h5"
             )

tests/test_uk_regions.py

@@ -68,7 +68,7 @@ def test__given_uk_registry__then_has_national_region(self):
         assert national.region_type == "national"
         assert (
             national.dataset_path
-            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.4"
+            == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.55.5"
         )
         assert not national.requires_filter