Refresh US bundle to policyengine-us 1.715.2

changelog.d/us-bundle-17152.changed.md

@@ -0,0 +1 @@
+Refresh the bundled US release manifest to policyengine-us 1.715.2 and require explicit data-release compatibility assertions when refreshing bundles across model versions.

pyproject.toml

@@ -49,7 +49,7 @@ uk = [
 ]
 us = [
     "policyengine_core==3.26.1",
-    "policyengine-us==1.700.0",
+    "policyengine-us==1.715.2",
 ]
 dev = [
     "pytest",
@@ -64,7 +64,7 @@ dev = [
     "ruff>=0.9.0",
     "policyengine_core==3.26.1",
     "policyengine-uk==2.88.20",
-    "policyengine-us==1.700.0",
+    "policyengine-us==1.715.2",
     "towncrier>=24.8.0",
     "mypy>=1.11.0",
     "pytest-cov>=5.0.0",

src/policyengine/data/release_manifests/us.json

@@ -5,16 +5,16 @@
   "policyengine_version": "4.13.0",
   "model_package": {
     "name": "policyengine-us",
-    "version": "1.700.0",
-    "sha256": "7633d8aefcaf02d7628f841bc56750606f1d7fe409ff3ae7b0ef7e364a88e945",
-    "wheel_url": "https://files.pythonhosted.org/packages/49/e9/2837a0d98e99efaf4d82aade276eee6eeff419df614863f08e3512961d2d/policyengine_us-1.700.0-py3-none-any.whl"
+    "version": "1.715.2",
+    "sha256": "abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399",
+    "wheel_url": "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl"
   },
   "data_package": {
     "name": "policyengine-us-data",
     "version": "1.115.5",
     "repo_id": "policyengine/policyengine-us-data",
     "release_manifest_path": "release_manifest.json",
-    "release_manifest_revision": "688f972425f5e858fc52bda2b696e0af74fea920"
+    "release_manifest_revision": "d47fb5475144260a75467d2f2e22b2d5d53d4d57"
   },
   "certified_data_artifact": {
     "data_package": {
@@ -23,14 +23,14 @@
     },
     "build_id": "policyengine-us-data-1.115.5",
     "dataset": "enhanced_cps_2024",
-    "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@688f972425f5e858fc52bda2b696e0af74fea920",
+    "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@d47fb5475144260a75467d2f2e22b2d5d53d4d57",
     "sha256": "0a6b961ad363a421bde99f2c8e5d8f20370bcba45fd303050537a25bdd805b14"
   },
   "certification": {
-    "compatibility_basis": "exact_build_model_version",
+    "compatibility_basis": "legacy_compatible_model_package",
     "data_build_id": "policyengine-us-data-1.115.5",
     "built_with_model_version": "1.700.0",
-    "certified_for_model_version": "1.700.0",
+    "certified_for_model_version": "1.715.2",
     "certified_by": "policyengine-us-data release manifest",
     "data_build_fingerprint": "sha256:b0862de383ffcbe45f4ba0aa9c6aaec286cd4c6688c6ccb33f939bc176f9a8a0"
   },

src/policyengine/data/release_manifests/us.trace.tro.jsonld

@@ -45,23 +45,23 @@
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/data_release_manifest"
               },
-              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/688f972425f5e858fc52bda2b696e0af74fea920/release_manifest.json"
+              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/d47fb5475144260a75467d2f2e22b2d5d53d4d57/release_manifest.json"
             },
             {
               "@id": "arrangement/1/location/dataset",
               "@type": "trov:ArtifactLocation",
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/dataset"
               },
-              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/688f972425f5e858fc52bda2b696e0af74fea920/enhanced_cps_2024.h5"
+              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/d47fb5475144260a75467d2f2e22b2d5d53d4d57/enhanced_cps_2024.h5"
             },
             {
               "@id": "arrangement/1/location/model_wheel",
               "@type": "trov:ArtifactLocation",
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/model_wheel"
               },
-              "trov:hasLocation": "https://files.pythonhosted.org/packages/49/e9/2837a0d98e99efaf4d82aade276eee6eeff419df614863f08e3512961d2d/policyengine_us-1.700.0-py3-none-any.whl"
+              "trov:hasLocation": "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl"
             }
           ]
         }
@@ -75,14 +75,14 @@
             "@type": "trov:ResearchArtifact",
             "schema:name": "policyengine.py bundle manifest for us",
             "trov:mimeType": "application/json",
-            "trov:sha256": "194c8b296cc505c6ee54f4791ae4fd6c6cd2fff1aed1854de1031bc9e3472a24"
+            "trov:sha256": "53526a0cc0ed96d2b0970318356101fa3d2802960e4fd007ef7cac250fbcfc47"
           },
           {
             "@id": "composition/1/artifact/data_release_manifest",
             "@type": "trov:ResearchArtifact",
             "schema:name": "policyengine-us-data release manifest 1.115.5",
             "trov:mimeType": "application/json",
-            "trov:sha256": "f5387c6b5acc0507cc965087da5059f59a4c6cb43b3778f13f065355f05d900e"
+            "trov:sha256": "577fbf704da44f63d0432dc3e80f3686eb3a32020d13ca6b7b6cf7eb60b4742c"
           },
           {
             "@id": "composition/1/artifact/dataset",
@@ -94,31 +94,28 @@
           {
             "@id": "composition/1/artifact/model_wheel",
             "@type": "trov:ResearchArtifact",
-            "schema:name": "policyengine-us==1.700.0 wheel",
+            "schema:name": "policyengine-us==1.715.2 wheel",
             "trov:mimeType": "application/zip",
-            "trov:sha256": "7633d8aefcaf02d7628f841bc56750606f1d7fe409ff3ae7b0ef7e364a88e945"
+            "trov:sha256": "abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399"
           }
         ],
         "trov:hasFingerprint": {
           "@id": "composition/1/fingerprint",
           "@type": "trov:CompositionFingerprint",
-          "trov:sha256": "9d62ba7d9209af8d0070073092ea185b109abf3180153f4c864b662c40433911"
+          "trov:sha256": "5fb36ba0da904c19438a0c5a5c996af03bdfcdc9879a2e36532c8b8975b87026"
         }
       },
       "trov:hasPerformance": {
         "@id": "trp/1",
         "@type": "trov:TransparentResearchPerformance",
         "pe:builtWithModelVersion": "1.700.0",
         "pe:certifiedBy": "policyengine-us-data release manifest",
-        "pe:certifiedForModelVersion": "1.700.0",
-        "pe:ciGitRef": "refs/heads/main",
-        "pe:ciGitSha": "9a36be42b27739ec2a509f605c301a8caed1b078",
-        "pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/26760540408",
-        "pe:compatibilityBasis": "exact_build_model_version",
+        "pe:certifiedForModelVersion": "1.715.2",
+        "pe:compatibilityBasis": "legacy_compatible_model_package",
         "pe:dataBuildFingerprint": "sha256:b0862de383ffcbe45f4ba0aa9c6aaec286cd4c6688c6ccb33f939bc176f9a8a0",
         "pe:dataBuildId": "policyengine-us-data-1.115.5",
-        "pe:emittedIn": "github-actions",
-        "rdfs:comment": "Certification of build policyengine-us-data-1.115.5 for policyengine-us 1.700.0.",
+        "pe:emittedIn": "local",
+        "rdfs:comment": "Certification of build policyengine-us-data-1.115.5 for policyengine-us 1.715.2.",
         "trov:accessedArrangement": {
           "@id": "arrangement/1"
         },

src/policyengine/provenance/bundle.py

@@ -42,6 +42,9 @@
 from typing import Optional
 from urllib.request import Request, urlopen
 
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
+from packaging.version import InvalidVersion, Version
+
 from policyengine.provenance.manifest import (
     CountryReleaseManifest,
     get_release_manifest,
@@ -253,6 +256,69 @@ def _metadata_sidecar_path(path: str) -> str:
     return f"{path}.metadata.json"
 
 
+def _specifier_matches(*, version: str, specifier: str) -> bool:
+    try:
+        return Version(version) in SpecifierSet(specifier)
+    except (InvalidSpecifier, InvalidVersion):
+        return False
+
+
+def _release_manifest_has_compatible_model_package(
+    release_manifest_json: dict,
+    *,
+    package_name: str,
+    model_version: str,
+) -> bool:
+    for compatible_model_package in release_manifest_json.get(
+        "compatible_model_packages",
+        [],
+    ):
+        if compatible_model_package.get("name") != package_name:
+            continue
+        if _specifier_matches(
+            version=model_version,
+            specifier=compatible_model_package.get("specifier", ""),
+        ):
+            return True
+    return False
+
+
+def _release_manifest_compatibility_basis(
+    *,
+    release_manifest_json: dict,
+    current_manifest: CountryReleaseManifest,
+    package_name: str,
+    model_version: str,
+    built_with_model_version: str | None,
+    data_build_fingerprint: str | None,
+) -> str:
+    if built_with_model_version == model_version:
+        return "exact_build_model_version"
+
+    current_certification = current_manifest.certification
+    if (
+        current_certification is not None
+        and current_certification.certified_for_model_version == model_version
+        and current_certification.data_build_fingerprint is not None
+        and current_certification.data_build_fingerprint == data_build_fingerprint
+    ):
+        return "matching_data_build_fingerprint"
+
+    if _release_manifest_has_compatible_model_package(
+        release_manifest_json,
+        package_name=package_name,
+        model_version=model_version,
+    ):
+        return "legacy_compatible_model_package"
+
+    raise ValueError(
+        "Data release manifest is not certified for "
+        f"{package_name}=={model_version}. Publish a data release manifest with "
+        "a matching build model, matching data-build fingerprint, or compatible "
+        "model-package specifier before refreshing the bundle."
+    )
+
+
 def _refresh_dataset_path_references_from_data_release(
     manifest_json: dict,
     release_manifest_json: dict,
@@ -566,16 +632,16 @@ def refresh_release_bundle(
             certification_json["data_build_fingerprint"] = data_build_fingerprint
         else:
             certification_json.pop("data_build_fingerprint", None)
-        if built_with_model_version == new_model:
-            certification_json["compatibility_basis"] = "exact_build_model_version"
-        elif data_build_fingerprint is not None:
-            certification_json["compatibility_basis"] = (
-                "matching_data_build_fingerprint"
-            )
-        else:
-            certification_json["compatibility_basis"] = (
-                "legacy_compatible_model_package"
+        certification_json["compatibility_basis"] = (
+            _release_manifest_compatibility_basis(
+                release_manifest_json=release_manifest_json,
+                current_manifest=current,
+                package_name=package_name,
+                model_version=new_model,
+                built_with_model_version=built_with_model_version,
+                data_build_fingerprint=data_build_fingerprint,
             )
+        )
         _refresh_dataset_path_references_from_data_release(
             manifest_json,
             release_manifest_json,

src/policyengine/provenance/manifest.py

@@ -322,6 +322,8 @@ def certify_data_release_compatibility(
             if (
                 runtime_data_build_fingerprint is not None
                 and bundled_certification.data_build_fingerprint is not None
+                and bundled_certification.compatibility_basis
+                != "legacy_compatible_model_package"
                 and runtime_data_build_fingerprint
                 != bundled_certification.data_build_fingerprint
             ):

src/policyengine/tax_benefit_models/us/datasets.py

@@ -567,6 +567,13 @@ def _runtime_policyengine_us_metadata() -> dict[str, Any]:
     return result
 
 
+def _runtime_policyengine_us_version() -> Optional[str]:
+    try:
+        return importlib_metadata.version("policyengine-us")
+    except importlib_metadata.PackageNotFoundError:
+        return None
+
+
 def _runtime_policyengine_us_package_file() -> Optional[Path]:
     spec = importlib.util.find_spec("policyengine_us")
     if spec is None or spec.origin is None:
@@ -629,6 +636,30 @@ def _validate_runtime_policyengine_us_match(
             )
 
 
+def _validate_runtime_policyengine_us_version(expected_version: Optional[str]) -> None:
+    if expected_version is None:
+        return
+    runtime_version = _runtime_policyengine_us_version()
+    if runtime_version != expected_version:
+        raise ValueError(
+            "Managed long-term datasets require policyengine-us runtime "
+            f"version {expected_version!r}, but the installed runtime is "
+            f"{runtime_version!r}."
+        )
+
+
+def _managed_long_term_dataset_model_version(manifest: Any) -> Optional[str]:
+    certification = getattr(manifest, "certification", None)
+    built_with_model_version = getattr(
+        certification,
+        "built_with_model_version",
+        None,
+    )
+    if built_with_model_version:
+        return built_with_model_version
+    return manifest.model_package.version
+
+
 def validate_long_term_dataset_metadata(
     metadata: dict,
     *,
@@ -1006,7 +1037,14 @@ def load_managed_long_term_datasets(
 
     manifest = get_release_manifest("us")
     if required_policyengine_us_version is None:
-        required_policyengine_us_version = manifest.model_package.version
+        required_policyengine_us_version = _managed_long_term_dataset_model_version(
+            manifest
+        )
+    runtime_policyengine_us_version = manifest.model_package.version
+    require_metadata_runtime_match = (
+        require_runtime_policyengine_us_match
+        and required_policyengine_us_version == runtime_policyengine_us_version
+    )
 
     result = {}
     for year in years:
@@ -1085,8 +1123,10 @@ def load_managed_long_term_datasets(
             required_policyengine_us_version=required_policyengine_us_version,
             required_policyengine_us_git_sha=required_policyengine_us_git_sha,
             require_policyengine_us_clean_build=require_policyengine_us_clean_build,
-            require_runtime_policyengine_us_match=require_runtime_policyengine_us_match,
+            require_runtime_policyengine_us_match=require_metadata_runtime_match,
         )
+        if require_runtime_policyengine_us_match:
+            _validate_runtime_policyengine_us_version(runtime_policyengine_us_version)
         result[key] = _build_long_term_dataset(
             path=path,
             year=year,

src/policyengine/tax_benefit_models/us/model.py

@@ -187,6 +187,12 @@ def run(self, simulation: "Simulation") -> "Simulation":
         # leaves the module-level one untouched. Building populations
         # against the module-level system would hide reform-registered
         # variables like ``ctc_minimum_refundable_amount`` at calc time.
+        if microsim.baseline is not None:
+            self._build_simulation_from_dataset(
+                microsim.baseline,
+                dataset,
+                microsim.baseline.tax_benefit_system,
+            )
         self._build_simulation_from_dataset(
             microsim, dataset, microsim.tax_benefit_system
         )

tests/fixtures/household_calculator_snapshots/us_model_surface.json

@@ -5,7 +5,7 @@
   "has_income_tax": true,
   "has_region_registry": true,
   "model_package_name": "policyengine-us",
-  "num_parameters_bucketed_100s": 873,
-  "num_variables_bucketed_100s": 49,
+  "num_parameters_bucketed_100s": 883,
+  "num_variables_bucketed_100s": 51,
   "region_registry_country": "us"
 }

tests/fixtures/household_calculator_snapshots/us_single_adult_no_income.json

@@ -20,7 +20,7 @@
   "person[0].is_child": 0.0,
   "person[0].is_male": 1.0,
   "person[0].marital_unit_id": 0.0,
-  "person[0].medicaid": 6439.11,
+  "person[0].medicaid": 9236.48,
   "person[0].medicare_cost": 0.0,
   "person[0].person_id": 0.0,
   "person[0].person_weight": 1.0,

tests/fixtures/household_calculator_snapshots/us_single_parent_one_child.json

@@ -38,7 +38,7 @@
   "person[1].is_child": 1.0,
   "person[1].is_male": 1.0,
   "person[1].marital_unit_id": 0.0,
-  "person[1].medicaid": 3258.31,
+  "person[1].medicaid": 7254.54,
   "person[1].medicare_cost": 0.0,
   "person[1].person_id": 1.0,
   "person[1].person_weight": 1.0,