From 55981f52a6e1dbc12a321bd907a9fcf431c1b269 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Apr 2026 13:24:31 +0100 Subject: [PATCH 1/9] feat: add Simulation.from_situation for situation-JSON input (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a classmethod to the Python wrapper that accepts the PolicyEngine web-app situation-JSON format (people / benunits / households with `members` lists and period-keyed values) and converts it into the three input DataFrames the Rust engine consumes. Closes #51 in part — the small, low-risk piece. Datasets-from-URL and a direct dataframe entry point can follow in subsequent PRs. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 11 + changelog.d/added/from-situation.md | 1 + .../python/policyengine_uk_compiled/engine.py | 256 ++++++++++++++++ interfaces/python/tests/__init__.py | 0 .../python/tests/test_from_situation.py | 274 ++++++++++++++++++ 5 files changed, 542 insertions(+) create mode 100644 changelog.d/added/from-situation.md create mode 100644 interfaces/python/tests/__init__.py create mode 100644 interfaces/python/tests/test_from_situation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d46c4c..7f55424 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,3 +29,14 @@ jobs: - name: Test run: cargo test + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install Python wrapper test dependencies + run: pip install pandas pydantic pytest + + - name: Test Python wrapper + run: PYTHONPATH=interfaces/python pytest interfaces/python/tests -q diff --git a/changelog.d/added/from-situation.md b/changelog.d/added/from-situation.md new file mode 100644 index 0000000..78eff1d --- /dev/null +++ b/changelog.d/added/from-situation.md @@ -0,0 +1 @@ +Add `Simulation.from_situation(situation, year)` to the Python wrapper, accepting the PolicyEngine web-app situation-JSON format (people / benunits / households with `members` lists and period-keyed values) and converting it into the input DataFrames the Rust engine expects. diff --git a/interfaces/python/policyengine_uk_compiled/engine.py b/interfaces/python/policyengine_uk_compiled/engine.py index 74e670a..92c6d7c 100644 --- a/interfaces/python/policyengine_uk_compiled/engine.py +++ b/interfaces/python/policyengine_uk_compiled/engine.py @@ -120,6 +120,202 @@ def _parse_stdin_payload(payload: str): ) +# Region values accepted in situation dicts → canonical form the Rust engine expects. +# Accepts the upper-snake names used by the PolicyEngine web app and the +# title-case forms used by `PERSON_DEFAULTS`/`HOUSEHOLD_DEFAULTS` and the +# `parse_region` function in `src/data/clean.rs`. +_REGION_CANONICAL = { + "NORTH_EAST": "North East", "North East": "North East", + "NORTH_WEST": "North West", "North West": "North West", + "YORKSHIRE": "Yorkshire", "Yorkshire": "Yorkshire", + "EAST_MIDLANDS": "East Midlands", "East Midlands": "East Midlands", + "WEST_MIDLANDS": "West Midlands", "West Midlands": "West Midlands", + "EAST_OF_ENGLAND": "East of England", "East of England": "East of England", + "LONDON": "London", "London": "London", + "SOUTH_EAST": "South East", "South East": "South East", + "SOUTH_WEST": "South West", "South West": "South West", + "WALES": "Wales", "Wales": "Wales", + "SCOTLAND": "Scotland", "Scotland": "Scotland", + "NORTHERN_IRELAND": "Northern Ireland", "Northern Ireland": "Northern Ireland", +} + + +def _resolve_period_value(value, year: int): + """Pick a value out of a period-keyed dict, or return the scalar unchanged. + + Picks an exact match on ``year`` first, then any period whose first four + characters match (covers ``"2025-01"`` style entries), then the most-recent + period that is not later than ``year``, then the earliest period. + """ + if not isinstance(value, dict): + return value + year_str = str(year) + if year_str in value: + return value[year_str] + for k, v in value.items(): + if str(k)[:4] == year_str: + return v + # Numeric-period fallback + candidates = [] + for k, v in value.items(): + try: + candidates.append((int(str(k)[:4]), v)) + except (ValueError, TypeError): + continue + if not candidates: + # Single non-period entry (e.g. {"ETERNITY": x}) → use it + return next(iter(value.values())) + candidates.sort() + earlier_or_equal = [v for y, v in candidates if y <= year] + return earlier_or_equal[-1] if earlier_or_equal else candidates[0][1] + + +def _situation_to_dataframes(situation: dict, year: int): + """Convert a PolicyEngine situation-JSON dict into the three input DataFrames. + + See ``Simulation.from_situation`` for the supported dict shape. + """ + if not HAS_PANDAS: + raise ImportError("pandas is required for from_situation") + + people = situation.get("people") or {} + benunits = situation.get("benunits") or {} + households = situation.get("households") or {} + + if not people: + raise ValueError("situation must contain at least one entry under 'people'") + if not households: + raise ValueError("situation must contain at least one entry under 'households'") + if not benunits: + # Fold all people into a single implicit benunit so callers don't + # have to supply one for trivial cases. + benunits = {"_default": {"members": list(people.keys())}} + + person_id_map = {pid: i for i, pid in enumerate(people.keys())} + benunit_id_map = {bid: i for i, bid in enumerate(benunits.keys())} + household_id_map = {hid: i for i, hid in enumerate(households.keys())} + + # Build reverse lookups: person → benunit, person → household + person_to_benunit: dict[str, str] = {} + for bid, fields in benunits.items(): + for member in (fields.get("members") or []): + person_to_benunit[member] = bid + person_to_household: dict[str, str] = {} + for hid, fields in households.items(): + for member in (fields.get("members") or []): + person_to_household[member] = hid + + person_rows = [] + for pid, fields in people.items(): + if pid not in person_to_benunit: + raise ValueError(f"person {pid!r} is not a member of any benunit") + if pid not in person_to_household: + raise ValueError(f"person {pid!r} is not a member of any household") + row = dict(PERSON_DEFAULTS) + row["person_id"] = person_id_map[pid] + row["benunit_id"] = benunit_id_map[person_to_benunit[pid]] + row["household_id"] = household_id_map[person_to_household[pid]] + for var, val in (fields or {}).items(): + if var == "members": + continue + resolved = _resolve_period_value(val, year) + if var == "gender" and isinstance(resolved, str): + resolved = resolved.lower() + row[var] = resolved + person_rows.append(row) + + # Mark the first member of each benunit as benunit head, and the first + # member of each household as household head, unless the situation + # already specified these flags. + seen_bu_head: set[int] = set() + seen_hh_head: set[int] = set() + explicit_bu_head: set[str] = set() + explicit_hh_head: set[str] = set() + for pid, fields in people.items(): + if "is_benunit_head" in (fields or {}): + explicit_bu_head.add(pid) + if "is_household_head" in (fields or {}): + explicit_hh_head.add(pid) + for pid, row in zip(people.keys(), person_rows): + bu = row["benunit_id"] + hh = row["household_id"] + if pid in explicit_bu_head: + seen_bu_head.add(bu) + else: + row["is_benunit_head"] = bu not in seen_bu_head + if bu not in seen_bu_head: + seen_bu_head.add(bu) + if pid in explicit_hh_head: + seen_hh_head.add(hh) + else: + row["is_household_head"] = hh not in seen_hh_head + if hh not in seen_hh_head: + seen_hh_head.add(hh) + + benunit_rows = [] + for bid, fields in benunits.items(): + members = fields.get("members") or [] + member_int_ids = [person_id_map[m] for m in members if m in person_id_map] + # Single household owns this benunit — pick from the first member. + if member_int_ids: + owner_household = next( + household_id_map[person_to_household[m]] + for m in members + if m in person_to_household + ) + else: + owner_household = 0 + row = dict(BENUNIT_DEFAULTS) + row["benunit_id"] = benunit_id_map[bid] + row["household_id"] = owner_household + row["person_ids"] = ";".join(str(i) for i in member_int_ids) + for var, val in (fields or {}).items(): + if var == "members": + continue + row[var] = _resolve_period_value(val, year) + benunit_rows.append(row) + + household_rows = [] + for hid, fields in households.items(): + members = fields.get("members") or [] + member_int_ids = [person_id_map[m] for m in members if m in person_id_map] + member_benunits = sorted({ + benunit_id_map[person_to_benunit[m]] + for m in members + if m in person_to_benunit + }) + row = dict(HOUSEHOLD_DEFAULTS) + row["household_id"] = household_id_map[hid] + row["person_ids"] = ";".join(str(i) for i in member_int_ids) + row["benunit_ids"] = ";".join(str(i) for i in member_benunits) + for var, val in (fields or {}).items(): + if var == "members": + continue + resolved = _resolve_period_value(val, year) + if var == "region" and isinstance(resolved, str): + resolved = _REGION_CANONICAL.get(resolved, resolved) + row[var] = resolved + household_rows.append(row) + + # Propagate `is_in_scotland` from each person's household region unless + # the situation already set it explicitly. + region_by_household = {h["household_id"]: h.get("region") for h in household_rows} + explicit_in_scotland = { + pid for pid, fields in people.items() + if "is_in_scotland" in (fields or {}) + } + for pid, row in zip(people.keys(), person_rows): + if pid in explicit_in_scotland: + continue + row["is_in_scotland"] = region_by_household.get(row["household_id"]) == "Scotland" + + return ( + pd.DataFrame(person_rows), + pd.DataFrame(benunit_rows), + pd.DataFrame(household_rows), + ) + + def _parse_microdata_stdout(raw: str) -> MicrodataResult: """Parse the concatenated CSV protocol output into a MicrodataResult.""" sections = {} @@ -612,6 +808,66 @@ def get_baseline_params(self, timeout: int = 10) -> dict: # ── Convenience constructors for hypothetical households ────────────── + @staticmethod + def from_situation( + situation: dict, + year: int = 2025, + **kwargs, + ) -> "Simulation": + """Build a Simulation from a PolicyEngine situation-JSON dict. + + The situation dict mirrors the PolicyEngine web-app format:: + + { + "people": {"": {"": {"": }, ...}, ...}, + "benunits": {"": {"members": [...], "": ..., ...}, ...}, + "households": {"": {"members": [...], "": ..., ...}, ...}, + } + + Each variable's value may be either a period-keyed dict (e.g. + ``{"2025": 50000}``) or a plain scalar — scalars are treated as + applying to ``year``. + + Variable names map directly to the wrapper input columns (see + ``PERSON_DEFAULTS``, ``BENUNIT_DEFAULTS``, ``HOUSEHOLD_DEFAULTS``). + ``region`` accepts either the title-case form (``"London"``, + ``"North East"``) or the upper-snake form used by the + PolicyEngine web app (``"LONDON"``, ``"NORTH_EAST"``); it is + normalised before being passed to the Rust engine and + ``is_in_scotland`` is set automatically. ``gender`` is + case-insensitive. + + Members lists on benunits/households reference the keys used in + ``situation["people"]``; people are assigned integer ``person_id`` + values in the order they appear under ``people``, and benunits/ + households receive the ``person_ids`` / ``benunit_ids`` strings + the engine expects. + + Example:: + + sim = Simulation.from_situation( + { + "people": { + "you": {"age": 30, "employment_income": {"2025": 50000}}, + }, + "benunits": {"yours": {"members": ["you"]}}, + "households": {"yours": {"members": ["you"], "region": "LONDON"}}, + }, + year=2025, + ) + result = sim.run() + """ + persons_df, benunits_df, households_df = _situation_to_dataframes( + situation, year + ) + return Simulation( + year=year, + persons=persons_df, + benunits=benunits_df, + households=households_df, + **kwargs, + ) + @staticmethod def single_person( age: float = 30, diff --git a/interfaces/python/tests/__init__.py b/interfaces/python/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/interfaces/python/tests/test_from_situation.py b/interfaces/python/tests/test_from_situation.py new file mode 100644 index 0000000..3e648ee --- /dev/null +++ b/interfaces/python/tests/test_from_situation.py @@ -0,0 +1,274 @@ +"""Tests for ``Simulation.from_situation``. + +These exercise the pure-Python conversion from a situation dict into the three +input DataFrames the wrapper passes to the Rust binary. They do not invoke the +binary itself, so they run quickly with no data dependencies. +""" + +from __future__ import annotations + +import pytest + +pd = pytest.importorskip("pandas") + +from policyengine_uk_compiled.engine import ( + _resolve_period_value, + _situation_to_dataframes, + Simulation, + PERSON_DEFAULTS, + BENUNIT_DEFAULTS, + HOUSEHOLD_DEFAULTS, +) + + +# ── _resolve_period_value ───────────────────────────────────────────────────── + +class TestResolvePeriodValue: + def test_scalar_passes_through(self): + assert _resolve_period_value(42, year=2025) == 42 + assert _resolve_period_value("LONDON", year=2025) == "LONDON" + assert _resolve_period_value(None, year=2025) is None + + def test_exact_year_match(self): + assert _resolve_period_value({"2024": 100, "2025": 200}, year=2025) == 200 + + def test_falls_back_to_most_recent_earlier_year(self): + # No 2025 entry — pick the latest period <= 2025. + assert _resolve_period_value({"2020": 1, "2023": 2}, year=2025) == 2 + + def test_falls_back_to_earliest_when_only_later_years_present(self): + assert _resolve_period_value({"2030": 9, "2040": 10}, year=2025) == 9 + + def test_handles_year_month_keys(self): + assert _resolve_period_value({"2025-04": 50}, year=2025) == 50 + + def test_handles_eternity_style_keys(self): + assert _resolve_period_value({"ETERNITY": "x"}, year=2025) == "x" + + +# ── _situation_to_dataframes ────────────────────────────────────────────────── + +class TestSituationToDataframes: + def test_minimal_single_person(self): + situation = { + "people": {"alice": {"age": 30, "employment_income": {"2025": 50_000}}}, + "benunits": {"bu_1": {"members": ["alice"]}}, + "households": {"hh_1": {"members": ["alice"], "region": "LONDON"}}, + } + persons, benunits, households = _situation_to_dataframes(situation, year=2025) + + assert len(persons) == 1 + assert persons.loc[0, "person_id"] == 0 + assert persons.loc[0, "benunit_id"] == 0 + assert persons.loc[0, "household_id"] == 0 + assert persons.loc[0, "age"] == 30 + assert persons.loc[0, "employment_income"] == 50_000 + assert bool(persons.loc[0, "is_benunit_head"]) is True + assert bool(persons.loc[0, "is_household_head"]) is True + assert bool(persons.loc[0, "is_in_scotland"]) is False + + assert len(benunits) == 1 + assert benunits.loc[0, "person_ids"] == "0" + assert benunits.loc[0, "household_id"] == 0 + + assert len(households) == 1 + assert households.loc[0, "person_ids"] == "0" + assert households.loc[0, "benunit_ids"] == "0" + assert households.loc[0, "region"] == "London" + + def test_matches_single_person_constructor(self): + """`from_situation` produces the same input frames as `single_person`.""" + sp_persons, sp_benunits, sp_households = Simulation.single_person( + age=40, employment_income=30_000, region="London" + ) + situation = { + "people": {"alice": {"age": 40, "employment_income": 30_000}}, + "benunits": {"bu": {"members": ["alice"]}}, + "households": {"hh": {"members": ["alice"], "region": "London"}}, + } + s_persons, s_benunits, s_households = _situation_to_dataframes( + situation, year=2025 + ) + # Compare on the columns single_person is expected to set / leave as defaults. + for col in PERSON_DEFAULTS: + assert sp_persons.loc[0, col] == s_persons.loc[0, col], col + for col in BENUNIT_DEFAULTS: + assert sp_benunits.loc[0, col] == s_benunits.loc[0, col], col + for col in HOUSEHOLD_DEFAULTS: + assert sp_households.loc[0, col] == s_households.loc[0, col], col + + def test_couple_with_children(self): + situation = { + "people": { + "p1": {"age": 35, "employment_income": {"2025": 40_000}}, + "p2": {"age": 33, "employment_income": {"2025": 25_000}}, + "c1": {"age": 6}, + "c2": {"age": 3}, + }, + "benunits": {"bu": {"members": ["p1", "p2", "c1", "c2"]}}, + "households": {"hh": {"members": ["p1", "p2", "c1", "c2"], "region": "South East"}}, + } + persons, benunits, households = _situation_to_dataframes(situation, year=2025) + + assert len(persons) == 4 + assert list(persons["person_id"]) == [0, 1, 2, 3] + # p1 is the implicit head of both benunit and household. + assert bool(persons.loc[0, "is_benunit_head"]) is True + assert bool(persons.loc[0, "is_household_head"]) is True + assert bool(persons.loc[1, "is_benunit_head"]) is False + assert bool(persons.loc[1, "is_household_head"]) is False + assert benunits.loc[0, "person_ids"] == "0;1;2;3" + assert households.loc[0, "person_ids"] == "0;1;2;3" + assert households.loc[0, "benunit_ids"] == "0" + assert households.loc[0, "region"] == "South East" + + def test_region_normalisation_upper_snake(self): + situation = { + "people": {"p": {"age": 30}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "NORTH_EAST"}}, + } + _, _, households = _situation_to_dataframes(situation, year=2025) + assert households.loc[0, "region"] == "North East" + + def test_region_normalisation_title_case_passthrough(self): + situation = { + "people": {"p": {"age": 30}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "South West"}}, + } + _, _, households = _situation_to_dataframes(situation, year=2025) + assert households.loc[0, "region"] == "South West" + + def test_scotland_sets_is_in_scotland(self): + situation = { + "people": {"p": {"age": 40}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "SCOTLAND"}}, + } + persons, _, households = _situation_to_dataframes(situation, year=2025) + assert households.loc[0, "region"] == "Scotland" + assert bool(persons.loc[0, "is_in_scotland"]) is True + + def test_explicit_is_in_scotland_overrides_region(self): + situation = { + "people": {"p": {"age": 40, "is_in_scotland": True}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "London"}}, + } + persons, _, _ = _situation_to_dataframes(situation, year=2025) + # Explicit value wins even though the region is London. + assert bool(persons.loc[0, "is_in_scotland"]) is True + + def test_gender_lowercased(self): + situation = { + "people": {"p": {"age": 30, "gender": "FEMALE"}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"]}}, + } + persons, _, _ = _situation_to_dataframes(situation, year=2025) + assert persons.loc[0, "gender"] == "female" + + def test_period_keyed_value_picks_year(self): + situation = { + "people": { + "p": { + "age": 30, + "employment_income": {"2024": 1000, "2025": 2000, "2026": 3000}, + } + }, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"]}}, + } + persons, _, _ = _situation_to_dataframes(situation, year=2025) + assert persons.loc[0, "employment_income"] == 2000 + + def test_implicit_benunit_when_omitted(self): + situation = { + "people": {"p": {"age": 30}}, + # benunits omitted entirely + "households": {"h": {"members": ["p"]}}, + } + persons, benunits, households = _situation_to_dataframes(situation, year=2025) + assert len(benunits) == 1 + assert benunits.loc[0, "person_ids"] == "0" + assert persons.loc[0, "benunit_id"] == 0 + + def test_multiple_benunits_in_one_household(self): + situation = { + "people": { + "lodger": {"age": 28, "employment_income": {"2025": 20_000}}, + "owner": {"age": 45, "employment_income": {"2025": 60_000}}, + }, + "benunits": { + "bu_lodger": {"members": ["lodger"]}, + "bu_owner": {"members": ["owner"]}, + }, + "households": { + "hh": {"members": ["lodger", "owner"], "region": "London"}, + }, + } + persons, benunits, households = _situation_to_dataframes(situation, year=2025) + assert len(benunits) == 2 + # Each benunit head is the first (and only) member of its benunit. + assert bool(persons.loc[0, "is_benunit_head"]) is True + assert bool(persons.loc[1, "is_benunit_head"]) is True + # But only the first person in the household is the household head. + assert bool(persons.loc[0, "is_household_head"]) is True + assert bool(persons.loc[1, "is_household_head"]) is False + assert households.loc[0, "person_ids"] == "0;1" + assert households.loc[0, "benunit_ids"] == "0;1" + + def test_missing_person_membership_raises(self): + situation = { + "people": {"orphan": {"age": 30}}, + "benunits": {"b": {"members": []}}, + "households": {"h": {"members": []}}, + } + with pytest.raises(ValueError, match="not a member of any benunit"): + _situation_to_dataframes(situation, year=2025) + + def test_no_people_raises(self): + with pytest.raises(ValueError, match="people"): + _situation_to_dataframes( + {"people": {}, "benunits": {}, "households": {"h": {"members": []}}}, + year=2025, + ) + + def test_no_households_raises(self): + with pytest.raises(ValueError, match="households"): + _situation_to_dataframes( + {"people": {"p": {"age": 30}}, "benunits": {}, "households": {}}, + year=2025, + ) + + +# ── Simulation.from_situation ──────────────────────────────────────────────── + +class TestFromSituationClassmethod: + def test_returns_simulation_with_year(self): + sim = Simulation.from_situation( + { + "people": {"p": {"age": 30}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "London"}}, + }, + year=2024, + ) + assert isinstance(sim, Simulation) + assert sim.year == 2024 + + def test_passes_through_dataframe_to_constructor(self): + sim = Simulation.from_situation( + { + "people": {"p": {"age": 30, "employment_income": 25_000}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": "LONDON"}}, + }, + ) + # Constructor stored the DataFrames so structural pre-hooks can see them. + assert sim._persons_df is not None + assert sim._benunits_df is not None + assert sim._households_df is not None + assert sim._stdin_payload is not None + assert "===PERSONS===" in sim._stdin_payload From 9a75e02b6d15b38c431ff51252c98a8da6584d99 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Apr 2026 14:41:33 +0100 Subject: [PATCH 2/9] =?UTF-8?q?feat:=20add=20Python=20=E2=86=94=20Rust=20p?= =?UTF-8?q?arity=20harness=20(#48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `scripts/parity.py`, which runs a fixed set of synthetic households through both the Python `policyengine-uk` package and the Rust `policyengine_uk_compiled` wrapper, diffs key tax / benefit / net-income outputs cell-for-cell, and prints a summary. Skips Python comparison gracefully when the Python package isn't installed. Wired into CI as a non-failing smoke step so it surfaces drift on every PR without breaking on the divergences that already exist (currently up to £3,276 on couple-with-children scenarios). Tolerance can be tightened once those gaps close. Stacked on top of #52 (Simulation.from_situation). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 3 + changelog.d/added/parity-harness.md | 1 + .../python/tests/test_parity_harness.py | 163 ++++++++++ scripts/parity.py | 298 ++++++++++++++++++ 4 files changed, 465 insertions(+) create mode 100644 changelog.d/added/parity-harness.md create mode 100644 interfaces/python/tests/test_parity_harness.py create mode 100644 scripts/parity.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f55424..129ced8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,3 +40,6 @@ jobs: - name: Test Python wrapper run: PYTHONPATH=interfaces/python pytest interfaces/python/tests -q + + - name: Parity smoke (Rust ↔ Python; non-failing) + run: PYTHONPATH=interfaces/python python scripts/parity.py --no-fail diff --git a/changelog.d/added/parity-harness.md b/changelog.d/added/parity-harness.md new file mode 100644 index 0000000..8e73501 --- /dev/null +++ b/changelog.d/added/parity-harness.md @@ -0,0 +1 @@ +Add `scripts/parity.py`, a Python ↔ Rust parity harness that runs a fixed set of synthetic households (single, couple, lone parent, pensioner, Scotland) through both the Python `policyengine-uk` package and the Rust `policyengine_uk_compiled` wrapper, diffs key tax / benefit / net-income outputs, and prints a summary. Surfaces drift introduced by Rust ports of Python variables; skips Python comparison gracefully when `policyengine-uk` isn't installed. diff --git a/interfaces/python/tests/test_parity_harness.py b/interfaces/python/tests/test_parity_harness.py new file mode 100644 index 0000000..f8fe51c --- /dev/null +++ b/interfaces/python/tests/test_parity_harness.py @@ -0,0 +1,163 @@ +"""Tests for the parity harness in `scripts/parity.py`. + +These cover the pure-Python pieces (period substitution, scenario builder, +diff computation) without invoking either engine, plus an end-to-end smoke +test that runs the harness through the Rust binary if it is available. +""" + +from __future__ import annotations + +import math +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPT = _REPO / "scripts" / "parity.py" + +# Make `import parity` work — the script is a top-level module, not a package. +sys.path.insert(0, str(_REPO / "scripts")) +parity = pytest.importorskip("parity") + + +# ── _replace_period ────────────────────────────────────────────────────────── + +class TestReplacePeriod: + def test_replaces_year_keys(self): + out = parity._replace_period({"YEAR": 50_000}, year=2025) + assert out == {"2025": 50_000} + + def test_recurses_into_nested_dicts(self): + out = parity._replace_period( + {"people": {"p": {"age": {"YEAR": 30}}}}, year=2024 + ) + assert out == {"people": {"p": {"age": {"2024": 30}}}} + + def test_recurses_into_lists(self): + out = parity._replace_period({"a": [{"YEAR": 1}, {"YEAR": 2}]}, year=2025) + assert out == {"a": [{"2025": 1}, {"2025": 2}]} + + def test_leaves_other_keys_untouched(self): + out = parity._replace_period({"members": ["p1"], "YEAR": 1}, year=2025) + assert out == {"members": ["p1"], "2025": 1} + + +# ── Scenario list ──────────────────────────────────────────────────────────── + +class TestScenarios: + def test_returns_non_empty_list(self): + scenarios = parity._scenarios(2025) + assert len(scenarios) > 0 + + def test_each_scenario_has_required_entities(self): + for name, situation in parity._scenarios(2025): + assert "people" in situation, name + assert "benunits" in situation, name + assert "households" in situation, name + assert situation["people"], f"{name}: empty people" + + def test_year_substituted_into_periods(self): + scenarios = parity._scenarios(2024) + # Pick one that has period-keyed values and check they're 2024. + for name, situation in scenarios: + for person_fields in situation["people"].values(): + for var, val in person_fields.items(): + if isinstance(val, dict): + assert "YEAR" not in val, f"{name}/{var}: YEAR not substituted" + assert all(k == "2024" for k in val.keys()), f"{name}/{var}: wrong year" + + def test_scenarios_cover_diverse_household_types(self): + names = [n for n, _ in parity._scenarios(2025)] + joined = "|".join(names).lower() + assert "single" in joined + assert "couple" in joined + assert "lone_parent" in joined + assert "pensioner" in joined + assert "scotland" in joined + + +# ── ScenarioResult diff computation ────────────────────────────────────────── + +class TestScenarioResult: + def test_no_python_means_no_diffs(self): + sr = parity.ScenarioResult( + name="test", rust={"income_tax": 100.0}, python=None + ) + sr.compute_diffs() + assert sr.diffs == {} + assert sr.max_abs_diff == 0.0 + + def test_diffs_populated_when_python_present(self): + sr = parity.ScenarioResult( + name="test", + rust={"income_tax": 100.0, "child_benefit": 50.0}, + python={"income_tax": 95.0, "child_benefit": 50.0}, + ) + sr.compute_diffs() + assert sr.diffs["income_tax"] == 5.0 + assert sr.diffs["child_benefit"] == 0.0 + assert sr.max_abs_diff == 5.0 + + def test_max_abs_diff_handles_negative(self): + sr = parity.ScenarioResult( + name="test", + rust={"a": 100.0, "b": 50.0}, + python={"a": 110.0, "b": 50.0}, + ) + sr.compute_diffs() + assert sr.diffs["a"] == -10.0 + assert sr.max_abs_diff == 10.0 + + def test_nan_values_dont_pollute_max(self): + sr = parity.ScenarioResult( + name="test", + rust={"a": 100.0, "b": float("nan")}, + python={"a": 100.0, "b": float("nan")}, + ) + sr.compute_diffs() + # NaN diff is NaN, but compute_diffs's max-abs guard skips it. + assert sr.max_abs_diff == 0.0 + + +# ── End-to-end smoke (requires the Rust binary) ────────────────────────────── + +def _has_rust_binary() -> bool: + candidates = [ + _REPO / "target" / "release" / "policyengine-uk-rust", + _REPO / "target" / "debug" / "policyengine-uk-rust", + ] + return any(c.is_file() for c in candidates) + + +@pytest.mark.skipif(not _has_rust_binary(), reason="Rust binary not built") +class TestEndToEnd: + def test_run_rust_returns_expected_keys_for_one_scenario(self): + # Use the simplest synthetic scenario manually. + situation = { + "people": {"p": {"age": {"2025": 30}, "employment_income": {"2025": 50_000}}}, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "region": {"2025": "LONDON"}}}, + } + out = parity.run_rust(situation, year=2025) + assert "income_tax" in out + assert "household_net_income" in out + # £50k single in 2025: income tax should land in the £7k–8k range. + assert 7_000 < out["income_tax"] < 8_000 + + def test_parity_runs_to_completion_in_no_fail_mode(self): + # `parity()` returns 0 in --no-fail mode regardless of diffs. + rc = parity.parity(year=2025, tolerance=1.0, fail_on_diff=False) + assert rc == 0 + + def test_cli_invocation(self): + result = subprocess.run( + [sys.executable, str(_SCRIPT), "--no-fail"], + capture_output=True, + text=True, + cwd=str(_REPO), + timeout=120, + ) + assert result.returncode == 0 + assert "income_tax" in result.stdout diff --git a/scripts/parity.py b/scripts/parity.py new file mode 100644 index 0000000..1599183 --- /dev/null +++ b/scripts/parity.py @@ -0,0 +1,298 @@ +"""Python ↔ Rust parity harness for the PolicyEngine UK engine. + +Runs a fixed set of synthetic households through both the Python +``policyengine-uk`` package and the Rust ``policyengine_uk_compiled`` wrapper, +diffs key tax / benefit / net-income outputs cell-for-cell, and prints a +summary. Exits non-zero when any diff exceeds the configured tolerance. + +Designed to surface drift introduced by Rust ports of Python variables. Uses +synthetic households so it has no FRS data dependency. If +``policyengine-uk`` isn't installed the Python comparison is skipped and the +script still produces a Rust-only smoke check. + +Usage:: + + python scripts/parity.py + python scripts/parity.py --tolerance 5 + python scripts/parity.py --year 2024 --no-fail +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +# Allow running from a checkout without `pip install -e .` +_REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPO / "interfaces" / "python")) + +from policyengine_uk_compiled import Simulation as RustSimulation + + +# ── Variable definitions ────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class Variable: + """A variable to compare across the two engines. + + `python_name` is the Python policyengine-uk name passed to + ``sim.calculate``. `rust_table` and `rust_column` locate the value in the + Rust microdata output (sum over all rows in that table). + """ + python_name: str + rust_table: str # "persons" | "benunits" | "households" + rust_column: str # column in that table + + +VARIABLES: list[Variable] = [ + Variable("income_tax", "persons", "baseline_income_tax"), + Variable("ni_employee", "persons", "baseline_employee_ni"), + Variable("ni_employer", "persons", "baseline_employer_ni"), + Variable("universal_credit", "benunits", "baseline_universal_credit"), + Variable("child_benefit", "benunits", "baseline_child_benefit"), + Variable("state_pension", "benunits", "baseline_state_pension"), + Variable("pension_credit", "benunits", "baseline_pension_credit"), + Variable("housing_benefit", "benunits", "baseline_housing_benefit"), + Variable("household_net_income", "households", "baseline_net_income"), +] + + +# ── Synthetic households ───────────────────────────────────────────────────── + +def _person(age: int, **kwargs) -> dict: + """Helper: build a person record with year-keyed period values.""" + p = {"age": {"YEAR": age}} + for key, val in kwargs.items(): + p[key] = {"YEAR": val} + return p + + +def _scenarios(year: int) -> list[tuple[str, dict]]: + """Return (name, situation_dict) pairs. Period 'YEAR' is rewritten below.""" + s: list[tuple[str, dict]] = [] + + # Single person at a range of incomes — exercises personal allowance taper, + # higher- and additional-rate bands, NI primary threshold and UEL. + for income in (0, 12_000, 25_000, 50_000, 80_000, 150_000): + s.append(( + f"single_£{income:,}".replace(",", "k"), + { + "people": {"you": _person(35, employment_income=income)}, + "benunits": {"b": {"members": ["you"]}}, + "households": {"h": {"members": ["you"], "region": {"YEAR": "LONDON"}}}, + }, + )) + + # Couple, no children — both earn, second-earner interaction + s.append(( + "couple_no_kids_40k_25k", + { + "people": { + "p1": _person(35, employment_income=40_000), + "p2": _person(33, employment_income=25_000), + }, + "benunits": {"b": {"members": ["p1", "p2"]}}, + "households": {"h": {"members": ["p1", "p2"], "region": {"YEAR": "LONDON"}}}, + }, + )) + + # Couple with two children — should activate Child Benefit + s.append(( + "couple_2kids_30k_15k", + { + "people": { + "p1": _person(38, employment_income=30_000), + "p2": _person(36, employment_income=15_000), + "c1": _person(8), + "c2": _person(4), + }, + "benunits": {"b": {"members": ["p1", "p2", "c1", "c2"]}}, + "households": {"h": {"members": ["p1", "p2", "c1", "c2"], "region": {"YEAR": "LONDON"}}}, + }, + )) + + # Lone parent, low income — should activate UC + CB + s.append(( + "lone_parent_2kids_18k", + { + "people": { + "p1": _person(32, employment_income=18_000), + "c1": _person(7), + "c2": _person(3), + }, + "benunits": {"b": {"members": ["p1", "c1", "c2"], "is_lone_parent": {"YEAR": True}}}, + "households": {"h": {"members": ["p1", "c1", "c2"], "region": {"YEAR": "NORTH_EAST"}}}, + }, + )) + + # Pensioner couple + s.append(( + "pensioner_couple", + { + "people": { + "p1": _person(70, state_pension=11_500), + "p2": _person(68, state_pension=11_500), + }, + "benunits": {"b": {"members": ["p1", "p2"]}}, + "households": {"h": {"members": ["p1", "p2"], "region": {"YEAR": "WALES"}}}, + }, + )) + + # Scotland resident — exercises devolved income-tax bands + s.append(( + "scotland_single_45k", + { + "people": {"you": _person(40, employment_income=45_000)}, + "benunits": {"b": {"members": ["you"]}}, + "households": {"h": {"members": ["you"], "region": {"YEAR": "SCOTLAND"}}}, + }, + )) + + # Substitute the placeholder period key with the real year string. + return [(name, _replace_period(sit, year)) for name, sit in s] + + +def _replace_period(obj, year: int): + """Recursively replace 'YEAR' keys with the real year string.""" + if isinstance(obj, dict): + return {(str(year) if k == "YEAR" else k): _replace_period(v, year) for k, v in obj.items()} + if isinstance(obj, list): + return [_replace_period(x, year) for x in obj] + return obj + + +# ── Engine drivers ──────────────────────────────────────────────────────────── + +def run_rust(situation: dict, year: int) -> dict[str, float]: + """Run the Rust engine and extract per-variable totals.""" + sim = RustSimulation.from_situation(situation, year=year) + micro = sim.run_microdata() + tables = {"persons": micro.persons, "benunits": micro.benunits, "households": micro.households} + out: dict[str, float] = {} + for v in VARIABLES: + df = tables[v.rust_table] + if v.rust_column in df.columns: + out[v.python_name] = float(df[v.rust_column].sum()) + else: + out[v.python_name] = float("nan") + return out + + +def run_python(situation: dict, year: int): + """Run the Python policyengine-uk engine, or return None if unavailable.""" + try: + from policyengine_uk import Simulation as PySimulation + except Exception: + return None + py_sim = PySimulation(situation=situation) + out: dict[str, float] = {} + for v in VARIABLES: + try: + out[v.python_name] = float(py_sim.calculate(v.python_name, year).sum()) + except Exception as e: + out[v.python_name] = float("nan") + return out + + +# ── Reporting ──────────────────────────────────────────────────────────────── + +@dataclass +class ScenarioResult: + name: str + rust: dict[str, float] + python: Optional[dict[str, float]] + diffs: dict[str, float] = field(default_factory=dict) + max_abs_diff: float = 0.0 + + def compute_diffs(self) -> None: + if self.python is None: + return + for var in self.rust: + r = self.rust.get(var, float("nan")) + p = self.python.get(var, float("nan")) + d = r - p + self.diffs[var] = d + if d == d and abs(d) > self.max_abs_diff: # NaN check via self-equality + self.max_abs_diff = abs(d) + + +def _fmt_money(x: float) -> str: + if x != x: # NaN + return " n/a" + return f"{x:>10,.0f}" + + +def print_report(results: list[ScenarioResult], comparing: bool) -> None: + headers = ["scenario"] + [v.python_name for v in VARIABLES] + if comparing: + print("\n=== Rust vs Python parity report ===\n") + for r in results: + print(f"-- {r.name} --") + for var in [v.python_name for v in VARIABLES]: + rv = r.rust.get(var, float("nan")) + pv = r.python.get(var, float("nan")) # type: ignore[union-attr] + diff = r.diffs.get(var, 0.0) + marker = " " if abs(diff) < 0.5 else " *" + print(f" {var:<24} rust={_fmt_money(rv)} py={_fmt_money(pv)} diff={_fmt_money(diff)}{marker}") + print(f" → max |diff|: {r.max_abs_diff:,.2f}") + print() + else: + print("\n=== Rust-only smoke output (policyengine-uk not installed) ===\n") + for r in results: + print(f"-- {r.name} --") + for var, val in r.rust.items(): + print(f" {var:<24} {_fmt_money(val)}") + print() + + +# ── Entry point ────────────────────────────────────────────────────────────── + +def parity(year: int = 2025, tolerance: float = 1.0, fail_on_diff: bool = True) -> int: + """Run the parity harness; return 0 on success, 1 on diff exceeded.""" + scenarios = _scenarios(year) + results: list[ScenarioResult] = [] + for name, situation in scenarios: + rust_out = run_rust(situation, year) + py_out = run_python(situation, year) + sr = ScenarioResult(name=name, rust=rust_out, python=py_out) + sr.compute_diffs() + results.append(sr) + + comparing = any(r.python is not None for r in results) + print_report(results, comparing) + + if not comparing: + print("Note: install `policyengine-uk` to enable parity comparison.") + return 0 + + over_tolerance = [r for r in results if r.max_abs_diff > tolerance] + if over_tolerance: + print(f"\n{len(over_tolerance)} scenarios exceeded tolerance £{tolerance:,.2f}:") + for r in over_tolerance: + print(f" - {r.name}: max |diff| = £{r.max_abs_diff:,.2f}") + return 1 if fail_on_diff else 0 + + print(f"\nAll {len(results)} scenarios within tolerance £{tolerance:,.2f}.") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--year", type=int, default=2025, help="Fiscal year (default 2025)") + parser.add_argument( + "--tolerance", type=float, default=1.0, + help="Per-scenario max-abs-diff in pounds (default 1.0)", + ) + parser.add_argument( + "--no-fail", action="store_true", + help="Always exit 0 even when diffs exceed tolerance", + ) + args = parser.parse_args() + return parity(year=args.year, tolerance=args.tolerance, fail_on_diff=not args.no_fail) + + +if __name__ == "__main__": + sys.exit(main()) From 4b57f833819aece2994c8ad80e0605007d1cf50b Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Apr 2026 14:51:08 +0100 Subject: [PATCH 3/9] feat: add YAML policy-test harness (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `policyengine_uk_compiled.yaml_tests` — a runner that mirrors the format used by `policyengine_uk/tests/policy/` so cases can be ported one at a time. The runner accepts either single-person flat input (`input: { employment_income: 50000 }`) or full-situation input (`input: { people: ..., benunits: ..., households: ... }`), supports absolute and relative error margins, and writes outputs against the Rust microdata column names (`baseline_income_tax`, `baseline_universal_credit`, `baseline_net_income`, etc.). This PR ships: - The runner module with CLI: `python -m policyengine_uk_compiled.yaml_tests tests/policy` - 11 hand-written YAML cases under `tests/policy/` covering income tax, employee NI, and Child Benefit (single + multi-person) - A pytest module that auto-discovers and parametrizes the YAML cases - 21 unit tests for the runner itself (input mapping, tolerance, parsing) - pyyaml added to the package's runtime dependencies Stacked on #53 (parity harness) which is itself stacked on #52 (Simulation.from_situation). Future PRs port more of the 196 Python YAML tests that already exist in `policyengine_uk/tests/policy/`. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 2 +- changelog.d/added/yaml-test-harness.md | 1 + .../policyengine_uk_compiled/yaml_tests.py | 252 ++++++++++++++++++ .../python/tests/test_yaml_policy_cases.py | 45 ++++ .../python/tests/test_yaml_test_runner.py | 142 ++++++++++ pyproject.toml | 1 + tests/policy/child_benefit.yaml | 31 +++ tests/policy/income_tax.yaml | 42 +++ tests/policy/national_insurance.yaml | 33 +++ 9 files changed, 548 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added/yaml-test-harness.md create mode 100644 interfaces/python/policyengine_uk_compiled/yaml_tests.py create mode 100644 interfaces/python/tests/test_yaml_policy_cases.py create mode 100644 interfaces/python/tests/test_yaml_test_runner.py create mode 100644 tests/policy/child_benefit.yaml create mode 100644 tests/policy/income_tax.yaml create mode 100644 tests/policy/national_insurance.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 129ced8..26dcfb5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: python-version: "3.11" - name: Install Python wrapper test dependencies - run: pip install pandas pydantic pytest + run: pip install pandas pydantic pytest pyyaml - name: Test Python wrapper run: PYTHONPATH=interfaces/python pytest interfaces/python/tests -q diff --git a/changelog.d/added/yaml-test-harness.md b/changelog.d/added/yaml-test-harness.md new file mode 100644 index 0000000..009919e --- /dev/null +++ b/changelog.d/added/yaml-test-harness.md @@ -0,0 +1 @@ +Add a YAML policy-test harness mirroring the format used by `policyengine_uk/tests/policy/`. Adds `policyengine_uk_compiled.yaml_tests` (a runner with `name`/`period`/`input`/`output`/`absolute_error_margin`/`relative_error_margin` fields, supporting both single-person flat input and full-situation input), an initial set of YAML cases under `tests/policy/` covering income tax, employee NI, and Child Benefit, and a pytest module that auto-discovers and runs them. Also runnable directly: `python -m policyengine_uk_compiled.yaml_tests tests/policy`. diff --git a/interfaces/python/policyengine_uk_compiled/yaml_tests.py b/interfaces/python/policyengine_uk_compiled/yaml_tests.py new file mode 100644 index 0000000..cbbac24 --- /dev/null +++ b/interfaces/python/policyengine_uk_compiled/yaml_tests.py @@ -0,0 +1,252 @@ +"""YAML-based policy test runner for the Rust engine. + +Mirrors the format used by `policyengine_uk/tests/policy/`, so YAML test +cases can be ported across one at a time. + +A YAML file contains a list of cases:: + + - name: A descriptive name + period: 2025 # year + absolute_error_margin: 1 # optional, default 1.0 + relative_error_margin: 0.01 # optional + input: + employment_income: 50000 # flat single-person shorthand + output: + baseline_income_tax: 7486 + +The ``input`` section can be either: + +1. **Flat** (single-person shorthand) — variable names map to the wrapper's + input columns and a one-person household is auto-built. Adding a + ``region`` key sets the household's region. + +2. **Full situation** — a dict with ``people``, ``benunits``, ``households``, + matching :func:`Simulation.from_situation`. + +Output names match the columns Rust microdata exposes (e.g. +``baseline_income_tax`` on persons, ``baseline_universal_credit`` on +benunits, ``baseline_net_income`` on households). Each output is summed +over all rows of the table it lives in, so single-person scenarios just +get the per-person value. +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from policyengine_uk_compiled.engine import ( + Simulation, + _situation_to_dataframes, +) + + +# Map output variable names → which microdata table they live in. +# Order matters: persons checked first, then benunits, then households. +_OUTPUT_TABLES = ("persons", "benunits", "households") + + +@dataclass +class YamlTestCase: + name: str + period: int + input: dict + output: dict + absolute_error_margin: float = 1.0 + relative_error_margin: Optional[float] = None + file: Optional[str] = None + + @classmethod + def from_dict(cls, raw: dict, file: Optional[str] = None) -> "YamlTestCase": + if "name" not in raw: + raise ValueError(f"YAML test case missing 'name' (file={file})") + if "period" not in raw: + raise ValueError(f"{raw.get('name')!r} missing 'period'") + if "input" not in raw or "output" not in raw: + raise ValueError(f"{raw.get('name')!r} needs both 'input' and 'output'") + return cls( + name=raw["name"], + period=int(raw["period"]), + input=dict(raw["input"]), + output=dict(raw["output"]), + absolute_error_margin=float(raw.get("absolute_error_margin", 1.0)), + relative_error_margin=( + float(raw["relative_error_margin"]) + if "relative_error_margin" in raw else None + ), + file=file, + ) + + +@dataclass +class YamlTestResult: + case: YamlTestCase + passed: bool + actual: dict + failures: list[str] + + +def _is_full_situation(input_dict: dict) -> bool: + """A situation has any of the three entity-keyed top-level keys.""" + return any(k in input_dict for k in ("people", "benunits", "households")) + + +def _flat_input_to_situation(flat: dict) -> dict: + """Wrap a flat single-person input dict in a full situation. + + Treats every key as a person-level field except ``region`` (household-level) + and a few benunit-only flags (``is_lone_parent``, ``rent_monthly``, + ``would_claim_*``, ``on_uc``, ``on_legacy``). + """ + benunit_keys = { + "is_lone_parent", "rent_monthly", "on_uc", "on_legacy", + "would_claim_uc", "would_claim_cb", "would_claim_hb", + "would_claim_pc", "would_claim_ctc", "would_claim_wtc", + "would_claim_is", "would_claim_esa", "would_claim_jsa", + } + household_keys = {"region", "rent_annual", "council_tax_annual", "weight"} + + person_fields: dict = {} + benunit_fields: dict = {} + household_fields: dict = {"members": ["you"]} + for key, val in flat.items(): + if key in household_keys: + household_fields[key] = val + elif key in benunit_keys: + benunit_fields[key] = val + else: + person_fields[key] = val + + benunit_fields["members"] = ["you"] + + return { + "people": {"you": person_fields}, + "benunits": {"b": benunit_fields}, + "households": {"h": household_fields}, + } + + +def _run_case(case: YamlTestCase) -> YamlTestResult: + """Execute one YAML test case against the Rust engine.""" + if _is_full_situation(case.input): + situation = case.input + else: + situation = _flat_input_to_situation(case.input) + + # Pre-build the DataFrames (validates the situation early). + persons, benunits, households = _situation_to_dataframes(situation, case.period) + sim = Simulation(year=case.period, persons=persons, benunits=benunits, households=households) + micro = sim.run_microdata() + tables = {"persons": micro.persons, "benunits": micro.benunits, "households": micro.households} + + actual: dict[str, float] = {} + failures: list[str] = [] + for var, expected in case.output.items(): + # Find which table holds the column. + for table_name in _OUTPUT_TABLES: + df = tables[table_name] + if var in df.columns: + got = float(df[var].sum()) + actual[var] = got + break + else: + failures.append( + f"{var!r}: column not found in any output table " + f"(persons/benunits/households)" + ) + continue + + if not _within_tolerance(got, expected, case): + failures.append( + f"{var!r}: expected {expected}, got {got:.4f} " + f"(diff={got - float(expected):+.4f}, " + f"tol=abs:{case.absolute_error_margin}" + + (f", rel:{case.relative_error_margin}" if case.relative_error_margin else "") + + ")" + ) + + return YamlTestResult(case=case, passed=not failures, actual=actual, failures=failures) + + +def _within_tolerance(got: float, expected: Any, case: YamlTestCase) -> bool: + # Booleans are int subclasses in Python; compare exactly so a 1-£ tolerance + # doesn't make False ≈ True. + if isinstance(expected, bool) or isinstance(got, bool): + return bool(got) == bool(expected) + try: + e = float(expected) + except (TypeError, ValueError): + return got == expected # non-numeric string — exact match + abs_ok = abs(got - e) <= case.absolute_error_margin + if case.relative_error_margin is None or e == 0: + return abs_ok + rel_ok = abs(got - e) / abs(e) <= case.relative_error_margin + return abs_ok or rel_ok + + +def load_yaml_file(path: Path) -> list[YamlTestCase]: + """Load all cases from a single YAML file.""" + import yaml + with open(path) as f: + raw = yaml.safe_load(f) or [] + if not isinstance(raw, list): + raise ValueError(f"{path}: top level must be a list of test cases") + return [YamlTestCase.from_dict(c, file=str(path)) for c in raw] + + +def discover_cases(root: Path) -> list[YamlTestCase]: + """Recursively load every YAML test case under ``root``.""" + cases: list[YamlTestCase] = [] + for path in sorted(root.rglob("*.yaml")): + cases.extend(load_yaml_file(path)) + return cases + + +def run_cases(cases: list[YamlTestCase]) -> list[YamlTestResult]: + """Run every case and return per-case results.""" + return [_run_case(c) for c in cases] + + +def _print_results(results: list[YamlTestResult]) -> None: + n_pass = sum(1 for r in results if r.passed) + n_fail = len(results) - n_pass + for r in results: + status = "PASS" if r.passed else "FAIL" + loc = f" [{Path(r.case.file).name}]" if r.case.file else "" + print(f" {status} {r.case.name}{loc}") + for f in r.failures: + print(f" ✗ {f}") + print(f"\n{n_pass} passed, {n_fail} failed of {len(results)}") + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Run YAML policy tests against the Rust engine." + ) + parser.add_argument( + "path", nargs="?", default="tests/policy", + help="YAML file or directory of YAML files (default: tests/policy)", + ) + args = parser.parse_args(argv) + + target = Path(args.path) + if not target.exists(): + print(f"error: {target} does not exist", file=sys.stderr) + return 2 + cases = ( + load_yaml_file(target) if target.is_file() else discover_cases(target) + ) + if not cases: + print(f"warning: no YAML test cases found under {target}") + return 0 + + results = run_cases(cases) + _print_results(results) + return 0 if all(r.passed for r in results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/interfaces/python/tests/test_yaml_policy_cases.py b/interfaces/python/tests/test_yaml_policy_cases.py new file mode 100644 index 0000000..57fb25b --- /dev/null +++ b/interfaces/python/tests/test_yaml_policy_cases.py @@ -0,0 +1,45 @@ +"""Auto-discovers every YAML test case under ``tests/policy/`` and runs each +through the Rust engine as a parametrized pytest case. + +Skips cleanly when the Rust binary isn't available. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytest.importorskip("yaml") + +from policyengine_uk_compiled.yaml_tests import ( + YamlTestCase, + _run_case, + discover_cases, +) + +_REPO = Path(__file__).resolve().parents[3] +_POLICY_DIR = _REPO / "tests" / "policy" + + +def _has_rust_binary() -> bool: + candidates = [ + _REPO / "target" / "release" / "policyengine-uk-rust", + _REPO / "target" / "debug" / "policyengine-uk-rust", + ] + return any(c.is_file() for c in candidates) + + +CASES = discover_cases(_POLICY_DIR) if _POLICY_DIR.exists() else [] + + +@pytest.mark.skipif(not _has_rust_binary(), reason="Rust binary not built") +@pytest.mark.skipif(not CASES, reason="No YAML test cases discovered") +@pytest.mark.parametrize("case", CASES, ids=[c.name for c in CASES]) +def test_yaml_policy_case(case: YamlTestCase) -> None: + result = _run_case(case) + if not result.passed: + msg = f"\n{case.name} ({Path(case.file).name if case.file else ''})" + for f in result.failures: + msg += f"\n ✗ {f}" + pytest.fail(msg) diff --git a/interfaces/python/tests/test_yaml_test_runner.py b/interfaces/python/tests/test_yaml_test_runner.py new file mode 100644 index 0000000..1efb8f8 --- /dev/null +++ b/interfaces/python/tests/test_yaml_test_runner.py @@ -0,0 +1,142 @@ +"""Unit tests for the YAML test runner itself (pure Python, no engine). + +The end-to-end execution of YAML cases against the Rust binary is handled by +``test_yaml_policy_cases.py``. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +pytest.importorskip("yaml") + +from policyengine_uk_compiled.yaml_tests import ( + YamlTestCase, + _flat_input_to_situation, + _is_full_situation, + _within_tolerance, + load_yaml_file, +) + + +class TestIsFullSituation: + def test_flat_input(self): + assert _is_full_situation({"employment_income": 50_000}) is False + + def test_situation_with_people(self): + assert _is_full_situation({"people": {}}) is True + + def test_situation_with_only_households(self): + assert _is_full_situation({"households": {}}) is True + + +class TestFlatInputToSituation: + def test_person_field_lands_on_person(self): + out = _flat_input_to_situation({"employment_income": 50_000, "age": 30}) + assert out["people"]["you"]["employment_income"] == 50_000 + assert out["people"]["you"]["age"] == 30 + + def test_region_lands_on_household(self): + out = _flat_input_to_situation({"region": "London"}) + assert out["households"]["h"]["region"] == "London" + assert "region" not in out["people"]["you"] + + def test_is_lone_parent_lands_on_benunit(self): + out = _flat_input_to_situation({"is_lone_parent": True}) + assert out["benunits"]["b"]["is_lone_parent"] is True + assert "is_lone_parent" not in out["people"]["you"] + + def test_rent_monthly_lands_on_benunit(self): + out = _flat_input_to_situation({"rent_monthly": 1200}) + assert out["benunits"]["b"]["rent_monthly"] == 1200 + + def test_council_tax_lands_on_household(self): + out = _flat_input_to_situation({"council_tax_annual": 1500}) + assert out["households"]["h"]["council_tax_annual"] == 1500 + + def test_members_always_set(self): + out = _flat_input_to_situation({"employment_income": 0}) + assert out["benunits"]["b"]["members"] == ["you"] + assert out["households"]["h"]["members"] == ["you"] + + +class TestWithinTolerance: + def _case(self, abs_margin: float = 1.0, rel_margin: float | None = None): + return YamlTestCase( + name="t", period=2025, input={}, output={}, + absolute_error_margin=abs_margin, relative_error_margin=rel_margin, + ) + + def test_exact_match_within_zero_margin(self): + assert _within_tolerance(100.0, 100, self._case(abs_margin=0)) is True + + def test_within_absolute_margin(self): + assert _within_tolerance(100.5, 100, self._case(abs_margin=1.0)) is True + + def test_outside_absolute_margin(self): + assert _within_tolerance(102.0, 100, self._case(abs_margin=1.0)) is False + + def test_relative_margin_kicks_in_for_large_values(self): + # 5 over 1000 is 0.5%, within rel=1% even though abs > 1 + assert _within_tolerance(1005.0, 1000, self._case(abs_margin=0, rel_margin=0.01)) is True + + def test_relative_margin_ignored_when_expected_zero(self): + assert _within_tolerance(2.0, 0, self._case(abs_margin=1.0, rel_margin=0.5)) is False + + def test_boolean_values_compared_exactly(self): + assert _within_tolerance(True, True, self._case()) is True + assert _within_tolerance(False, True, self._case()) is False + + +class TestLoadYamlFile: + def test_loads_multiple_cases(self, tmp_path: Path): + f = tmp_path / "x.yaml" + f.write_text(textwrap.dedent("""\ + - name: a + period: 2025 + input: {employment_income: 0} + output: {baseline_income_tax: 0} + - name: b + period: 2024 + absolute_error_margin: 5 + input: {employment_income: 50000} + output: {baseline_income_tax: 7486} + """)) + cases = load_yaml_file(f) + assert len(cases) == 2 + assert cases[0].name == "a" + assert cases[0].period == 2025 + assert cases[0].absolute_error_margin == 1.0 + assert cases[1].absolute_error_margin == 5.0 + + def test_empty_file_yields_empty_list(self, tmp_path: Path): + f = tmp_path / "empty.yaml" + f.write_text("") + assert load_yaml_file(f) == [] + + def test_missing_name_raises(self, tmp_path: Path): + f = tmp_path / "bad.yaml" + f.write_text("- period: 2025\n input: {}\n output: {}\n") + with pytest.raises(ValueError, match="missing 'name'"): + load_yaml_file(f) + + def test_missing_period_raises(self, tmp_path: Path): + f = tmp_path / "bad.yaml" + f.write_text("- name: x\n input: {}\n output: {}\n") + with pytest.raises(ValueError, match="period"): + load_yaml_file(f) + + def test_missing_input_or_output_raises(self, tmp_path: Path): + f = tmp_path / "bad.yaml" + f.write_text("- name: x\n period: 2025\n input: {}\n") + with pytest.raises(ValueError, match="input.*output"): + load_yaml_file(f) + + def test_top_level_must_be_a_list(self, tmp_path: Path): + f = tmp_path / "bad.yaml" + f.write_text("name: not-a-list\n") + with pytest.raises(ValueError, match="list"): + load_yaml_file(f) diff --git a/pyproject.toml b/pyproject.toml index b0afd5d..ea0d361 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "openpyxl>=3.1.5", "pandas>=2.3.3", "pydantic>=2.0", + "pyyaml>=6.0", "requests>=2.33.1", "rich>=14.3.3", ] diff --git a/tests/policy/child_benefit.yaml b/tests/policy/child_benefit.yaml new file mode 100644 index 0000000..94f455c --- /dev/null +++ b/tests/policy/child_benefit.yaml @@ -0,0 +1,31 @@ +# Child Benefit baseline cases (full-situation form). + +- name: Two children, modest income — full Child Benefit + period: 2025 + absolute_error_margin: 1 + input: + people: + p1: { age: 38, employment_income: 30_000 } + p2: { age: 36, employment_income: 15_000 } + c1: { age: 8 } + c2: { age: 4 } + benunits: + b: { members: [p1, p2, c1, c2] } + households: + h: { members: [p1, p2, c1, c2], region: London } + output: + baseline_child_benefit: 2252 + +- name: No children — no Child Benefit + period: 2025 + absolute_error_margin: 0 + input: + people: + p1: { age: 35, employment_income: 30_000 } + p2: { age: 33, employment_income: 15_000 } + benunits: + b: { members: [p1, p2] } + households: + h: { members: [p1, p2], region: London } + output: + baseline_child_benefit: 0 diff --git a/tests/policy/income_tax.yaml b/tests/policy/income_tax.yaml new file mode 100644 index 0000000..cc487ad --- /dev/null +++ b/tests/policy/income_tax.yaml @@ -0,0 +1,42 @@ +# Baseline income-tax cases for the Rust engine. +# Numbers verified against the engine on 2026-04-30 at year=2025. + +- name: Zero income — no tax + period: 2025 + absolute_error_margin: 0 + input: + employment_income: 0 + output: + baseline_income_tax: 0 + +- name: Below personal allowance — no tax + period: 2025 + absolute_error_margin: 0 + input: + employment_income: 12_000 + output: + baseline_income_tax: 0 + +- name: Basic-rate band only + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 25_000 + output: + baseline_income_tax: 2486 + +- name: Spans basic and higher rates + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 50_000 + output: + baseline_income_tax: 7486 + +- name: Personal allowance fully tapered + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 150_000 + output: + baseline_income_tax: 53703 diff --git a/tests/policy/national_insurance.yaml b/tests/policy/national_insurance.yaml new file mode 100644 index 0000000..bd073f8 --- /dev/null +++ b/tests/policy/national_insurance.yaml @@ -0,0 +1,33 @@ +# Employee NI baseline cases. + +- name: Below primary threshold — no employee NI + period: 2025 + absolute_error_margin: 0 + input: + employment_income: 12_000 + output: + baseline_employee_ni: 0 + +- name: Inside main band + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 25_000 + output: + baseline_employee_ni: 994 + +- name: Above UEL — partial higher-band tier + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 50_000 + output: + baseline_employee_ni: 2994 + +- name: Well above UEL + period: 2025 + absolute_error_margin: 1 + input: + employment_income: 80_000 + output: + baseline_employee_ni: 3611 From 7a9662c82e3984adce0a66fdaaf5376caabb1dc7 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 10:15:29 +0100 Subject: [PATCH 4/9] feat: add LBTT (Scotland) and LTT (Wales) (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Property-transaction tax now dispatches by region: - Scotland → LBTT (LBTT (Scotland) Act 2013) - Wales → LTT (LTT and Anti-avoidance of Devolved Taxes (Wales) Act 2017) - elsewhere → SDLT (Finance Act 2003 s.55, unchanged) 2025/26 residential bands per: - SSI 2015/126 (Scotland) - WSI 2018/128 (Wales) Adds: - `lbtt` and `ltt` parameter blocks in `parameters/2025_26.yaml` - `Parameters.lbtt`/`Parameters.ltt` Rust fields and Python wrapper exposure - `calculate_property_transaction_tax` dispatch function in `src/variables/wealth_taxes.rs` - New `baseline_property_transaction_tax` and `reform_property_transaction_tax` per-household microdata columns - Six Rust unit tests covering LBTT/LTT/SDLT dispatch and nil-band edges - Six YAML policy-test cases (`tests/policy/property_transaction_tax.yaml`) Stacked on #54 (YAML test harness). Co-Authored-By: Claude Opus 4.7 (1M context) --- changelog.d/added/lbtt-ltt.md | 1 + .../python/policyengine_uk_compiled/models.py | 2 + parameters/2025_26.yaml | 27 +++- src/data/clean.rs | 4 + src/engine/simulation.rs | 13 +- src/parameters/mod.rs | 10 +- src/variables/wealth_taxes.rs | 146 +++++++++++++++++- tests/policy/property_transaction_tax.yaml | 88 +++++++++++ 8 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added/lbtt-ltt.md create mode 100644 tests/policy/property_transaction_tax.yaml diff --git a/changelog.d/added/lbtt-ltt.md b/changelog.d/added/lbtt-ltt.md new file mode 100644 index 0000000..8d7685e --- /dev/null +++ b/changelog.d/added/lbtt-ltt.md @@ -0,0 +1 @@ +Add Scottish LBTT (Land and Buildings Transaction Tax) and Welsh LTT (Land Transaction Tax) as devolved replacements for SDLT. Property transactions now dispatch by region: Scotland → LBTT, Wales → LTT, England + NI → SDLT. New `lbtt` and `ltt` reform parameters in the Python wrapper, 2025/26 residential bands sourced from the Land and Buildings Transaction Tax (Tax Rates and Tax Bands) (Scotland) Order 2015 and the Land Transaction Tax (Tax Bands and Tax Rates) (Wales) Regulations 2018, and a new `baseline_property_transaction_tax` / `reform_property_transaction_tax` per-household microdata column. diff --git a/interfaces/python/policyengine_uk_compiled/models.py b/interfaces/python/policyengine_uk_compiled/models.py index 79e44ac..0882553 100644 --- a/interfaces/python/policyengine_uk_compiled/models.py +++ b/interfaces/python/policyengine_uk_compiled/models.py @@ -224,6 +224,8 @@ class Parameters(BaseModel): income_related_benefits: Optional[IncomeRelatedBenefitParams] = None capital_gains_tax: Optional[CapitalGainsTaxParams] = None stamp_duty: Optional[StampDutyParams] = None + lbtt: Optional[StampDutyParams] = None + ltt: Optional[StampDutyParams] = None wealth_tax: Optional[WealthTaxParams] = None labour_supply: Optional[LabourSupplyParams] = None diff --git a/parameters/2025_26.yaml b/parameters/2025_26.yaml index 48ec239..41df0d5 100644 --- a/parameters/2025_26.yaml +++ b/parameters/2025_26.yaml @@ -247,7 +247,7 @@ capital_gains_tax: realisation_rate: 0.50 stamp_duty: - # Finance Act 2003 s.55, as amended; bands from 1 April 2025 + # SDLT — England + NI. Finance Act 2003 s.55, as amended; bands from 1 April 2025 bands: - { rate: 0.0, threshold: 0 } - { rate: 0.02, threshold: 125001 } @@ -256,6 +256,31 @@ stamp_duty: - { rate: 0.12, threshold: 1500001 } annual_purchase_probability: 0.043 # ~1/23 year average holding period +lbtt: + # LBTT — Scotland's devolved replacement for SDLT. Land and Buildings Transaction + # Tax (Tax Rates and Tax Bands) (Scotland) Order 2015 (SSI 2015/126), Schedule. + # Residential primary-residence bands. + bands: + - { rate: 0.0, threshold: 0 } + - { rate: 0.02, threshold: 145001 } + - { rate: 0.05, threshold: 250001 } + - { rate: 0.10, threshold: 325001 } + - { rate: 0.12, threshold: 750001 } + annual_purchase_probability: 0.043 + +ltt: + # LTT — Wales's devolved replacement for SDLT. Land Transaction Tax (Tax Bands + # and Tax Rates) (Wales) Regulations 2018 (WSI 2018/128), Schedule. + # Residential primary-residence bands. + bands: + - { rate: 0.0, threshold: 0 } + - { rate: 0.035, threshold: 180001 } + - { rate: 0.05, threshold: 250001 } + - { rate: 0.075, threshold: 400001 } + - { rate: 0.10, threshold: 750001 } + - { rate: 0.12, threshold: 1500001 } + annual_purchase_probability: 0.043 + wealth_tax: # Hypothetical — no current UK legislation. Disabled by default. # Wealth Tax Commission (2020) proposed 1% above £10m. diff --git a/src/data/clean.rs b/src/data/clean.rs index 6872c7f..af1e383 100644 --- a/src/data/clean.rs +++ b/src/data/clean.rs @@ -544,10 +544,12 @@ fn write_microdata_csv_households( // ── Baseline outputs ── "baseline_net_income", "baseline_gross_income", "baseline_total_tax", "baseline_total_benefits", + "baseline_property_transaction_tax", "baseline_equivalisation_factor", "baseline_equivalised_net_income", // ── Reform outputs ── "reform_net_income", "reform_gross_income", "reform_total_tax", "reform_total_benefits", + "reform_property_transaction_tax", "reform_equivalisation_factor", "reform_equivalised_net_income", ])?; @@ -567,6 +569,7 @@ fn write_microdata_csv_households( format!("{:.2}", bl.gross_income), format!("{:.2}", bl.total_tax), format!("{:.2}", bl.total_benefits), + format!("{:.2}", bl.stamp_duty), format!("{:.4}", bl.equivalisation_factor), format!("{:.2}", bl.equivalised_net_income), // Reform @@ -574,6 +577,7 @@ fn write_microdata_csv_households( format!("{:.2}", rf.gross_income), format!("{:.2}", rf.total_tax), format!("{:.2}", rf.total_benefits), + format!("{:.2}", rf.stamp_duty), format!("{:.4}", rf.equivalisation_factor), format!("{:.2}", rf.equivalised_net_income), ])?; diff --git a/src/engine/simulation.rs b/src/engine/simulation.rs index e39ab2c..ffe8ec5 100644 --- a/src/engine/simulation.rs +++ b/src/engine/simulation.rs @@ -299,10 +299,15 @@ impl Simulation { .map(|&pid| person_results[pid].capital_gains_tax) .sum(); - // Stamp duty (annualised) - let stamp_duty = self.parameters.stamp_duty.as_ref() - .map(|p| variables::wealth_taxes::calculate_stamp_duty(hh, p)) - .unwrap_or(0.0); + // Property transaction tax (annualised): SDLT in England/NI, LBTT in + // Scotland, LTT in Wales. Stored on the household result as + // `stamp_duty` for backwards compatibility. + let stamp_duty = variables::wealth_taxes::calculate_property_transaction_tax( + hh, + self.parameters.stamp_duty.as_ref(), + self.parameters.lbtt.as_ref(), + self.parameters.ltt.as_ref(), + ); // Wealth tax let wealth_tax = self.parameters.wealth_tax.as_ref() diff --git a/src/parameters/mod.rs b/src/parameters/mod.rs index aae4867..ad69075 100644 --- a/src/parameters/mod.rs +++ b/src/parameters/mod.rs @@ -52,9 +52,17 @@ pub struct Parameters { /// Capital gains tax. TCGA 1992; 18%/24% from October 2024 Budget. #[serde(default)] pub capital_gains_tax: Option, - /// Stamp duty land tax on residential property. FA 2003 s.55. + /// Stamp duty land tax on residential property (England + NI). FA 2003 s.55. #[serde(default)] pub stamp_duty: Option, + /// Land and Buildings Transaction Tax — Scotland's devolved replacement for + /// SDLT. Land and Buildings Transaction Tax (Scotland) Act 2013, s.24. + #[serde(default)] + pub lbtt: Option, + /// Land Transaction Tax — Wales's devolved replacement for SDLT. + /// Land Transaction Tax and Anti-avoidance of Devolved Taxes (Wales) Act 2017. + #[serde(default)] + pub ltt: Option, /// Annual wealth tax (hypothetical — disabled by default). #[serde(default)] pub wealth_tax: Option, diff --git a/src/variables/wealth_taxes.rs b/src/variables/wealth_taxes.rs index 8203c3e..866c2d9 100644 --- a/src/variables/wealth_taxes.rs +++ b/src/variables/wealth_taxes.rs @@ -1,4 +1,4 @@ -use crate::engine::entities::{Household, Person}; +use crate::engine::entities::{Household, Person, Region}; use crate::parameters::{CouncilTaxParams, CapitalGainsTaxParams, StampDutyParams, WealthTaxParams}; /// Determine the council tax band (0=A .. 7=H) from a 1991 property value. @@ -82,6 +82,30 @@ pub fn calculate_stamp_duty(hh: &Household, params: &StampDutyParams) -> f64 { sdlt * params.annual_purchase_probability } +/// Calculate annualised property-transaction tax for a household, dispatching +/// to the regime that applies in the household's region. +/// +/// - Scotland → LBTT (Land and Buildings Transaction Tax (Scotland) Act 2013) +/// - Wales → LTT (Land Transaction Tax and Anti-avoidance of Devolved Taxes (Wales) Act 2017) +/// - elsewhere (England + NI) → SDLT (Finance Act 2003 s.55) +/// +/// Each parameter argument is optional; the function returns 0.0 when the +/// regime that would apply is unset (e.g. no LBTT params loaded for a Scottish +/// household), matching the existing behaviour for missing SDLT params. +pub fn calculate_property_transaction_tax( + hh: &Household, + sdlt: Option<&StampDutyParams>, + lbtt: Option<&StampDutyParams>, + ltt: Option<&StampDutyParams>, +) -> f64 { + let params = match hh.region { + Region::Scotland => lbtt, + Region::Wales => ltt, + _ => sdlt, + }; + params.map(|p| calculate_stamp_duty(hh, p)).unwrap_or(0.0) +} + /// Calculate annual wealth tax for a household. /// /// Hypothetical flat-rate tax on net wealth above a threshold. @@ -197,6 +221,126 @@ mod tests { assert!((sdlt - 15000.0).abs() < 1.0); } + fn make_sdlt() -> StampDutyParams { + StampDutyParams { + bands: vec![ + StampDutyBand { rate: 0.0, threshold: 0.0 }, + StampDutyBand { rate: 0.02, threshold: 125001.0 }, + StampDutyBand { rate: 0.05, threshold: 250001.0 }, + StampDutyBand { rate: 0.10, threshold: 925001.0 }, + StampDutyBand { rate: 0.12, threshold: 1500001.0 }, + ], + annual_purchase_probability: 1.0, + } + } + + fn make_lbtt() -> StampDutyParams { + // Scotland 2025/26 (residential). + StampDutyParams { + bands: vec![ + StampDutyBand { rate: 0.0, threshold: 0.0 }, + StampDutyBand { rate: 0.02, threshold: 145001.0 }, + StampDutyBand { rate: 0.05, threshold: 250001.0 }, + StampDutyBand { rate: 0.10, threshold: 325001.0 }, + StampDutyBand { rate: 0.12, threshold: 750001.0 }, + ], + annual_purchase_probability: 1.0, + } + } + + fn make_ltt() -> StampDutyParams { + // Wales 2025/26 (residential primary). + StampDutyParams { + bands: vec![ + StampDutyBand { rate: 0.0, threshold: 0.0 }, + StampDutyBand { rate: 0.035, threshold: 180001.0 }, + StampDutyBand { rate: 0.05, threshold: 250001.0 }, + StampDutyBand { rate: 0.075, threshold: 400001.0 }, + StampDutyBand { rate: 0.10, threshold: 750001.0 }, + StampDutyBand { rate: 0.12, threshold: 1500001.0 }, + ], + annual_purchase_probability: 1.0, + } + } + + #[test] + fn property_tax_routes_to_lbtt_in_scotland() { + let mut hh = Household::default(); + hh.main_residence_value = 500_000.0; + hh.region = Region::Scotland; + // LBTT on £500k: + // 0% on first £145k = £0 + // 2% on £145k-£250k (£105k) = £2,100 + // 5% on £250k-£325k (£75k) = £3,750 + // 10% on £325k-£500k (£175k) = £17,500 + // total = £23,350 + let tax = calculate_property_transaction_tax( + &hh, Some(&make_sdlt()), Some(&make_lbtt()), Some(&make_ltt()) + ); + assert!((tax - 23_350.0).abs() < 1.0, "got {}", tax); + } + + #[test] + fn property_tax_routes_to_ltt_in_wales() { + let mut hh = Household::default(); + hh.main_residence_value = 500_000.0; + hh.region = Region::Wales; + // LTT on £500k: + // 0% on first £180k = £0 + // 3.5% on £180k-£250k (£70k) = £2,450 + // 5% on £250k-£400k (£150k) = £7,500 + // 7.5% on £400k-£500k (£100k) = £7,500 + // total = £17,450 + let tax = calculate_property_transaction_tax( + &hh, Some(&make_sdlt()), Some(&make_lbtt()), Some(&make_ltt()) + ); + assert!((tax - 17_450.0).abs() < 1.0, "got {}", tax); + } + + #[test] + fn property_tax_routes_to_sdlt_outside_scotland_and_wales() { + let mut hh = Household::default(); + hh.main_residence_value = 500_000.0; + hh.region = Region::London; + // Same as the existing stamp_duty_marginal test: £15,000. + let tax = calculate_property_transaction_tax( + &hh, Some(&make_sdlt()), Some(&make_lbtt()), Some(&make_ltt()) + ); + assert!((tax - 15_000.0).abs() < 1.0, "got {}", tax); + } + + #[test] + fn property_tax_returns_zero_when_devolved_params_missing() { + let mut hh = Household::default(); + hh.main_residence_value = 500_000.0; + hh.region = Region::Scotland; + // No LBTT params loaded → tax is 0 (regime doesn't fall back to SDLT). + let tax = calculate_property_transaction_tax(&hh, Some(&make_sdlt()), None, None); + assert_eq!(tax, 0.0); + } + + #[test] + fn lbtt_zero_below_nil_band() { + let mut hh = Household::default(); + hh.main_residence_value = 100_000.0; // below £145k LBTT nil-band ceiling + hh.region = Region::Scotland; + let tax = calculate_property_transaction_tax( + &hh, Some(&make_sdlt()), Some(&make_lbtt()), Some(&make_ltt()) + ); + assert_eq!(tax, 0.0); + } + + #[test] + fn ltt_zero_below_nil_band() { + let mut hh = Household::default(); + hh.main_residence_value = 150_000.0; // below £180k LTT nil-band ceiling + hh.region = Region::Wales; + let tax = calculate_property_transaction_tax( + &hh, Some(&make_sdlt()), Some(&make_lbtt()), Some(&make_ltt()) + ); + assert_eq!(tax, 0.0); + } + #[test] fn wealth_tax_disabled() { let params = WealthTaxParams { enabled: false, threshold: 10_000_000.0, rate: 0.01 }; diff --git a/tests/policy/property_transaction_tax.yaml b/tests/policy/property_transaction_tax.yaml new file mode 100644 index 0000000..af00926 --- /dev/null +++ b/tests/policy/property_transaction_tax.yaml @@ -0,0 +1,88 @@ +# Property-transaction tax bands by region: +# - England + NI → SDLT (Finance Act 2003 s.55) +# - Scotland → LBTT (LBTT (Scotland) Act 2013) +# - Wales → LTT (Land Transaction Tax (Wales) Act 2017) +# +# All amounts below are annualised (×0.043 ≈ 1/23-yr average holding period). +# Numbers verified against the engine on 2026-05-01 at year=2025. + +- name: SDLT in London on a £500k property + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: London, main_residence_value: 500_000 } + output: + # SDLT one-off: 0 + 2%×125k + 5%×250k = £15,000; annualised = £645.00 + baseline_property_transaction_tax: 645 + +- name: LBTT in Scotland on a £500k property + period: 2025 + absolute_error_margin: 2 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: Scotland, main_residence_value: 500_000 } + output: + # LBTT one-off: 2%×105k + 5%×75k + 10%×175k = £23,350; annualised = £1004.05 + baseline_property_transaction_tax: 1004 + +- name: LTT in Wales on a £500k property + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: Wales, main_residence_value: 500_000 } + output: + # LTT one-off: 3.5%×70k + 5%×150k + 7.5%×100k = £17,450; annualised = £750.35 + baseline_property_transaction_tax: 750 + +- name: Below LBTT nil-band (£100k in Scotland) — no tax + period: 2025 + absolute_error_margin: 0 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: Scotland, main_residence_value: 100_000 } + output: + baseline_property_transaction_tax: 0 + +- name: Below LTT nil-band (£150k in Wales) — no tax + period: 2025 + absolute_error_margin: 0 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: Wales, main_residence_value: 150_000 } + output: + baseline_property_transaction_tax: 0 + +- name: No property — no tax (Scotland) + period: 2025 + absolute_error_margin: 0 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: Scotland } + output: + baseline_property_transaction_tax: 0 From 710db926dd75f736751b0c7dea97995fcdde57d6 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 11:02:07 +0100 Subject: [PATCH 5/9] feat: compute PIP daily-living and mobility amounts from flags (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now, PIP amount fields (`pip_daily_living`, `pip_mobility`) were only populated from FRS recorded values; setting an eligibility flag on a synthetic household built via `from_situation` produced £0 PIP, and PIP-rate reforms had no effect even on FRS data when the recorded amount sat outside the modelled rate. This change adds: - `PipParams` Rust struct (and Python wrapper class) with the four PIP weekly rates: daily-living standard/enhanced and mobility standard/enhanced - 2025/26 rates per gov.uk/pip/what-youll-get sourced under Welfare Reform Act 2012 s.79 / SI 2013/377 - `pip_daily_living_amount` and `pip_mobility_amount` helpers in `src/variables/benefits.rs` that: - Pass through any FRS-recorded amount unchanged (preserves existing calibration behaviour) - Otherwise compute from the eligibility flag × the rate parameter - Return 0 when neither holds or `params.pip` is unset - `passthrough_benefits` now uses these helpers, so PIP from flags flows into total_benefits and downstream household net income Tests: - 8 Rust unit tests covering the std/enh/recorded-override/no-flag/no- params/reform-scaling paths - 4 YAML policy-test cases covering the same paths end-to-end Stacked on #55 (LBTT/LTT). Co-Authored-By: Claude Opus 4.7 (1M context) --- changelog.d/added/pip-from-flags.md | 1 + .../policyengine_uk_compiled/__init__.py | 2 + .../python/policyengine_uk_compiled/models.py | 15 ++ parameters/2025_26.yaml | 8 ++ src/parameters/mod.rs | 16 +++ src/variables/benefits.rs | 131 +++++++++++++++++- tests/policy/pip.yaml | 99 +++++++++++++ 7 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added/pip-from-flags.md create mode 100644 tests/policy/pip.yaml diff --git a/changelog.d/added/pip-from-flags.md b/changelog.d/added/pip-from-flags.md new file mode 100644 index 0000000..939886f --- /dev/null +++ b/changelog.d/added/pip-from-flags.md @@ -0,0 +1 @@ +Add PIP daily-living and mobility amount computation from existing eligibility flags. Synthetic households built via `Simulation.from_situation` that set `pip_dl_std`/`pip_dl_enh`/`pip_mob_std`/`pip_mob_enh` now produce non-zero PIP amounts (using the new `PipParams` weekly rates), and PIP-rate reforms now flow through to the modelled amount on those households. FRS-recorded amounts continue to pass through unchanged. diff --git a/interfaces/python/policyengine_uk_compiled/__init__.py b/interfaces/python/policyengine_uk_compiled/__init__.py index e418d5e..0f455ab 100644 --- a/interfaces/python/policyengine_uk_compiled/__init__.py +++ b/interfaces/python/policyengine_uk_compiled/__init__.py @@ -58,6 +58,7 @@ def print_guide(): StampDutyParams, CapitalGainsTaxParams, WealthTaxParams, + PipParams, LabourSupplyParams, Parameters, ) @@ -107,6 +108,7 @@ def print_guide(): "StampDutyParams", "CapitalGainsTaxParams", "WealthTaxParams", + "PipParams", "LabourSupplyParams", "Parameters", ] diff --git a/interfaces/python/policyengine_uk_compiled/models.py b/interfaces/python/policyengine_uk_compiled/models.py index 0882553..3edb23a 100644 --- a/interfaces/python/policyengine_uk_compiled/models.py +++ b/interfaces/python/policyengine_uk_compiled/models.py @@ -169,6 +169,20 @@ class WealthTaxParams(BaseModel): rate: Optional[float] = None +class PipParams(BaseModel): + """Personal Independence Payment weekly rates. + + Welfare Reform Act 2012 s.79; SI 2013/377. Set any of the four weekly + rates to model PIP-rate reforms; recipients are identified by the + `pip_dl_std` / `pip_dl_enh` / `pip_mob_std` / `pip_mob_enh` flags on + each Person. + """ + daily_living_standard_weekly: Optional[float] = None + daily_living_enhanced_weekly: Optional[float] = None + mobility_standard_weekly: Optional[float] = None + mobility_enhanced_weekly: Optional[float] = None + + class LabourSupplyParams(BaseModel): """OBR labour supply elasticities (Slutsky decomposition). @@ -226,6 +240,7 @@ class Parameters(BaseModel): stamp_duty: Optional[StampDutyParams] = None lbtt: Optional[StampDutyParams] = None ltt: Optional[StampDutyParams] = None + pip: Optional["PipParams"] = None wealth_tax: Optional[WealthTaxParams] = None labour_supply: Optional[LabourSupplyParams] = None diff --git a/parameters/2025_26.yaml b/parameters/2025_26.yaml index 41df0d5..d6a0b67 100644 --- a/parameters/2025_26.yaml +++ b/parameters/2025_26.yaml @@ -288,6 +288,14 @@ wealth_tax: threshold: 10000000.0 rate: 0.01 +pip: + # Personal Independence Payment weekly rates from 7 April 2025. + # Welfare Reform Act 2012 s.79 / SI 2013/377. Source: gov.uk/pip/what-youll-get. + daily_living_standard_weekly: 73.90 + daily_living_enhanced_weekly: 110.40 + mobility_standard_weekly: 29.20 + mobility_enhanced_weekly: 77.05 + lha: # Local Housing Allowance rates for 2025/26. # LHA was re-frozen in April 2025 at the 2024/25 reset rates. diff --git a/src/parameters/mod.rs b/src/parameters/mod.rs index ad69075..0752ddf 100644 --- a/src/parameters/mod.rs +++ b/src/parameters/mod.rs @@ -71,6 +71,9 @@ pub struct Parameters { /// for their bedroom entitlement category. Authority: HB Regs 2006 reg.13D. #[serde(default)] pub lha: Option, + /// Personal Independence Payment weekly rates. Welfare Reform Act 2012 s.79. + #[serde(default)] + pub pip: Option, /// OBR labour supply response elasticities. /// When enabled, the Slutsky-decomposition elasticities from OBR (2023) are applied /// to estimate intensive-margin labour supply responses to tax-benefit reforms. @@ -454,6 +457,19 @@ pub struct StampDutyParams { fn default_purchase_probability() -> f64 { 0.043 } +/// Personal Independence Payment weekly component rates. +/// +/// PIP has two components — daily living and mobility — each at a standard or +/// enhanced rate. Welfare Reform Act 2012 s.79 / Social Security (Personal +/// Independence Payment) Regulations 2013 (SI 2013/377). Rates uprated annually. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipParams { + pub daily_living_standard_weekly: f64, + pub daily_living_enhanced_weekly: f64, + pub mobility_standard_weekly: f64, + pub mobility_enhanced_weekly: f64, +} + /// OBR labour supply response elasticities (Slutsky decomposition). /// /// Source: OBR (2023) "Costing a cut in National Insurance contributions: the diff --git a/src/variables/benefits.rs b/src/variables/benefits.rs index 8971331..5884c00 100644 --- a/src/variables/benefits.rs +++ b/src/variables/benefits.rs @@ -101,7 +101,7 @@ pub fn calculate_benunit( // All exempt from the benefit cap. let passthrough_benefits: f64 = bu.person_ids.iter().map(|&pid| { let p = &people[pid]; - p.pip_daily_living + p.pip_mobility + pip_daily_living_amount(p, params) + pip_mobility_amount(p, params) + p.dla_care + p.dla_mobility + p.attendance_allowance + p.esa_contributory @@ -312,6 +312,43 @@ fn calculate_universal_credit( /// New SP started April 2016. SP age is 66. So in fiscal year Y, the cutoff /// is: anyone aged > 66 + (Y - 2016) was already SP-age when new SP began, /// and is therefore on basic SP. Everyone else on SP is on new SP. +/// PIP daily-living component amount (annual). +/// +/// If the person has a recorded amount (`p.pip_daily_living > 0`), returns that +/// amount unchanged — preserves FRS-recorded values which may reflect partial- +/// year claims, transitional protection, or amounts predating a reform. If the +/// recorded amount is 0 but the standard or enhanced flag is set, returns the +/// computed weekly rate × 52 from `params.pip`. Returns 0 when neither holds +/// or when no PIP parameters are loaded. +pub fn pip_daily_living_amount(p: &Person, params: &Parameters) -> f64 { + if p.pip_daily_living > 0.0 { + return p.pip_daily_living; + } + let pip = match ¶ms.pip { Some(p) => p, None => return 0.0 }; + if p.pip_dl_enh { + pip.daily_living_enhanced_weekly * 52.0 + } else if p.pip_dl_std { + pip.daily_living_standard_weekly * 52.0 + } else { + 0.0 + } +} + +/// PIP mobility component amount (annual). See `pip_daily_living_amount`. +pub fn pip_mobility_amount(p: &Person, params: &Parameters) -> f64 { + if p.pip_mobility > 0.0 { + return p.pip_mobility; + } + let pip = match ¶ms.pip { Some(p) => p, None => return 0.0 }; + if p.pip_mob_enh { + pip.mobility_enhanced_weekly * 52.0 + } else if p.pip_mob_std { + pip.mobility_standard_weekly * 52.0 + } else { + 0.0 + } +} + /// Calculate reform-adjusted state pension for a single person. /// New SP recipients get the full parameter rate; basic SP recipients get /// their reported amount scaled by the reform ratio. @@ -2130,4 +2167,96 @@ mod parameter_impact_tests { // HB for private renter at £2500/month rent in London should be capped at 1-bed LHA £1200.81/month assert!(hb_private <= 1200.81 * 12.0 + 1.0, "HB should not exceed LHA cap for private renter"); } + + // ── PIP amount-from-flags ───────────────────────────────────────────────── + + #[test] + fn pip_dl_enhanced_from_flag_when_amount_zero() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 35.0; + p.pip_dl_enh = true; + // 2025/26 PIP DL enhanced: £110.40/week × 52 = £5,740.80 + let amount = pip_daily_living_amount(&p, ¶ms); + assert!((amount - 5_740.80).abs() < 0.01, "got {}", amount); + } + + #[test] + fn pip_dl_standard_from_flag_when_amount_zero() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 35.0; + p.pip_dl_std = true; + // £73.90 × 52 = £3,842.80 + assert!((pip_daily_living_amount(&p, ¶ms) - 3_842.80).abs() < 0.01); + } + + #[test] + fn pip_mob_enhanced_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 35.0; + p.pip_mob_enh = true; + // £77.05 × 52 = £4,006.60 + assert!((pip_mobility_amount(&p, ¶ms) - 4_006.60).abs() < 0.01); + } + + #[test] + fn pip_recorded_amount_overrides_flag() { + // FRS data: amount may differ from full annual rate (partial year, etc.). + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 35.0; + p.pip_dl_enh = true; + p.pip_daily_living = 4_000.0; // recorded — should pass through unchanged + assert_eq!(pip_daily_living_amount(&p, ¶ms), 4_000.0); + } + + #[test] + fn pip_no_flag_no_recorded_returns_zero() { + let params = Parameters::for_year(2025).unwrap(); + let p = Person::default(); + assert_eq!(pip_daily_living_amount(&p, ¶ms), 0.0); + assert_eq!(pip_mobility_amount(&p, ¶ms), 0.0); + } + + #[test] + fn pip_returns_zero_when_params_missing() { + let mut params = Parameters::for_year(2025).unwrap(); + params.pip = None; + let mut p = Person::default(); + p.pip_dl_enh = true; + assert_eq!(pip_daily_living_amount(&p, ¶ms), 0.0); + } + + #[test] + fn pip_flows_into_passthrough_benefits() { + // A synthetic household with PIP enhanced flag should see the benefit + // amount appear in `total_benefits`. + let (params, mut p, bu, hh) = base_person_uc(); + p.pip_dl_enh = true; + p.pip_mob_std = true; + let result = calc(¶ms, &[p], &bu, &hh); + // Passthrough = DL enhanced (£5740.80) + Mob standard (£1518.40) = £7259.20 + let expected_passthrough = 5_740.80 + 1_518.40; + assert!(result.passthrough_benefits >= expected_passthrough - 0.01, + "passthrough_benefits = {}, expected at least {}", + result.passthrough_benefits, expected_passthrough); + } + + #[test] + fn pip_param_change_flows_through() { + // Reform: doubling the DL enhanced rate should double the synthetic + // household's PIP DL amount. + let (mut params, mut p, bu, hh) = base_person_uc(); + p.pip_dl_enh = true; + let baseline = calc(¶ms, &[p.clone()], &bu, &hh).passthrough_benefits; + if let Some(pip) = params.pip.as_mut() { + pip.daily_living_enhanced_weekly *= 2.0; + } + let reformed = calc(¶ms, &[p], &bu, &hh).passthrough_benefits; + // Reform should add another £5,740.80 of PIP DL enhanced. + assert!((reformed - baseline - 5_740.80).abs() < 0.01, + "baseline={}, reformed={}, delta={}", baseline, reformed, reformed - baseline); + } } diff --git a/tests/policy/pip.yaml b/tests/policy/pip.yaml new file mode 100644 index 0000000..857b766 --- /dev/null +++ b/tests/policy/pip.yaml @@ -0,0 +1,99 @@ +# Personal Independence Payment from eligibility flags. +# +# 2025/26 weekly rates (gov.uk/pip/what-youll-get): +# Daily living standard: £73.90 → £3,842.80/yr +# Daily living enhanced: £110.40 → £5,740.80/yr +# Mobility standard: £29.20 → £1,518.40/yr +# Mobility enhanced: £77.05 → £4,006.60/yr +# +# These cases set `would_claim_*: false` on the benunit to suppress modelled +# means-tested benefits (UC/HB/PC), isolating PIP in `baseline_total_benefits`. + +- name: PIP DL enhanced + mobility standard from flags + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 35, employment_income: 0, pip_dl_enh: true, pip_mob_std: true } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + # £5,740.80 + £1,518.40 = £7,259.20 + baseline_total_benefits: 7259 + +- name: PIP DL standard alone + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 40, employment_income: 0, pip_dl_std: true } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + baseline_total_benefits: 3843 + +- name: PIP mobility enhanced alone + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 40, employment_income: 0, pip_mob_enh: true } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + baseline_total_benefits: 4007 + +- name: No PIP flags — no benefits when claiming disabled + period: 2025 + absolute_error_margin: 0 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + baseline_total_benefits: 0 From 85327fa2fcaeee2474d0a21979a23abed94a6d67 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 11:10:00 +0100 Subject: [PATCH 6/9] feat: extend amount-from-flags to DLA and AA (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the pattern from #56 (PIP) to: - DLA care component (low/mid/high) — SSCBA 1992 Sch.2 para.2 - DLA mobility component (low/high) — SSCBA 1992 Sch.2 para.3 - Attendance Allowance (low/high) — SSCBA 1992 s.64 Synthetic households that set `dla_care_*` / `dla_mob_*` / `aa_*` eligibility flags now produce non-zero amounts via the new `DlaParams` and `AaParams` structs (with 2025/26 weekly rates from gov.uk). FRS-recorded amounts continue to pass through unchanged. Adds: - 2025/26 rates in `parameters/2025_26.yaml`: DLA care low/mid/high £29.20/£73.90/£110.40 weekly, DLA mob low/high £29.20/£77.05 weekly, AA low/high £73.90/£110.40 weekly - Helpers `dla_care_amount`, `dla_mobility_amount`, `attendance_allowance_amount` in `src/variables/benefits.rs` - 10 Rust unit tests (recorded-override / no-flag / per-band-rate / passthrough flow) - 4 YAML policy-test cases under `tests/policy/dla_aa.yaml` - Python wrapper exposure (`DlaParams`, `AaParams`, `Parameters.dla`, `Parameters.aa`) Stacked on #56. Co-Authored-By: Claude Opus 4.7 (1M context) --- changelog.d/added/dla-aa-from-flags.md | 1 + .../policyengine_uk_compiled/__init__.py | 4 + .../python/policyengine_uk_compiled/models.py | 26 +++ parameters/2025_26.yaml | 15 ++ src/parameters/mod.rs | 34 ++++ src/variables/benefits.rs | 151 +++++++++++++++++- tests/policy/dla_aa.yaml | 108 +++++++++++++ 7 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added/dla-aa-from-flags.md create mode 100644 tests/policy/dla_aa.yaml diff --git a/changelog.d/added/dla-aa-from-flags.md b/changelog.d/added/dla-aa-from-flags.md new file mode 100644 index 0000000..db0c02e --- /dev/null +++ b/changelog.d/added/dla-aa-from-flags.md @@ -0,0 +1 @@ +Extend the PIP-from-flags pattern to DLA care/mobility components and Attendance Allowance. New `DlaParams` and `AaParams` structs (with 2025/26 weekly rates from gov.uk under SSCBA 1992 Sch.2 paras 2–3 and SSCBA 1992 s.64) are exposed in the Python wrapper. Synthetic households built via `Simulation.from_situation` that set the `dla_care_*` / `dla_mob_*` / `aa_*` flags now produce non-zero amounts; FRS-recorded amounts continue to pass through unchanged. diff --git a/interfaces/python/policyengine_uk_compiled/__init__.py b/interfaces/python/policyengine_uk_compiled/__init__.py index 0f455ab..d82014a 100644 --- a/interfaces/python/policyengine_uk_compiled/__init__.py +++ b/interfaces/python/policyengine_uk_compiled/__init__.py @@ -59,6 +59,8 @@ def print_guide(): CapitalGainsTaxParams, WealthTaxParams, PipParams, + DlaParams, + AaParams, LabourSupplyParams, Parameters, ) @@ -109,6 +111,8 @@ def print_guide(): "CapitalGainsTaxParams", "WealthTaxParams", "PipParams", + "DlaParams", + "AaParams", "LabourSupplyParams", "Parameters", ] diff --git a/interfaces/python/policyengine_uk_compiled/models.py b/interfaces/python/policyengine_uk_compiled/models.py index 3edb23a..d4e1a0b 100644 --- a/interfaces/python/policyengine_uk_compiled/models.py +++ b/interfaces/python/policyengine_uk_compiled/models.py @@ -183,6 +183,30 @@ class PipParams(BaseModel): mobility_enhanced_weekly: Optional[float] = None +class DlaParams(BaseModel): + """Disability Living Allowance weekly rates. + + SSCBA 1992 Sch.2 paras 2–3. Recipients are identified by the + `dla_care_low` / `dla_care_mid` / `dla_care_high` and + `dla_mob_low` / `dla_mob_high` flags on each Person. + """ + care_low_weekly: Optional[float] = None + care_mid_weekly: Optional[float] = None + care_high_weekly: Optional[float] = None + mobility_low_weekly: Optional[float] = None + mobility_high_weekly: Optional[float] = None + + +class AaParams(BaseModel): + """Attendance Allowance weekly rates. + + SSCBA 1992 s.64. Recipients are identified by the `aa_low` / `aa_high` + flags on each Person. + """ + low_weekly: Optional[float] = None + high_weekly: Optional[float] = None + + class LabourSupplyParams(BaseModel): """OBR labour supply elasticities (Slutsky decomposition). @@ -241,6 +265,8 @@ class Parameters(BaseModel): lbtt: Optional[StampDutyParams] = None ltt: Optional[StampDutyParams] = None pip: Optional["PipParams"] = None + dla: Optional["DlaParams"] = None + aa: Optional["AaParams"] = None wealth_tax: Optional[WealthTaxParams] = None labour_supply: Optional[LabourSupplyParams] = None diff --git a/parameters/2025_26.yaml b/parameters/2025_26.yaml index d6a0b67..6c22bb7 100644 --- a/parameters/2025_26.yaml +++ b/parameters/2025_26.yaml @@ -296,6 +296,21 @@ pip: mobility_standard_weekly: 29.20 mobility_enhanced_weekly: 77.05 +dla: + # Disability Living Allowance weekly rates from 7 April 2025. + # SSCBA 1992 Sch.2 paras 2–3. Source: gov.uk/dla-rates. + care_low_weekly: 29.20 + care_mid_weekly: 73.90 + care_high_weekly: 110.40 + mobility_low_weekly: 29.20 + mobility_high_weekly: 77.05 + +aa: + # Attendance Allowance weekly rates from 7 April 2025. + # SSCBA 1992 s.64. Source: gov.uk/attendance-allowance/what-youll-get. + low_weekly: 73.90 + high_weekly: 110.40 + lha: # Local Housing Allowance rates for 2025/26. # LHA was re-frozen in April 2025 at the 2024/25 reset rates. diff --git a/src/parameters/mod.rs b/src/parameters/mod.rs index 0752ddf..44b2e73 100644 --- a/src/parameters/mod.rs +++ b/src/parameters/mod.rs @@ -74,6 +74,14 @@ pub struct Parameters { /// Personal Independence Payment weekly rates. Welfare Reform Act 2012 s.79. #[serde(default)] pub pip: Option, + /// Disability Living Allowance weekly rates (under-16 successor: now PIP/ADP). + /// SSCBA 1992 Sch.2 paras 2–3. + #[serde(default)] + pub dla: Option, + /// Attendance Allowance weekly rates (over-SP-age disability benefit). + /// SSCBA 1992 s.64. + #[serde(default)] + pub aa: Option, /// OBR labour supply response elasticities. /// When enabled, the Slutsky-decomposition elasticities from OBR (2023) are applied /// to estimate intensive-margin labour supply responses to tax-benefit reforms. @@ -470,6 +478,32 @@ pub struct PipParams { pub mobility_enhanced_weekly: f64, } +/// Disability Living Allowance weekly component rates. +/// +/// DLA has a care component (lowest, middle, highest) and a mobility component +/// (lower, higher). Working-age claimants migrated to PIP from 2013; remaining +/// DLA caseload is mostly children and pre-PIP-migration adults. +/// SSCBA 1992 Sch.2 para.2 (care) and para.3 (mobility). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DlaParams { + pub care_low_weekly: f64, + pub care_mid_weekly: f64, + pub care_high_weekly: f64, + pub mobility_low_weekly: f64, + pub mobility_high_weekly: f64, +} + +/// Attendance Allowance weekly rates. +/// +/// Non-means-tested benefit for people over State Pension age who need help +/// with personal care. SSCBA 1992 s.64. Two rates: lower (day-only or night-only +/// care needed) and higher (both). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AaParams { + pub low_weekly: f64, + pub high_weekly: f64, +} + /// OBR labour supply response elasticities (Slutsky decomposition). /// /// Source: OBR (2023) "Costing a cut in National Insurance contributions: the diff --git a/src/variables/benefits.rs b/src/variables/benefits.rs index 5884c00..57ed764 100644 --- a/src/variables/benefits.rs +++ b/src/variables/benefits.rs @@ -102,8 +102,8 @@ pub fn calculate_benunit( let passthrough_benefits: f64 = bu.person_ids.iter().map(|&pid| { let p = &people[pid]; pip_daily_living_amount(p, params) + pip_mobility_amount(p, params) - + p.dla_care + p.dla_mobility - + p.attendance_allowance + + dla_care_amount(p, params) + dla_mobility_amount(p, params) + + attendance_allowance_amount(p, params) + p.esa_contributory + p.jsa_contributory + p.other_benefits @@ -349,6 +349,54 @@ pub fn pip_mobility_amount(p: &Person, params: &Parameters) -> f64 { } } +/// DLA care component amount (annual). Same recorded-overrides-flag pattern as +/// `pip_daily_living_amount`. SSCBA 1992 Sch.2 para.2. +pub fn dla_care_amount(p: &Person, params: &Parameters) -> f64 { + if p.dla_care > 0.0 { + return p.dla_care; + } + let dla = match ¶ms.dla { Some(p) => p, None => return 0.0 }; + if p.dla_care_high { + dla.care_high_weekly * 52.0 + } else if p.dla_care_mid { + dla.care_mid_weekly * 52.0 + } else if p.dla_care_low { + dla.care_low_weekly * 52.0 + } else { + 0.0 + } +} + +/// DLA mobility component amount (annual). SSCBA 1992 Sch.2 para.3. +pub fn dla_mobility_amount(p: &Person, params: &Parameters) -> f64 { + if p.dla_mobility > 0.0 { + return p.dla_mobility; + } + let dla = match ¶ms.dla { Some(p) => p, None => return 0.0 }; + if p.dla_mob_high { + dla.mobility_high_weekly * 52.0 + } else if p.dla_mob_low { + dla.mobility_low_weekly * 52.0 + } else { + 0.0 + } +} + +/// Attendance Allowance amount (annual). SSCBA 1992 s.64. +pub fn attendance_allowance_amount(p: &Person, params: &Parameters) -> f64 { + if p.attendance_allowance > 0.0 { + return p.attendance_allowance; + } + let aa = match ¶ms.aa { Some(p) => p, None => return 0.0 }; + if p.aa_high { + aa.high_weekly * 52.0 + } else if p.aa_low { + aa.low_weekly * 52.0 + } else { + 0.0 + } +} + /// Calculate reform-adjusted state pension for a single person. /// New SP recipients get the full parameter rate; basic SP recipients get /// their reported amount scaled by the reform ratio. @@ -2244,6 +2292,105 @@ mod parameter_impact_tests { result.passthrough_benefits, expected_passthrough); } + // ── DLA amount-from-flags ───────────────────────────────────────────────── + + #[test] + fn dla_care_high_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 12.0; + p.dla_care_high = true; + // £110.40 × 52 = £5,740.80 + assert!((dla_care_amount(&p, ¶ms) - 5_740.80).abs() < 0.01); + } + + #[test] + fn dla_care_mid_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 12.0; + p.dla_care_mid = true; + assert!((dla_care_amount(&p, ¶ms) - 3_842.80).abs() < 0.01); + } + + #[test] + fn dla_mobility_high_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 12.0; + p.dla_mob_high = true; + // £77.05 × 52 = £4,006.60 + assert!((dla_mobility_amount(&p, ¶ms) - 4_006.60).abs() < 0.01); + } + + #[test] + fn dla_recorded_amount_overrides_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.dla_care_high = true; + p.dla_care = 4_000.0; + assert_eq!(dla_care_amount(&p, ¶ms), 4_000.0); + } + + #[test] + fn dla_returns_zero_when_no_flag() { + let params = Parameters::for_year(2025).unwrap(); + let p = Person::default(); + assert_eq!(dla_care_amount(&p, ¶ms), 0.0); + assert_eq!(dla_mobility_amount(&p, ¶ms), 0.0); + } + + // ── AA amount-from-flags ────────────────────────────────────────────────── + + #[test] + fn aa_high_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 70.0; + p.aa_high = true; + assert!((attendance_allowance_amount(&p, ¶ms) - 5_740.80).abs() < 0.01); + } + + #[test] + fn aa_low_from_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 70.0; + p.aa_low = true; + assert!((attendance_allowance_amount(&p, ¶ms) - 3_842.80).abs() < 0.01); + } + + #[test] + fn aa_recorded_amount_overrides_flag() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.aa_high = true; + p.attendance_allowance = 3_000.0; + assert_eq!(attendance_allowance_amount(&p, ¶ms), 3_000.0); + } + + #[test] + fn aa_returns_zero_when_no_flag() { + let params = Parameters::for_year(2025).unwrap(); + let p = Person::default(); + assert_eq!(attendance_allowance_amount(&p, ¶ms), 0.0); + } + + #[test] + fn dla_aa_flow_into_passthrough_benefits() { + // A child on DLA care high + mobility high should see both flow into + // total_benefits, in parallel to the PIP test. + let (params, mut p, bu, hh) = base_person_uc(); + p.age = 10.0; + p.dla_care_high = true; + p.dla_mob_high = true; + let result = calc(¶ms, &[p], &bu, &hh); + // £5,740.80 + £4,006.60 = £9,747.40 + assert!(result.passthrough_benefits >= 9_747.40 - 0.01, + "passthrough_benefits = {}, expected at least 9747.40", + result.passthrough_benefits); + } + #[test] fn pip_param_change_flows_through() { // Reform: doubling the DL enhanced rate should double the synthetic diff --git a/tests/policy/dla_aa.yaml b/tests/policy/dla_aa.yaml new file mode 100644 index 0000000..a0fd8d7 --- /dev/null +++ b/tests/policy/dla_aa.yaml @@ -0,0 +1,108 @@ +# Disability Living Allowance and Attendance Allowance from eligibility flags. +# +# 2025/26 weekly rates (gov.uk): +# DLA care: low £29.20 / mid £73.90 / high £110.40 +# DLA mobility: low £29.20 / high £77.05 +# Attendance Allowance: low £73.90 / high £110.40 +# +# `would_claim_*: false` on the benunit suppresses modelled means-tested +# benefits so `baseline_total_benefits` isolates DLA/AA passthrough. + +- name: DLA care high + mobility high (child) + period: 2025 + absolute_error_margin: 1 + input: + people: + c: { age: 10, dla_care_high: true, dla_mob_high: true } + benunits: + b: + members: [c] + would_claim_uc: false + would_claim_cb: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [c], region: London } + output: + # £5,740.80 + £4,006.60 = £9,747.40 + baseline_total_benefits: 9747 + +- name: DLA care low alone + period: 2025 + absolute_error_margin: 1 + input: + people: + c: { age: 8, dla_care_low: true } + benunits: + b: + members: [c] + would_claim_uc: false + would_claim_cb: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [c], region: London } + output: + # £29.20 × 52 = £1,518.40 + baseline_total_benefits: 1518 + +- name: Attendance Allowance high (post-SP-age, new SP cohort) + period: 2025 + absolute_error_margin: 1 + input: + people: + # Age 70 < basic_sp_min_age (75 in 2025/26) → new-SP path; full £11,973/yr + p: { age: 70, employment_income: 0, aa_high: true } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_cb: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + # £110.40 × 52 = £5,740.80; SP also kicks in for age ≥ SP age + # The SP component reduces what we can isolate, so check just the AA delta + # by using a non-SP-age person with the AA flag in a separate test: + baseline_total_benefits: 17714 # = AA £5740.80 + new SP £11,973 (param) + +- name: AA low alone (pre-SP age) + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 60, employment_income: 0, aa_low: true } + benunits: + b: + members: [p] + would_claim_uc: false + would_claim_cb: false + would_claim_hb: false + would_claim_pc: false + would_claim_is: false + would_claim_ctc: false + would_claim_wtc: false + would_claim_esa: false + would_claim_jsa: false + households: + h: { members: [p], region: London } + output: + # £73.90 × 52 = £3,842.80, no SP (under SP age) + baseline_total_benefits: 3843 From 2097c87586271e8a158d6ceafafcdfee9d21659c Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 11:25:43 +0100 Subject: [PATCH 7/9] feat: add council-tax single-person discount (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Households with exactly one adult (18+) now receive a 25% discount on the calculated council tax — Local Government Finance Act 1992 s.11(1)(a). Adds: - `single_person_discount_rate` field on `CouncilTaxParams` (default 0.25) - Updates `calculate_council_tax(hh, params, is_single_adult)` to apply the discount - Counts adults via `Person::is_adult()` (age >= 18) in `simulation.rs` - New `baseline_council_tax_calculated` / `reform_council_tax_calculated` per-household microdata columns - First-time exposure of `CouncilTaxParams` in the Python wrapper - 3 new Rust unit tests (band D + band A discount, zero-discount-rate edge) - 4 new YAML policy-test cases (`tests/policy/council_tax.yaml`) The baseline run still uses the FRS-recorded `hh.council_tax` for net income; the calculated value is for reform modelling, where now reforms to either band-D rate or the discount fraction take effect. Stacked on #57. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../council-tax-single-person-discount.md | 1 + .../policyengine_uk_compiled/__init__.py | 2 + .../python/policyengine_uk_compiled/models.py | 14 ++++ src/data/clean.rs | 4 ++ src/engine/simulation.rs | 9 ++- src/parameters/mod.rs | 7 ++ src/variables/wealth_taxes.rs | 66 +++++++++++++++++-- tests/policy/council_tax.yaml | 64 ++++++++++++++++++ 8 files changed, 158 insertions(+), 9 deletions(-) create mode 100644 changelog.d/added/council-tax-single-person-discount.md create mode 100644 tests/policy/council_tax.yaml diff --git a/changelog.d/added/council-tax-single-person-discount.md b/changelog.d/added/council-tax-single-person-discount.md new file mode 100644 index 0000000..fbd43a5 --- /dev/null +++ b/changelog.d/added/council-tax-single-person-discount.md @@ -0,0 +1 @@ +Add the 25% council-tax single-person discount (Local Government Finance Act 1992 s.11(1)(a)). Households with exactly one adult (18+) now have their `council_tax_calculated` reduced by `single_person_discount_rate` (default 0.25). Also exposes `CouncilTaxParams` in the Python wrapper for the first time, and adds `baseline_council_tax_calculated` / `reform_council_tax_calculated` per-household microdata columns so reform analyses can isolate the calculated council tax. diff --git a/interfaces/python/policyengine_uk_compiled/__init__.py b/interfaces/python/policyengine_uk_compiled/__init__.py index d82014a..989ae52 100644 --- a/interfaces/python/policyengine_uk_compiled/__init__.py +++ b/interfaces/python/policyengine_uk_compiled/__init__.py @@ -61,6 +61,7 @@ def print_guide(): PipParams, DlaParams, AaParams, + CouncilTaxParams, LabourSupplyParams, Parameters, ) @@ -113,6 +114,7 @@ def print_guide(): "PipParams", "DlaParams", "AaParams", + "CouncilTaxParams", "LabourSupplyParams", "Parameters", ] diff --git a/interfaces/python/policyengine_uk_compiled/models.py b/interfaces/python/policyengine_uk_compiled/models.py index d4e1a0b..4c7a9eb 100644 --- a/interfaces/python/policyengine_uk_compiled/models.py +++ b/interfaces/python/policyengine_uk_compiled/models.py @@ -147,6 +147,19 @@ class UcMigrationRates(BaseModel): income_support: Optional[float] = None +class CouncilTaxParams(BaseModel): + """Council tax parameters. + + Local Government Finance Act 1992. Used for reform modelling — baseline + runs use the FRS-recorded `council_tax` amount per household. Set + `single_person_discount_rate` to model reforms to the s.11(1)(a) discount. + """ + average_band_d: Optional[float] = None + band_multipliers: Optional[list[float]] = None + band_thresholds: Optional[list[float]] = None + single_person_discount_rate: Optional[float] = None + + class StampDutyBand(BaseModel): rate: float threshold: float @@ -260,6 +273,7 @@ class Parameters(BaseModel): uc_migration: Optional[UcMigrationRates] = None disability_premiums: Optional[DisabilityPremiumParams] = None income_related_benefits: Optional[IncomeRelatedBenefitParams] = None + council_tax: Optional[CouncilTaxParams] = None capital_gains_tax: Optional[CapitalGainsTaxParams] = None stamp_duty: Optional[StampDutyParams] = None lbtt: Optional[StampDutyParams] = None diff --git a/src/data/clean.rs b/src/data/clean.rs index af1e383..328601b 100644 --- a/src/data/clean.rs +++ b/src/data/clean.rs @@ -545,11 +545,13 @@ fn write_microdata_csv_households( "baseline_net_income", "baseline_gross_income", "baseline_total_tax", "baseline_total_benefits", "baseline_property_transaction_tax", + "baseline_council_tax_calculated", "baseline_equivalisation_factor", "baseline_equivalised_net_income", // ── Reform outputs ── "reform_net_income", "reform_gross_income", "reform_total_tax", "reform_total_benefits", "reform_property_transaction_tax", + "reform_council_tax_calculated", "reform_equivalisation_factor", "reform_equivalised_net_income", ])?; @@ -570,6 +572,7 @@ fn write_microdata_csv_households( format!("{:.2}", bl.total_tax), format!("{:.2}", bl.total_benefits), format!("{:.2}", bl.stamp_duty), + format!("{:.2}", bl.council_tax_calculated), format!("{:.4}", bl.equivalisation_factor), format!("{:.2}", bl.equivalised_net_income), // Reform @@ -578,6 +581,7 @@ fn write_microdata_csv_households( format!("{:.2}", rf.total_tax), format!("{:.2}", rf.total_benefits), format!("{:.2}", rf.stamp_duty), + format!("{:.2}", rf.council_tax_calculated), format!("{:.4}", rf.equivalisation_factor), format!("{:.2}", rf.equivalised_net_income), ])?; diff --git a/src/engine/simulation.rs b/src/engine/simulation.rs index ffe8ec5..9e81136 100644 --- a/src/engine/simulation.rs +++ b/src/engine/simulation.rs @@ -314,9 +314,14 @@ impl Simulation { .map(|p| variables::wealth_taxes::calculate_wealth_tax(hh, p)) .unwrap_or(0.0); - // Council tax (calculated from parameters for reform modelling) + // Council tax (calculated from parameters for reform modelling). + // Applies the single-person discount when the household has exactly + // one adult (18+) — Local Government Finance Act 1992 s.11(1)(a). + let adult_count = hh.person_ids.iter() + .filter(|&&pid| self.people[pid].is_adult()) + .count(); let council_tax_calculated = self.parameters.council_tax.as_ref() - .map(|p| variables::wealth_taxes::calculate_council_tax(hh, p)) + .map(|p| variables::wealth_taxes::calculate_council_tax(hh, p, adult_count == 1)) .unwrap_or(hh.council_tax); let total_tax = direct_tax + vat + fuel_duty + alcohol_duty + tobacco_duty diff --git a/src/parameters/mod.rs b/src/parameters/mod.rs index 44b2e73..3c6dccf 100644 --- a/src/parameters/mod.rs +++ b/src/parameters/mod.rs @@ -418,8 +418,15 @@ pub struct CouncilTaxParams { /// Property value thresholds for bands A–H (1991 values, England). #[serde(default = "default_band_thresholds")] pub band_thresholds: Vec, + /// Single-person discount: fraction subtracted from council tax when only + /// one adult (18+) is resident. 25% in England/Wales/Scotland — Local + /// Government Finance Act 1992 s.11(1)(a). + #[serde(default = "default_single_person_discount")] + pub single_person_discount_rate: f64, } +fn default_single_person_discount() -> f64 { 0.25 } + fn default_band_multipliers() -> Vec { vec![6.0/9.0, 7.0/9.0, 8.0/9.0, 1.0, 11.0/9.0, 13.0/9.0, 15.0/9.0, 18.0/9.0] } diff --git a/src/variables/wealth_taxes.rs b/src/variables/wealth_taxes.rs index 866c2d9..98918c9 100644 --- a/src/variables/wealth_taxes.rs +++ b/src/variables/wealth_taxes.rs @@ -20,10 +20,22 @@ pub fn council_tax_band(property_value: f64, thresholds: &[f64]) -> usize { /// Returns the Band D rate multiplied by the band multiplier for this household's /// property value. For baseline runs, the simulation uses the reported `hh.council_tax` /// instead. -pub fn calculate_council_tax(hh: &Household, params: &CouncilTaxParams) -> f64 { +/// +/// Applies the single-person discount when `is_single_adult` is true (only one +/// adult aged 18+ is resident). Local Government Finance Act 1992 s.11(1)(a). +pub fn calculate_council_tax( + hh: &Household, + params: &CouncilTaxParams, + is_single_adult: bool, +) -> f64 { let band = council_tax_band(hh.main_residence_value, ¶ms.band_thresholds); let multiplier = params.band_multipliers.get(band).copied().unwrap_or(1.0); - params.average_band_d * multiplier + let gross = params.average_band_d * multiplier; + if is_single_adult { + gross * (1.0 - params.single_person_discount_rate) + } else { + gross + } } /// Calculate capital gains tax for a person. @@ -136,19 +148,59 @@ mod tests { assert_eq!(council_tax_band(500000.0, &thresholds), 7); // Band H } - #[test] - fn council_tax_calculation() { - let params = CouncilTaxParams { + fn make_council_tax_params() -> CouncilTaxParams { + CouncilTaxParams { average_band_d: 2280.0, band_multipliers: vec![6.0/9.0, 7.0/9.0, 8.0/9.0, 1.0, 11.0/9.0, 13.0/9.0, 15.0/9.0, 18.0/9.0], band_thresholds: vec![0.0, 40001.0, 52001.0, 68001.0, 88001.0, 120001.0, 160001.0, 320001.0], - }; + single_person_discount_rate: 0.25, + } + } + + #[test] + fn council_tax_calculation() { + let params = make_council_tax_params(); let mut hh = Household::default(); hh.main_residence_value = 80000.0; // Band D - let ct = calculate_council_tax(&hh, ¶ms); + let ct = calculate_council_tax(&hh, ¶ms, false); assert!((ct - 2280.0).abs() < 1.0); // Band D = 1.0 * band_d } + #[test] + fn council_tax_single_person_discount() { + let params = make_council_tax_params(); + let mut hh = Household::default(); + hh.main_residence_value = 80000.0; // Band D + let ct_full = calculate_council_tax(&hh, ¶ms, false); + let ct_single = calculate_council_tax(&hh, ¶ms, true); + assert!((ct_full - 2280.0).abs() < 1.0); + // 25% discount: £2,280 × 0.75 = £1,710 + assert!((ct_single - 1710.0).abs() < 1.0, "got {}", ct_single); + } + + #[test] + fn council_tax_single_person_discount_band_a() { + let params = make_council_tax_params(); + let mut hh = Household::default(); + hh.main_residence_value = 30000.0; // Band A + let band_a_full = 2280.0 * 6.0 / 9.0; // = £1,520 + let ct_full = calculate_council_tax(&hh, ¶ms, false); + let ct_single = calculate_council_tax(&hh, ¶ms, true); + assert!((ct_full - band_a_full).abs() < 1.0); + assert!((ct_single - band_a_full * 0.75).abs() < 1.0); + } + + #[test] + fn council_tax_zero_discount_rate_no_discount() { + let mut params = make_council_tax_params(); + params.single_person_discount_rate = 0.0; + let mut hh = Household::default(); + hh.main_residence_value = 80000.0; + let ct_full = calculate_council_tax(&hh, ¶ms, false); + let ct_single = calculate_council_tax(&hh, ¶ms, true); + assert_eq!(ct_full, ct_single); + } + #[test] fn cgt_basic_rate() { let params = CapitalGainsTaxParams { diff --git a/tests/policy/council_tax.yaml b/tests/policy/council_tax.yaml new file mode 100644 index 0000000..f760fec --- /dev/null +++ b/tests/policy/council_tax.yaml @@ -0,0 +1,64 @@ +# Council tax band & single-person discount. +# +# 2025/26 England average Band D = £2,280 (DLUHC Council Tax levels statistics). +# Band multipliers: A=6/9, B=7/9, C=8/9, D=1, E=11/9, F=13/9, G=15/9, H=18/9. +# Single-person discount: 25% (Local Government Finance Act 1992 s.11(1)(a)) +# applied when exactly one resident is aged 18+. + +- name: Band D, two adults — full council tax + period: 2025 + absolute_error_margin: 1 + input: + people: + p1: { age: 35, employment_income: 0 } + p2: { age: 33, employment_income: 0 } + benunits: + b: { members: [p1, p2] } + households: + h: { members: [p1, p2], region: London, main_residence_value: 80_000 } + output: + baseline_council_tax_calculated: 2280 + +- name: Band D, single adult — 25% discount applies + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 35, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: London, main_residence_value: 80_000 } + output: + # £2,280 × 0.75 = £1,710 + baseline_council_tax_calculated: 1710 + +- name: Band A, single adult + period: 2025 + absolute_error_margin: 1 + input: + people: + p: { age: 40, employment_income: 0 } + benunits: + b: { members: [p] } + households: + h: { members: [p], region: London, main_residence_value: 30_000 } + output: + # Band A multiplier = 6/9 → £2,280 × 6/9 = £1,520; ×0.75 = £1,140 + baseline_council_tax_calculated: 1140 + +- name: Band D, adult with two children — full rate (children don't count) + period: 2025 + absolute_error_margin: 1 + input: + people: + p1: { age: 35, employment_income: 0 } + c1: { age: 8 } + c2: { age: 4 } + benunits: + b: { members: [p1, c1, c2] } + households: + h: { members: [p1, c1, c2], region: London, main_residence_value: 80_000 } + output: + # Single adult → 25% discount: £2,280 × 0.75 = £1,710 + baseline_council_tax_calculated: 1710 From 89b4655a338599cf6ab63c4dd133a34ea2dc7b99 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 11:46:09 +0100 Subject: [PATCH 8/9] fix: state pension respects recorded amount for new-SP cohort (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the existing old-SP scaling pattern for the new-SP cohort: - If `person.state_pension > 0`: pass through, scaled by `(new_state_pension_weekly / baseline_new_sp_weekly)` for reform correctness - Else: fall back to `new_state_pension_weekly × 52` Previously the new-SP branch always returned the full parameter rate × 52, ignoring any recorded amount. This over-stated SP for partial- year claimants and broke parity for the pensioner_couple synthetic scenario in PR #53's parity harness (£946 diff). Implementation: - Plumb `baseline_new_sp_weekly` through `Simulation`, `calculate_benunit`, `calculate_state_pension`, and `person_state_pension`, parallel to the existing `baseline_old_sp_weekly` field - 3 new Rust unit tests (recorded-amount preserved, fallback to param when no record, recorded amount scales under reform) Parity-harness impact (synthetic pensioner_couple scenario): state_pension rust=23,000 py=23,000 diff=£0 (was £946) household_net_income diff=£-41 (was £905) Stacked on #58. Closes #59 (filed today as a follow-up to PR #53). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../fixed/state-pension-recorded-amount.md | 1 + src/engine/simulation.rs | 18 ++- src/main.rs | 3 +- src/variables/benefits.rs | 111 ++++++++++++++---- src/variables/labour_supply.rs | 12 +- 5 files changed, 110 insertions(+), 35 deletions(-) create mode 100644 changelog.d/fixed/state-pension-recorded-amount.md diff --git a/changelog.d/fixed/state-pension-recorded-amount.md b/changelog.d/fixed/state-pension-recorded-amount.md new file mode 100644 index 0000000..6e67bc3 --- /dev/null +++ b/changelog.d/fixed/state-pension-recorded-amount.md @@ -0,0 +1 @@ +Fix `person_state_pension` to respect the recorded `state_pension` amount for the new-SP cohort (those below the basic-SP age cutoff). Previously the new-SP branch always returned the full `new_state_pension_weekly × 52`, ignoring any recorded amount and over-stating SP for partial-year and partial-record claimants. Now mirrors the existing old-SP scaling pattern: recorded amounts pass through (scaled by reform-ratio when the new-SP rate changes), with fallback to the parameter rate when no amount is recorded. Closes #59. Shrinks the parity-harness pensioner-couple state-pension diff from £946 → £0. diff --git a/src/engine/simulation.rs b/src/engine/simulation.rs index 9e81136..f8f20c9 100644 --- a/src/engine/simulation.rs +++ b/src/engine/simulation.rs @@ -101,6 +101,8 @@ pub struct Simulation { pub parameters: Parameters, /// Baseline old basic SP weekly rate for scaling reported amounts under reforms. pub baseline_old_sp_weekly: f64, + /// Baseline new SP weekly rate for scaling reported amounts under reforms. + pub baseline_new_sp_weekly: f64, /// Fiscal year (e.g. 2025 for 2025/26) — used for new/basic SP cutoff. pub fiscal_year: u32, } @@ -114,25 +116,27 @@ impl Simulation { fiscal_year: u32, ) -> Self { let baseline_old_sp_weekly = parameters.state_pension.old_basic_pension_weekly; + let baseline_new_sp_weekly = parameters.state_pension.new_state_pension_weekly; Simulation { people, benunits, households, parameters, - baseline_old_sp_weekly, fiscal_year, + baseline_old_sp_weekly, baseline_new_sp_weekly, fiscal_year, } } - /// Create a simulation with explicit baseline old SP rate (for reform simulations - /// where the baseline rate differs from the reform parameters). + /// Create a simulation with explicit baseline SP rates (for reform simulations + /// where the baseline rates differ from the reform parameters). pub fn new_with_baseline_sp( people: Vec, benunits: Vec, households: Vec, parameters: Parameters, baseline_old_sp_weekly: f64, + baseline_new_sp_weekly: f64, fiscal_year: u32, ) -> Self { Simulation { people, benunits, households, parameters, - baseline_old_sp_weekly, fiscal_year, + baseline_old_sp_weekly, baseline_new_sp_weekly, fiscal_year, } } @@ -146,10 +150,11 @@ impl Simulation { // Phase 1a: Calculate each person's state pension under the current policy. // State pension is taxable income so must be computed before income tax. let baseline_old_sp = self.baseline_old_sp_weekly; + let baseline_new_sp = self.baseline_new_sp_weekly; let fiscal_year = self.fiscal_year; let person_sp: Vec = self.people.par_iter().map(|p| { variables::benefits::person_state_pension( - p, &self.parameters, baseline_old_sp, fiscal_year, + p, &self.parameters, baseline_old_sp, baseline_new_sp, fiscal_year, ) }).collect(); @@ -173,7 +178,7 @@ impl Simulation { let hh = &self.households[bu.household_id]; variables::benefits::calculate_benunit( bu, &self.people, &person_results, hh, &self.parameters, - baseline_old_sp, fiscal_year, + baseline_old_sp, baseline_new_sp, fiscal_year, ) }).collect(); benunit_results = br; @@ -707,6 +712,7 @@ mod tests { adjusted_people, benunits.clone(), households.clone(), policy_params.clone(), baseline_params.state_pension.old_basic_pension_weekly, + baseline_params.state_pension.new_state_pension_weekly, 2025, ); let dynamic_results = dynamic_sim.run(); diff --git a/src/main.rs b/src/main.rs index 25f5362..b4596b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -561,13 +561,14 @@ fn main() -> anyhow::Result<()> { dataset.people.clone() }; - // Run policy simulation (pass baseline old SP rate so reported amounts scale correctly) + // Run policy simulation (pass baseline old + new SP rates so reported amounts scale correctly) let policy_sim = Simulation::new_with_baseline_sp( policy_people, dataset.benunits.clone(), dataset.households.clone(), policy_params.clone(), baseline_params.state_pension.old_basic_pension_weekly, + baseline_params.state_pension.new_state_pension_weekly, cli.year, ); let reformed = policy_sim.run(); diff --git a/src/variables/benefits.rs b/src/variables/benefits.rs index 57ed764..f095cf8 100644 --- a/src/variables/benefits.rs +++ b/src/variables/benefits.rs @@ -15,12 +15,13 @@ pub fn calculate_benunit( household: &Household, params: &Parameters, baseline_old_sp_weekly: f64, + baseline_new_sp_weekly: f64, fiscal_year: u32, ) -> BenUnitResult { // Non-means-tested / universal benefits (available regardless of UC/legacy) let child_benefit = calculate_child_benefit(bu, people, person_results, params); let state_pension = calculate_state_pension( - bu, people, params, baseline_old_sp_weekly, fiscal_year, + bu, people, params, baseline_old_sp_weekly, baseline_new_sp_weekly, fiscal_year, ); // Carers Allowance: non-means-tested flat rate for informal carers. // Paid to individual, regardless of UC/legacy system. @@ -398,12 +399,17 @@ pub fn attendance_allowance_amount(p: &Person, params: &Parameters) -> f64 { } /// Calculate reform-adjusted state pension for a single person. -/// New SP recipients get the full parameter rate; basic SP recipients get -/// their reported amount scaled by the reform ratio. +/// +/// For both the basic-SP cohort (age ≥ `basic_sp_min_age` for the fiscal year) +/// and the new-SP cohort (younger but ≥ SP age), a recorded amount on the +/// person is preserved and scaled by the reform's ratio over the baseline +/// rate. When no amount is recorded (e.g. synthetic households built via +/// `from_situation`), fall back to the full parameter rate × 52. pub fn person_state_pension( person: &Person, params: &Parameters, baseline_old_sp_weekly: f64, + baseline_new_sp_weekly: f64, fiscal_year: u32, ) -> f64 { if !person.is_sp_age() || !person.is_adult() { @@ -415,17 +421,27 @@ pub fn person_state_pension( if person.age >= basic_sp_min_age { // Basic SP: scale reported amount by reform ratio - let old_sp_scale = if baseline_old_sp_weekly > 0.0 { + let scale = if baseline_old_sp_weekly > 0.0 { sp.old_basic_pension_weekly / baseline_old_sp_weekly } else { 1.0 }; if person.state_pension > 0.0 { - person.state_pension * old_sp_scale + person.state_pension * scale } else { sp.old_basic_pension_weekly * 52.0 } } else { - // New SP: use full parameter rate directly - sp.new_state_pension_weekly * 52.0 + // New SP: same scaling pattern as basic SP. Previously this branch + // ignored `person.state_pension` and always returned the full + // parameter rate, over-stating SP for partial-year / partial-record + // claimants. + let scale = if baseline_new_sp_weekly > 0.0 { + sp.new_state_pension_weekly / baseline_new_sp_weekly + } else { 1.0 }; + if person.state_pension > 0.0 { + person.state_pension * scale + } else { + sp.new_state_pension_weekly * 52.0 + } } } @@ -434,10 +450,13 @@ fn calculate_state_pension( people: &[Person], params: &Parameters, baseline_old_sp_weekly: f64, + baseline_new_sp_weekly: f64, fiscal_year: u32, ) -> f64 { bu.person_ids.iter() - .map(|&pid| person_state_pension(&people[pid], params, baseline_old_sp_weekly, fiscal_year)) + .map(|&pid| person_state_pension( + &people[pid], params, baseline_old_sp_weekly, baseline_new_sp_weekly, fiscal_year, + )) .sum() } @@ -1202,7 +1221,7 @@ mod tests { let person_results: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); let expected_cb = params.child_benefit.eldest_weekly * 52.0 + params.child_benefit.additional_weekly * 52.0; assert!((result.child_benefit - expected_cb).abs() < 1.0); @@ -1215,7 +1234,7 @@ mod tests { let person_results: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); assert!(result.universal_credit > 0.0, "Low earner should receive UC"); } @@ -1227,14 +1246,14 @@ mod tests { let person_results: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); assert!(result.uc_max_amount > 0.0); let (people2, bu2, hh2) = make_single_bu(10000.0, 1); let pr2: Vec = people2.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result2 = calculate_benunit(&bu2, &people2, &pr2, &hh2, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result2 = calculate_benunit(&bu2, &people2, &pr2, &hh2, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); assert!(result.uc_max_amount > result2.uc_max_amount, "Disabled child should increase UC max amount"); } @@ -1248,7 +1267,7 @@ mod tests { let person_results: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); let expected_min = (params.universal_credit.standard_allowance_single_over25 + params.universal_credit.lcwra_element + 800.0) * 12.0; @@ -1264,7 +1283,7 @@ mod tests { let person_results: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &person_results, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); assert!(result.uc_income_reduction >= 5000.0, "£5000 unearned income should reduce UC by at least £5000, got {}", result.uc_income_reduction); } @@ -1291,7 +1310,7 @@ mod tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); let mg_annual = params.pension_credit.standard_minimum_single * 52.0; // GC = mg - income assert!(result.pension_credit > 0.0, "Should receive pension credit"); @@ -1321,7 +1340,7 @@ mod tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); // seed=0.85 > migration rate 0.70 → not yet migrated, still on HB assert!(result.housing_benefit > 0.0, "Low earner not yet migrated should get HB"); assert!(result.housing_benefit <= 7200.0, "HB should not exceed rent"); @@ -1353,7 +1372,7 @@ mod tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); // seed=0.85 < migration rate 0.95 → migrated to UC assert!(result.universal_credit > 0.0, "Low-income lone parent migrated from tax credits should receive UC. UC={}", @@ -1369,7 +1388,7 @@ mod tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); // With 4 children and £3000/month rent, total benefits should hit cap if let Some(bc) = ¶ms.benefit_cap { let cap = bc.non_single_london; @@ -1403,7 +1422,7 @@ mod tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, ¶ms, p.state_pension)) .collect(); - let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, 2025); + let result = calculate_benunit(&bu, &people, &pr, &hh, ¶ms, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025); if let Some(scp) = ¶ms.scottish_child_payment { let expected = scp.weekly_amount * 52.0; assert!((result.scottish_child_payment - expected).abs() < 1.0, @@ -1444,7 +1463,7 @@ mod parameter_impact_tests { let pr: Vec = people.iter() .map(|p| crate::variables::income_tax::calculate(p, params, p.state_pension)) .collect(); - calculate_benunit(bu, people, &pr, hh, params, params.state_pension.old_basic_pension_weekly, 2025) + calculate_benunit(bu, people, &pr, hh, params, params.state_pension.old_basic_pension_weekly, params.state_pension.new_state_pension_weekly, 2025) } // ── UC parameters ──────────────────────────────────────────────────────── @@ -1648,6 +1667,50 @@ mod parameter_impact_tests { // ── State Pension parameters ────────────────────────────────────────────── + #[test] + fn new_sp_uses_recorded_amount_when_present() { + // Regression test for #59: a person aged 70 (new-SP cohort) with + // recorded state_pension = £11,500 should produce £11,500, not the + // full new-SP weekly rate × 52. + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 70.0; + p.state_pension = 11_500.0; + let baseline_old = params.state_pension.old_basic_pension_weekly; + let baseline_new = params.state_pension.new_state_pension_weekly; + let sp = person_state_pension(&p, ¶ms, baseline_old, baseline_new, 2025); + assert!((sp - 11_500.0).abs() < 0.01, + "expected 11500 (recorded), got {}", sp); + } + + #[test] + fn new_sp_falls_back_to_param_when_no_record() { + let params = Parameters::for_year(2025).unwrap(); + let mut p = Person::default(); + p.age = 70.0; + // state_pension = 0 (default) → use parameter rate × 52 + let baseline_old = params.state_pension.old_basic_pension_weekly; + let baseline_new = params.state_pension.new_state_pension_weekly; + let sp = person_state_pension(&p, ¶ms, baseline_old, baseline_new, 2025); + assert!((sp - params.state_pension.new_state_pension_weekly * 52.0).abs() < 0.01); + } + + #[test] + fn new_sp_recorded_amount_scales_under_reform() { + // Doubling the new-SP weekly rate should double the recorded SP for + // a new-SP cohort claimant. + let mut params = Parameters::for_year(2025).unwrap(); + let baseline_new = params.state_pension.new_state_pension_weekly; + params.state_pension.new_state_pension_weekly *= 2.0; + let mut p = Person::default(); + p.age = 70.0; + p.state_pension = 11_500.0; + let baseline_old = params.state_pension.old_basic_pension_weekly; + let sp = person_state_pension(&p, ¶ms, baseline_old, baseline_new, 2025); + assert!((sp - 23_000.0).abs() < 0.01, + "expected 23000 (11500 x 2.0 reform ratio), got {}", sp); + } + #[test] fn param_state_pension_new_weekly() { let (mut params, _, _, hh) = base_person_uc(); @@ -2162,7 +2225,7 @@ mod parameter_impact_tests { ..Household::default() }; let pr: Vec = vec![crate::variables::income_tax::calculate(&p, ¶ms, 0.0)]; - let result = calculate_benunit(&bu, &[p.clone()], &pr, &hh, ¶ms, 0.0, 2025); + let result = calculate_benunit(&bu, &[p.clone()], &pr, &hh, ¶ms, 0.0, 0.0, 2025); // UC housing element should be capped at 1-bed London LHA rate (£1,200.81/month) // uc_max_amount includes all elements; housing element monthly = 1200.81, annual = 14409.72 @@ -2178,7 +2241,7 @@ mod parameter_impact_tests { tenure_type: TenureType::RentFromCouncil, ..hh.clone() }; - let result_social = calculate_benunit(&bu, &[p], &pr, &hh_social, ¶ms, 0.0, 2025); + let result_social = calculate_benunit(&bu, &[p], &pr, &hh_social, ¶ms, 0.0, 0.0, 2025); assert!( result_social.uc_max_amount > result.uc_max_amount, "Social renter should get higher UC housing element (no LHA cap) vs private renter above cap" @@ -2207,8 +2270,8 @@ mod parameter_impact_tests { let hh_social = Household { tenure_type: TenureType::RentFromCouncil, ..hh_private.clone() }; let pr: Vec = vec![crate::variables::income_tax::calculate(&p, ¶ms, 0.0)]; - let hb_private = calculate_benunit(&bu, &[p.clone()], &pr, &hh_private, ¶ms, 0.0, 2025).housing_benefit; - let hb_social = calculate_benunit(&bu, &[p], &pr, &hh_social, ¶ms, 0.0, 2025).housing_benefit; + let hb_private = calculate_benunit(&bu, &[p.clone()], &pr, &hh_private, ¶ms, 0.0, 0.0, 2025).housing_benefit; + let hb_social = calculate_benunit(&bu, &[p], &pr, &hh_social, ¶ms, 0.0, 0.0, 2025).housing_benefit; assert!(hb_private > 0.0, "Private renter should still get some HB"); assert!(hb_social > hb_private, "Social renter (no cap) should get more HB than private renter above cap"); diff --git a/src/variables/labour_supply.rs b/src/variables/labour_supply.rs index 20e9f05..b2629a6 100644 --- a/src/variables/labour_supply.rs +++ b/src/variables/labour_supply.rs @@ -125,6 +125,7 @@ fn run_net_incomes( params: &Parameters, fiscal_year: u32, baseline_old_sp: f64, + baseline_new_sp: f64, ) -> Vec { let sim = Simulation::new_with_baseline_sp( people.to_vec(), @@ -132,6 +133,7 @@ fn run_net_incomes( households.to_vec(), params.clone(), baseline_old_sp, + baseline_new_sp, fiscal_year, ); sim.run().household_results.iter().map(|hr| hr.net_income).collect() @@ -170,6 +172,7 @@ fn batch_marginal_retention( params: &Parameters, fiscal_year: u32, baseline_old_sp: f64, + baseline_new_sp: f64, unperturbed_net: &[f64], slots: &[Option], max_slot: usize, @@ -188,7 +191,7 @@ fn batch_marginal_retention( let perturbed_net = run_net_incomes( &perturbed, benunits, households, - params, fiscal_year, baseline_old_sp, + params, fiscal_year, baseline_old_sp, baseline_new_sp, ); // Each perturbed household has exactly one bumped worker — attribute the @@ -225,6 +228,7 @@ pub fn apply_labour_supply_responses( } let baseline_old_sp = baseline_params.state_pension.old_basic_pension_weekly; + let baseline_new_sp = baseline_params.state_pension.new_state_pension_weekly; // Assign adult slots (O(n), no simulations) let slots = assign_adult_slots(people, households); @@ -237,18 +241,18 @@ pub fn apply_labour_supply_responses( // Unperturbed policy net incomes (the income-effect denominator) let unperturbed_policy_net = run_net_incomes( people, benunits, households, - policy_params, fiscal_year, baseline_old_sp, + policy_params, fiscal_year, baseline_old_sp, baseline_new_sp, ); // Batched marginal retention: 1 sim per slot per scenario let baseline_retention = batch_marginal_retention( people, benunits, households, - baseline_params, fiscal_year, baseline_old_sp, + baseline_params, fiscal_year, baseline_old_sp, baseline_new_sp, baseline_net, &slots, max_slot, ); let policy_retention = batch_marginal_retention( people, benunits, households, - policy_params, fiscal_year, baseline_old_sp, + policy_params, fiscal_year, baseline_old_sp, baseline_new_sp, &unperturbed_policy_net, &slots, max_slot, ); From eb8883a76ae921704758efc6d6298bbcf4e8f1fa Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 1 May 2026 12:05:48 +0100 Subject: [PATCH 9/9] fix: parity harness compares against hbai_household_net_income MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust's `baseline_net_income` is the HBAI net-income definition (gross minus direct taxes plus benefits, excluding council tax / TV licence / transaction taxes). The parity harness was comparing it against Python's broader `household_net_income`, which subtracts council_tax, TV licence, expected_sdlt/lbtt/ltt, etc., on top. Net effect: every single scenario showed a £159 diff that was exclusively the TV licence (£174.50 × ~0.911 take-up). That diff masked the real, smaller divergences and made the harness's output look worse than it was. Switching to `hbai_household_net_income` reveals: - single/couple scenarios: £1.20 / £2.40 diffs (just employer-NI rounding) - lone_parent_2kids: £554 (real UC entitlement gap) - pensioner_couple: £200 (Winter Fuel Allowance — Python includes, Rust doesn't yet) - scotland_single_45k: £1.20 The headline "couple_2kids £3,276 UC gap" from the original PR #53 description was an artefact of this measurement bug — that scenario now shows £2.40, well within tolerance. Stacked on #60. Co-Authored-By: Claude Opus 4.7 (1M context) --- changelog.d/fixed/parity-harness-hbai.md | 1 + .../python/tests/test_parity_harness.py | 2 +- scripts/parity.py | 24 ++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed/parity-harness-hbai.md diff --git a/changelog.d/fixed/parity-harness-hbai.md b/changelog.d/fixed/parity-harness-hbai.md new file mode 100644 index 0000000..63a0f4f --- /dev/null +++ b/changelog.d/fixed/parity-harness-hbai.md @@ -0,0 +1 @@ +Fix the parity harness to compare against Python's `hbai_household_net_income` rather than the broader `household_net_income`. Rust's `baseline_net_income` is the HBAI definition (gross minus direct taxes plus benefits, excluding council tax, TV licence, transaction taxes, etc.), so the broader Python variable was producing a spurious £159 diff on every synthetic scenario for the TV licence alone. With the correct comparison, single/couple scenarios now show £1–£2 diffs (employer-NI rounding only), making real gaps in lone-parent UC and pensioner Winter Fuel Allowance visible. diff --git a/interfaces/python/tests/test_parity_harness.py b/interfaces/python/tests/test_parity_harness.py index f8fe51c..4ae01eb 100644 --- a/interfaces/python/tests/test_parity_harness.py +++ b/interfaces/python/tests/test_parity_harness.py @@ -142,7 +142,7 @@ def test_run_rust_returns_expected_keys_for_one_scenario(self): } out = parity.run_rust(situation, year=2025) assert "income_tax" in out - assert "household_net_income" in out + assert "hbai_household_net_income" in out # £50k single in 2025: income tax should land in the £7k–8k range. assert 7_000 < out["income_tax"] < 8_000 diff --git a/scripts/parity.py b/scripts/parity.py index 1599183..077560a 100644 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -47,16 +47,22 @@ class Variable: rust_column: str # column in that table +# Compare against `hbai_household_net_income` rather than the broader +# `household_net_income`: Rust's `baseline_net_income` is the HBAI definition +# (gross minus direct taxes plus benefits, excluding council tax / TV licence / +# transaction taxes), so comparing against the broad Python net-income variable +# would surface a spurious ~£159 diff on every scenario for the TV licence +# alone. VARIABLES: list[Variable] = [ - Variable("income_tax", "persons", "baseline_income_tax"), - Variable("ni_employee", "persons", "baseline_employee_ni"), - Variable("ni_employer", "persons", "baseline_employer_ni"), - Variable("universal_credit", "benunits", "baseline_universal_credit"), - Variable("child_benefit", "benunits", "baseline_child_benefit"), - Variable("state_pension", "benunits", "baseline_state_pension"), - Variable("pension_credit", "benunits", "baseline_pension_credit"), - Variable("housing_benefit", "benunits", "baseline_housing_benefit"), - Variable("household_net_income", "households", "baseline_net_income"), + Variable("income_tax", "persons", "baseline_income_tax"), + Variable("ni_employee", "persons", "baseline_employee_ni"), + Variable("ni_employer", "persons", "baseline_employer_ni"), + Variable("universal_credit", "benunits", "baseline_universal_credit"), + Variable("child_benefit", "benunits", "baseline_child_benefit"), + Variable("state_pension", "benunits", "baseline_state_pension"), + Variable("pension_credit", "benunits", "baseline_pension_credit"), + Variable("housing_benefit", "benunits", "baseline_housing_benefit"), + Variable("hbai_household_net_income", "households", "baseline_net_income"), ]