From 51b4a17b3739cd330d753974ea83a1b078f883a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 3 Feb 2026 00:55:19 +0300 Subject: [PATCH 01/87] feat: First household CRUD setup --- src/policyengine_api/api/__init__.py | 2 + src/policyengine_api/api/households.py | 119 ++++++++++++++ src/policyengine_api/models/__init__.py | 4 + src/policyengine_api/models/household.py | 54 ++++++ .../20260203000000_create_households.sql | 14 ++ test_fixtures/fixtures_households.py | 66 ++++++++ tests/test_households.py | 155 ++++++++++++++++++ 7 files changed, 414 insertions(+) create mode 100644 src/policyengine_api/api/households.py create mode 100644 src/policyengine_api/models/household.py create mode 100644 supabase/migrations/20260203000000_create_households.sql create mode 100644 test_fixtures/fixtures_households.py create mode 100644 tests/test_households.py diff --git a/src/policyengine_api/api/__init__.py b/src/policyengine_api/api/__init__.py index 881af99..e688814 100644 --- a/src/policyengine_api/api/__init__.py +++ b/src/policyengine_api/api/__init__.py @@ -9,6 +9,7 @@ datasets, dynamics, household, + households, outputs, parameter_values, parameters, @@ -33,6 +34,7 @@ api_router.include_router(tax_benefit_model_versions.router) api_router.include_router(change_aggregates.router) api_router.include_router(household.router) +api_router.include_router(households.router) api_router.include_router(analysis.router) api_router.include_router(agent.router) diff --git a/src/policyengine_api/api/households.py b/src/policyengine_api/api/households.py new file mode 100644 index 0000000..fdee1f7 --- /dev/null +++ b/src/policyengine_api/api/households.py @@ -0,0 +1,119 @@ +"""Stored household CRUD endpoints. + +Households represent saved household definitions that can be reused across +calculations and impact analyses. Create a household once, then reference +it by ID for repeated simulations. + +These endpoints manage stored household *definitions* (people, entity groups, +model name, year). For running calculations on a household, use the +/household/calculate and /household/impact endpoints instead. +""" + +from typing import Any +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlmodel import Session, select + +from policyengine_api.models import Household, HouseholdCreate, HouseholdRead +from policyengine_api.services.database import get_session + +router = APIRouter(prefix="/households", tags=["households"]) + +_ENTITY_GROUP_KEYS = ( + "tax_unit", + "family", + "spm_unit", + "marital_unit", + "household", + "benunit", +) + + +def _pack_household_data(body: HouseholdCreate) -> dict[str, Any]: + """Pack the flat request fields into a single JSON blob for storage.""" + data: dict[str, Any] = {"people": body.people} + for key in _ENTITY_GROUP_KEYS: + val = getattr(body, key) + if val is not None: + data[key] = val + return data + + +def _to_read(record: Household) -> HouseholdRead: + """Unpack the JSON blob back into the flat response shape.""" + data = record.household_data + return HouseholdRead( + id=record.id, + tax_benefit_model_name=record.tax_benefit_model_name, + year=record.year, + label=record.label, + people=data["people"], + tax_unit=data.get("tax_unit"), + family=data.get("family"), + spm_unit=data.get("spm_unit"), + marital_unit=data.get("marital_unit"), + household=data.get("household"), + benunit=data.get("benunit"), + created_at=record.created_at, + updated_at=record.updated_at, + ) + + +@router.post("/", response_model=HouseholdRead, status_code=201) +def create_household(body: HouseholdCreate, session: Session = Depends(get_session)): + """Create a stored household definition. + + The household data (people + entity groups) is persisted so it can be + retrieved later by ID. Use the returned ID with /household/calculate + or /household/impact to run simulations. + """ + record = Household( + tax_benefit_model_name=body.tax_benefit_model_name, + year=body.year, + label=body.label, + household_data=_pack_household_data(body), + ) + session.add(record) + session.commit() + session.refresh(record) + return _to_read(record) + + +@router.get("/", response_model=list[HouseholdRead]) +def list_households( + tax_benefit_model_name: str | None = None, + limit: int = Query(default=50, le=200), + offset: int = Query(default=0, ge=0), + session: Session = Depends(get_session), +): + """List stored households with optional filtering.""" + query = select(Household) + if tax_benefit_model_name is not None: + query = query.where(Household.tax_benefit_model_name == tax_benefit_model_name) + query = query.offset(offset).limit(limit) + records = session.exec(query).all() + return [_to_read(r) for r in records] + + +@router.get("/{household_id}", response_model=HouseholdRead) +def get_household(household_id: UUID, session: Session = Depends(get_session)): + """Get a stored household by ID.""" + record = session.get(Household, household_id) + if not record: + raise HTTPException( + status_code=404, detail=f"Household {household_id} not found" + ) + return _to_read(record) + + +@router.delete("/{household_id}", status_code=204) +def delete_household(household_id: UUID, session: Session = Depends(get_session)): + """Delete a stored household.""" + record = session.get(Household, household_id) + if not record: + raise HTTPException( + status_code=404, detail=f"Household {household_id} not found" + ) + session.delete(record) + session.commit() diff --git a/src/policyengine_api/models/__init__.py b/src/policyengine_api/models/__init__.py index 4d64c02..7e76baa 100644 --- a/src/policyengine_api/models/__init__.py +++ b/src/policyengine_api/models/__init__.py @@ -11,6 +11,7 @@ from .dataset_version import DatasetVersion, DatasetVersionCreate, DatasetVersionRead from .decile_impact import DecileImpact, DecileImpactCreate, DecileImpactRead from .dynamic import Dynamic, DynamicCreate, DynamicRead +from .household import Household, HouseholdCreate, HouseholdRead from .household_job import ( HouseholdJob, HouseholdJobCreate, @@ -72,6 +73,9 @@ "Dynamic", "DynamicCreate", "DynamicRead", + "Household", + "HouseholdCreate", + "HouseholdRead", "HouseholdJob", "HouseholdJobCreate", "HouseholdJobRead", diff --git a/src/policyengine_api/models/household.py b/src/policyengine_api/models/household.py new file mode 100644 index 0000000..8a96850 --- /dev/null +++ b/src/policyengine_api/models/household.py @@ -0,0 +1,54 @@ +"""Stored household definition model.""" + +from datetime import datetime, timezone +from typing import Any, Literal +from uuid import UUID, uuid4 + +from sqlalchemy import JSON +from sqlmodel import Column, Field, SQLModel + + +class HouseholdBase(SQLModel): + """Base household fields.""" + + tax_benefit_model_name: str + year: int + label: str | None = None + household_data: dict[str, Any] = Field(sa_column=Column(JSON, nullable=False)) + + +class Household(HouseholdBase, table=True): + """Stored household database model.""" + + __tablename__ = "households" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class HouseholdCreate(SQLModel): + """Schema for creating a stored household. + + Accepts the flat structure matching the frontend Household interface: + people as an array, entity groups as optional dicts. + """ + + tax_benefit_model_name: Literal["policyengine_us", "policyengine_uk"] + year: int + label: str | None = None + people: list[dict[str, Any]] + tax_unit: dict[str, Any] | None = None + family: dict[str, Any] | None = None + spm_unit: dict[str, Any] | None = None + marital_unit: dict[str, Any] | None = None + household: dict[str, Any] | None = None + benunit: dict[str, Any] | None = None + + +class HouseholdRead(HouseholdCreate): + """Schema for reading a stored household.""" + + id: UUID + created_at: datetime + updated_at: datetime diff --git a/supabase/migrations/20260203000000_create_households.sql b/supabase/migrations/20260203000000_create_households.sql new file mode 100644 index 0000000..cc1907f --- /dev/null +++ b/supabase/migrations/20260203000000_create_households.sql @@ -0,0 +1,14 @@ +-- Create stored households table for persisting household definitions. + +CREATE TABLE IF NOT EXISTS households ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tax_benefit_model_name TEXT NOT NULL, + year INTEGER NOT NULL, + label TEXT, + household_data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_households_model_name ON households (tax_benefit_model_name); +CREATE INDEX idx_households_year ON households (year); diff --git a/test_fixtures/fixtures_households.py b/test_fixtures/fixtures_households.py new file mode 100644 index 0000000..4e676f4 --- /dev/null +++ b/test_fixtures/fixtures_households.py @@ -0,0 +1,66 @@ +"""Fixtures and helpers for household CRUD tests.""" + +from policyengine_api.models import Household + +# ----------------------------------------------------------------------------- +# Request payloads (match HouseholdCreate schema) +# ----------------------------------------------------------------------------- + +MOCK_US_HOUSEHOLD_CREATE = { + "tax_benefit_model_name": "policyengine_us", + "year": 2024, + "label": "US test household", + "people": [ + {"age": 30, "employment_income": 50000}, + {"age": 28, "employment_income": 30000}, + ], + "tax_unit": {}, + "family": {}, + "household": {"state_name": "CA"}, +} + +MOCK_UK_HOUSEHOLD_CREATE = { + "tax_benefit_model_name": "policyengine_uk", + "year": 2024, + "label": "UK test household", + "people": [ + {"age": 40, "employment_income": 35000}, + ], + "benunit": {"is_married": False}, + "household": {"region": "LONDON"}, +} + +MOCK_HOUSEHOLD_MINIMAL = { + "tax_benefit_model_name": "policyengine_us", + "year": 2024, + "people": [{"age": 25}], +} + + +# ----------------------------------------------------------------------------- +# Factory functions +# ----------------------------------------------------------------------------- + + +def create_household( + session, + tax_benefit_model_name: str = "policyengine_us", + year: int = 2024, + label: str | None = "Test household", + people: list | None = None, + **entity_groups, +) -> Household: + """Create and persist a Household record.""" + household_data = {"people": people or [{"age": 30}]} + household_data.update(entity_groups) + + record = Household( + tax_benefit_model_name=tax_benefit_model_name, + year=year, + label=label, + household_data=household_data, + ) + session.add(record) + session.commit() + session.refresh(record) + return record diff --git a/tests/test_households.py b/tests/test_households.py new file mode 100644 index 0000000..4c60062 --- /dev/null +++ b/tests/test_households.py @@ -0,0 +1,155 @@ +"""Tests for stored household CRUD endpoints.""" + +from uuid import uuid4 + +from test_fixtures.fixtures_households import ( + MOCK_HOUSEHOLD_MINIMAL, + MOCK_UK_HOUSEHOLD_CREATE, + MOCK_US_HOUSEHOLD_CREATE, + create_household, +) + +# --------------------------------------------------------------------------- +# POST /households +# --------------------------------------------------------------------------- + + +def test_create_us_household(client): + """Create a US household returns 201 with id and timestamps.""" + response = client.post("/households", json=MOCK_US_HOUSEHOLD_CREATE) + assert response.status_code == 201 + data = response.json() + assert "id" in data + assert "created_at" in data + assert "updated_at" in data + assert data["tax_benefit_model_name"] == "policyengine_us" + assert data["year"] == 2024 + assert data["label"] == "US test household" + + +def test_create_household_returns_people_and_entities(client): + """Created household response includes people and entity groups.""" + response = client.post("/households", json=MOCK_US_HOUSEHOLD_CREATE) + data = response.json() + assert len(data["people"]) == 2 + assert data["people"][0]["age"] == 30 + assert data["people"][0]["employment_income"] == 50000 + assert data["household"] == {"state_name": "CA"} + assert data["tax_unit"] == {} + assert data["family"] == {} + + +def test_create_uk_household(client): + """Create a UK household with benunit.""" + response = client.post("/households", json=MOCK_UK_HOUSEHOLD_CREATE) + assert response.status_code == 201 + data = response.json() + assert data["tax_benefit_model_name"] == "policyengine_uk" + assert data["benunit"] == {"is_married": False} + assert data["household"] == {"region": "LONDON"} + + +def test_create_household_minimal(client): + """Create a household with minimal fields.""" + response = client.post("/households", json=MOCK_HOUSEHOLD_MINIMAL) + assert response.status_code == 201 + data = response.json() + assert data["label"] is None + assert data["tax_unit"] is None + assert data["benunit"] is None + + +def test_create_household_invalid_model_name(client): + """Reject invalid tax_benefit_model_name.""" + payload = {**MOCK_HOUSEHOLD_MINIMAL, "tax_benefit_model_name": "invalid"} + response = client.post("/households", json=payload) + assert response.status_code == 422 + + +# --------------------------------------------------------------------------- +# GET /households/{id} +# --------------------------------------------------------------------------- + + +def test_get_household(client, session): + """Get a stored household by ID.""" + record = create_household(session) + response = client.get(f"/households/{record.id}") + assert response.status_code == 200 + data = response.json() + assert data["id"] == str(record.id) + assert data["tax_benefit_model_name"] == "policyengine_us" + + +def test_get_household_not_found(client): + """Get a non-existent household returns 404.""" + fake_id = uuid4() + response = client.get(f"/households/{fake_id}") + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + +# --------------------------------------------------------------------------- +# GET /households +# --------------------------------------------------------------------------- + + +def test_list_households_empty(client): + """List households returns empty list when none exist.""" + response = client.get("/households") + assert response.status_code == 200 + assert response.json() == [] + + +def test_list_households_with_data(client, session): + """List households returns all stored households.""" + create_household(session, label="first") + create_household(session, label="second") + response = client.get("/households") + assert response.status_code == 200 + data = response.json() + assert len(data) == 2 + + +def test_list_households_filter_by_model_name(client, session): + """Filter households by tax_benefit_model_name.""" + create_household(session, tax_benefit_model_name="policyengine_us") + create_household(session, tax_benefit_model_name="policyengine_uk") + response = client.get( + "/households", params={"tax_benefit_model_name": "policyengine_uk"} + ) + data = response.json() + assert len(data) == 1 + assert data[0]["tax_benefit_model_name"] == "policyengine_uk" + + +def test_list_households_limit_and_offset(client, session): + """Respect limit and offset pagination.""" + for i in range(5): + create_household(session, label=f"household-{i}") + response = client.get("/households", params={"limit": 2, "offset": 1}) + data = response.json() + assert len(data) == 2 + + +# --------------------------------------------------------------------------- +# DELETE /households/{id} +# --------------------------------------------------------------------------- + + +def test_delete_household(client, session): + """Delete a household returns 204.""" + record = create_household(session) + response = client.delete(f"/households/{record.id}") + assert response.status_code == 204 + + # Confirm it's gone + response = client.get(f"/households/{record.id}") + assert response.status_code == 404 + + +def test_delete_household_not_found(client): + """Delete a non-existent household returns 404.""" + fake_id = uuid4() + response = client.delete(f"/households/{fake_id}") + assert response.status_code == 404 From 58cd69158770dc12ad844fc8dc5f986d0ae74cf0 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 3 Feb 2026 01:16:25 +0300 Subject: [PATCH 02/87] feat: User-household associations --- src/policyengine_api/api/__init__.py | 2 + .../api/user_household_associations.py | 125 ++++++++++++ src/policyengine_api/models/__init__.py | 10 + .../models/user_household_association.py | 48 +++++ ...001_create_user_household_associations.sql | 14 ++ .../fixtures_user_household_associations.py | 62 ++++++ tests/test_user_household_associations.py | 189 ++++++++++++++++++ 7 files changed, 450 insertions(+) create mode 100644 src/policyengine_api/api/user_household_associations.py create mode 100644 src/policyengine_api/models/user_household_association.py create mode 100644 supabase/migrations/20260203000001_create_user_household_associations.sql create mode 100644 test_fixtures/fixtures_user_household_associations.py create mode 100644 tests/test_user_household_associations.py diff --git a/src/policyengine_api/api/__init__.py b/src/policyengine_api/api/__init__.py index e688814..92f5ea5 100644 --- a/src/policyengine_api/api/__init__.py +++ b/src/policyengine_api/api/__init__.py @@ -17,6 +17,7 @@ simulations, tax_benefit_model_versions, tax_benefit_models, + user_household_associations, variables, ) @@ -37,5 +38,6 @@ api_router.include_router(households.router) api_router.include_router(analysis.router) api_router.include_router(agent.router) +api_router.include_router(user_household_associations.router) __all__ = ["api_router"] diff --git a/src/policyengine_api/api/user_household_associations.py b/src/policyengine_api/api/user_household_associations.py new file mode 100644 index 0000000..fa40e06 --- /dev/null +++ b/src/policyengine_api/api/user_household_associations.py @@ -0,0 +1,125 @@ +"""User-household association endpoints. + +Associations link a user to a stored household definition with metadata +(label, country). A user can have multiple associations to the same +household (e.g. different labels or configurations). +""" + +from datetime import datetime, timezone +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlmodel import Session, select + +from policyengine_api.models import ( + Household, + UserHouseholdAssociation, + UserHouseholdAssociationCreate, + UserHouseholdAssociationRead, + UserHouseholdAssociationUpdate, +) +from policyengine_api.services.database import get_session + +router = APIRouter( + prefix="/user-household-associations", + tags=["user-household-associations"], +) + + +@router.post("/", response_model=UserHouseholdAssociationRead, status_code=201) +def create_association( + body: UserHouseholdAssociationCreate, + session: Session = Depends(get_session), +): + """Create a user-household association.""" + household = session.get(Household, body.household_id) + if not household: + raise HTTPException( + status_code=404, + detail=f"Household {body.household_id} not found", + ) + + record = UserHouseholdAssociation( + user_id=body.user_id, + household_id=body.household_id, + country_id=body.country_id, + label=body.label, + ) + session.add(record) + session.commit() + session.refresh(record) + return record + + +@router.get("/user/{user_id}", response_model=list[UserHouseholdAssociationRead]) +def list_by_user( + user_id: UUID, + country_id: str | None = None, + limit: int = Query(default=50, le=200), + offset: int = Query(default=0, ge=0), + session: Session = Depends(get_session), +): + """List all associations for a user, optionally filtered by country.""" + query = select(UserHouseholdAssociation).where( + UserHouseholdAssociation.user_id == user_id + ) + if country_id is not None: + query = query.where(UserHouseholdAssociation.country_id == country_id) + query = query.offset(offset).limit(limit) + return session.exec(query).all() + + +@router.get( + "/{user_id}/{household_id}", + response_model=list[UserHouseholdAssociationRead], +) +def list_by_user_and_household( + user_id: UUID, + household_id: UUID, + session: Session = Depends(get_session), +): + """List all associations for a specific user+household pair.""" + query = select(UserHouseholdAssociation).where( + UserHouseholdAssociation.user_id == user_id, + UserHouseholdAssociation.household_id == household_id, + ) + return session.exec(query).all() + + +@router.put("/{association_id}", response_model=UserHouseholdAssociationRead) +def update_association( + association_id: UUID, + body: UserHouseholdAssociationUpdate, + session: Session = Depends(get_session), +): + """Update a user-household association (label).""" + record = session.get(UserHouseholdAssociation, association_id) + if not record: + raise HTTPException( + status_code=404, + detail=f"Association {association_id} not found", + ) + update_data = body.model_dump(exclude_unset=True) + for key, value in update_data.items(): + setattr(record, key, value) + record.updated_at = datetime.now(timezone.utc) + session.add(record) + session.commit() + session.refresh(record) + return record + + +@router.delete("/{association_id}", status_code=204) +def delete_association( + association_id: UUID, + session: Session = Depends(get_session), +): + """Delete a user-household association.""" + record = session.get(UserHouseholdAssociation, association_id) + if not record: + raise HTTPException( + status_code=404, + detail=f"Association {association_id} not found", + ) + session.delete(record) + session.commit() diff --git a/src/policyengine_api/models/__init__.py b/src/policyengine_api/models/__init__.py index 7e76baa..546c538 100644 --- a/src/policyengine_api/models/__init__.py +++ b/src/policyengine_api/models/__init__.py @@ -48,6 +48,12 @@ TaxBenefitModelVersionRead, ) from .user import User, UserCreate, UserRead +from .user_household_association import ( + UserHouseholdAssociation, + UserHouseholdAssociationCreate, + UserHouseholdAssociationRead, + UserHouseholdAssociationUpdate, +) from .variable import Variable, VariableCreate, VariableRead __all__ = [ @@ -114,6 +120,10 @@ "TaxBenefitModelVersionRead", "User", "UserCreate", + "UserHouseholdAssociation", + "UserHouseholdAssociationCreate", + "UserHouseholdAssociationRead", + "UserHouseholdAssociationUpdate", "UserRead", "Variable", "VariableCreate", diff --git a/src/policyengine_api/models/user_household_association.py b/src/policyengine_api/models/user_household_association.py new file mode 100644 index 0000000..208279a --- /dev/null +++ b/src/policyengine_api/models/user_household_association.py @@ -0,0 +1,48 @@ +"""User-household association model.""" + +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlmodel import Field, SQLModel + + +class UserHouseholdAssociationBase(SQLModel): + """Base association fields.""" + + user_id: UUID = Field(foreign_key="users.id", index=True) + household_id: UUID = Field(foreign_key="households.id", index=True) + country_id: str + label: str | None = None + + +class UserHouseholdAssociation(UserHouseholdAssociationBase, table=True): + """User-household association database model.""" + + __tablename__ = "user_household_associations" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class UserHouseholdAssociationCreate(SQLModel): + """Schema for creating a user-household association.""" + + user_id: UUID + household_id: UUID + country_id: str + label: str | None = None + + +class UserHouseholdAssociationUpdate(SQLModel): + """Schema for updating a user-household association.""" + + label: str | None = None + + +class UserHouseholdAssociationRead(UserHouseholdAssociationBase): + """Schema for reading a user-household association.""" + + id: UUID + created_at: datetime + updated_at: datetime diff --git a/supabase/migrations/20260203000001_create_user_household_associations.sql b/supabase/migrations/20260203000001_create_user_household_associations.sql new file mode 100644 index 0000000..3fdcb03 --- /dev/null +++ b/supabase/migrations/20260203000001_create_user_household_associations.sql @@ -0,0 +1,14 @@ +-- Create user-household associations table for linking users to saved households. + +CREATE TABLE IF NOT EXISTS user_household_associations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + household_id UUID NOT NULL REFERENCES households(id) ON DELETE CASCADE, + country_id TEXT NOT NULL, + label TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_user_household_assoc_user ON user_household_associations (user_id); +CREATE INDEX idx_user_household_assoc_household ON user_household_associations (household_id); diff --git a/test_fixtures/fixtures_user_household_associations.py b/test_fixtures/fixtures_user_household_associations.py new file mode 100644 index 0000000..66b0835 --- /dev/null +++ b/test_fixtures/fixtures_user_household_associations.py @@ -0,0 +1,62 @@ +"""Fixtures and helpers for user-household association tests.""" + +from uuid import UUID + +from policyengine_api.models import Household, User, UserHouseholdAssociation + +# ----------------------------------------------------------------------------- +# Factory functions +# ----------------------------------------------------------------------------- + + +def create_user( + session, + first_name: str = "Test", + last_name: str = "User", + email: str = "test@example.com", +) -> User: + """Create and persist a User record.""" + record = User(first_name=first_name, last_name=last_name, email=email) + session.add(record) + session.commit() + session.refresh(record) + return record + + +def create_household( + session, + tax_benefit_model_name: str = "policyengine_us", + year: int = 2024, + label: str | None = "Test household", +) -> Household: + """Create and persist a Household record.""" + record = Household( + tax_benefit_model_name=tax_benefit_model_name, + year=year, + label=label, + household_data={"people": [{"age": 30}]}, + ) + session.add(record) + session.commit() + session.refresh(record) + return record + + +def create_association( + session, + user_id: UUID, + household_id: UUID, + country_id: str = "us", + label: str | None = "My household", +) -> UserHouseholdAssociation: + """Create and persist a UserHouseholdAssociation record.""" + record = UserHouseholdAssociation( + user_id=user_id, + household_id=household_id, + country_id=country_id, + label=label, + ) + session.add(record) + session.commit() + session.refresh(record) + return record diff --git a/tests/test_user_household_associations.py b/tests/test_user_household_associations.py new file mode 100644 index 0000000..25d8989 --- /dev/null +++ b/tests/test_user_household_associations.py @@ -0,0 +1,189 @@ +"""Tests for user-household association endpoints.""" + +from uuid import uuid4 + +from test_fixtures.fixtures_user_household_associations import ( + create_association, + create_household, + create_user, +) + +# --------------------------------------------------------------------------- +# POST /user-household-associations +# --------------------------------------------------------------------------- + + +def test_create_association(client, session): + """Create an association returns 201 with id and timestamps.""" + user = create_user(session) + household = create_household(session) + payload = { + "user_id": str(user.id), + "household_id": str(household.id), + "country_id": "us", + "label": "My US household", + } + response = client.post("/user-household-associations", json=payload) + assert response.status_code == 201 + data = response.json() + assert "id" in data + assert "created_at" in data + assert "updated_at" in data + assert data["user_id"] == str(user.id) + assert data["household_id"] == str(household.id) + assert data["country_id"] == "us" + assert data["label"] == "My US household" + + +def test_create_association_allows_duplicates(client, session): + """Multiple associations to the same household are allowed.""" + user = create_user(session) + household = create_household(session) + payload = { + "user_id": str(user.id), + "household_id": str(household.id), + "country_id": "us", + "label": "First label", + } + r1 = client.post("/user-household-associations", json=payload) + assert r1.status_code == 201 + + payload["label"] = "Second label" + r2 = client.post("/user-household-associations", json=payload) + assert r2.status_code == 201 + assert r1.json()["id"] != r2.json()["id"] + + +def test_create_association_household_not_found(client, session): + """Creating with a non-existent household returns 404.""" + user = create_user(session) + payload = { + "user_id": str(user.id), + "household_id": str(uuid4()), + "country_id": "us", + } + response = client.post("/user-household-associations", json=payload) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + +# --------------------------------------------------------------------------- +# GET /user-household-associations/user/{user_id} +# --------------------------------------------------------------------------- + + +def test_list_by_user_empty(client): + """List associations for a user with none returns empty list.""" + response = client.get(f"/user-household-associations/user/{uuid4()}") + assert response.status_code == 200 + assert response.json() == [] + + +def test_list_by_user(client, session): + """List all associations for a user.""" + user = create_user(session) + h1 = create_household(session, label="H1") + h2 = create_household(session, label="H2") + create_association(session, user.id, h1.id, label="First") + create_association(session, user.id, h2.id, label="Second") + + response = client.get(f"/user-household-associations/user/{user.id}") + assert response.status_code == 200 + data = response.json() + assert len(data) == 2 + + +def test_list_by_user_filter_country(client, session): + """Filter associations by country_id.""" + user = create_user(session) + household = create_household(session) + create_association(session, user.id, household.id, country_id="us") + create_association(session, user.id, household.id, country_id="uk") + + response = client.get( + f"/user-household-associations/user/{user.id}", + params={"country_id": "uk"}, + ) + data = response.json() + assert len(data) == 1 + assert data[0]["country_id"] == "uk" + + +# --------------------------------------------------------------------------- +# GET /user-household-associations/{user_id}/{household_id} +# --------------------------------------------------------------------------- + + +def test_list_by_user_and_household(client, session): + """List associations for a specific user+household pair.""" + user = create_user(session) + household = create_household(session) + create_association(session, user.id, household.id, label="Label A") + create_association(session, user.id, household.id, label="Label B") + + response = client.get(f"/user-household-associations/{user.id}/{household.id}") + assert response.status_code == 200 + data = response.json() + assert len(data) == 2 + + +def test_list_by_user_and_household_empty(client): + """Returns empty list when no associations exist for the pair.""" + response = client.get(f"/user-household-associations/{uuid4()}/{uuid4()}") + assert response.status_code == 200 + assert response.json() == [] + + +# --------------------------------------------------------------------------- +# PUT /user-household-associations/{association_id} +# --------------------------------------------------------------------------- + + +def test_update_association_label(client, session): + """Update label and verify updated_at changes.""" + user = create_user(session) + household = create_household(session) + assoc = create_association(session, user.id, household.id, label="Old") + + response = client.put( + f"/user-household-associations/{assoc.id}", + json={"label": "New label"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["label"] == "New label" + + +def test_update_association_not_found(client): + """Update a non-existent association returns 404.""" + response = client.put( + f"/user-household-associations/{uuid4()}", + json={"label": "Something"}, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + +# --------------------------------------------------------------------------- +# DELETE /user-household-associations/{association_id} +# --------------------------------------------------------------------------- + + +def test_delete_association(client, session): + """Delete an association returns 204.""" + user = create_user(session) + household = create_household(session) + assoc = create_association(session, user.id, household.id) + + response = client.delete(f"/user-household-associations/{assoc.id}") + assert response.status_code == 204 + + # Confirm it's gone + response = client.get(f"/user-household-associations/{user.id}/{household.id}") + assert response.json() == [] + + +def test_delete_association_not_found(client): + """Delete a non-existent association returns 404.""" + response = client.delete(f"/user-household-associations/{uuid4()}") + assert response.status_code == 404 From 77c5a3db89d096c73a3e283f83dfaaf897185baa Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 3 Feb 2026 22:13:16 +0300 Subject: [PATCH 03/87] feat: Household analysis --- src/policyengine_api/api/analysis.py | 400 +++++++++++++++++- src/policyengine_api/models/__init__.py | 9 +- src/policyengine_api/models/report.py | 1 + src/policyengine_api/models/simulation.py | 28 +- ...203000002_simulation_household_support.sql | 16 + test_fixtures/fixtures_analysis.py | 164 +++++++ tests/test_analysis_household_impact.py | 297 +++++++++++++ 7 files changed, 901 insertions(+), 14 deletions(-) create mode 100644 supabase/migrations/20260203000002_simulation_household_support.sql create mode 100644 test_fixtures/fixtures_analysis.py create mode 100644 tests/test_analysis_household_impact.py diff --git a/src/policyengine_api/api/analysis.py b/src/policyengine_api/api/analysis.py index c9aa86d..b1ab584 100644 --- a/src/policyengine_api/api/analysis.py +++ b/src/policyengine_api/api/analysis.py @@ -16,7 +16,8 @@ """ import math -from typing import Literal +from datetime import datetime, timezone +from typing import Any, Literal from uuid import UUID, uuid5 import logfire @@ -29,12 +30,15 @@ Dataset, DecileImpact, DecileImpactRead, + Household, + Policy, ProgramStatistics, ProgramStatisticsRead, Report, ReportStatus, Simulation, SimulationStatus, + SimulationType, TaxBenefitModel, TaxBenefitModelVersion, ) @@ -138,19 +142,24 @@ def _get_model_version( def _get_deterministic_simulation_id( - dataset_id: UUID, + simulation_type: SimulationType, model_version_id: UUID, policy_id: UUID | None, dynamic_id: UUID | None, + dataset_id: UUID | None = None, + household_id: UUID | None = None, ) -> UUID: """Generate a deterministic UUID from simulation parameters.""" - key = f"{dataset_id}:{model_version_id}:{policy_id}:{dynamic_id}" + if simulation_type == SimulationType.ECONOMY: + key = f"economy:{dataset_id}:{model_version_id}:{policy_id}:{dynamic_id}" + else: + key = f"household:{household_id}:{model_version_id}:{policy_id}:{dynamic_id}" return uuid5(SIMULATION_NAMESPACE, key) def _get_deterministic_report_id( baseline_sim_id: UUID, - reform_sim_id: UUID, + reform_sim_id: UUID | None, ) -> UUID: """Generate a deterministic UUID from report parameters.""" key = f"{baseline_sim_id}:{reform_sim_id}" @@ -158,15 +167,22 @@ def _get_deterministic_report_id( def _get_or_create_simulation( - dataset_id: UUID, + simulation_type: SimulationType, model_version_id: UUID, policy_id: UUID | None, dynamic_id: UUID | None, session: Session, + dataset_id: UUID | None = None, + household_id: UUID | None = None, ) -> Simulation: """Get existing simulation or create a new one.""" sim_id = _get_deterministic_simulation_id( - dataset_id, model_version_id, policy_id, dynamic_id + simulation_type, + model_version_id, + policy_id, + dynamic_id, + dataset_id=dataset_id, + household_id=household_id, ) existing = session.get(Simulation, sim_id) @@ -175,7 +191,9 @@ def _get_or_create_simulation( simulation = Simulation( id=sim_id, + simulation_type=simulation_type, dataset_id=dataset_id, + household_id=household_id, tax_benefit_model_version_id=model_version_id, policy_id=policy_id, dynamic_id=dynamic_id, @@ -189,8 +207,9 @@ def _get_or_create_simulation( def _get_or_create_report( baseline_sim_id: UUID, - reform_sim_id: UUID, + reform_sim_id: UUID | None, label: str, + report_type: str, session: Session, ) -> Report: """Get existing report or create a new one.""" @@ -203,6 +222,7 @@ def _get_or_create_report( report = Report( id=report_id, label=label, + report_type=report_type, baseline_simulation_id=baseline_sim_id, reform_simulation_id=reform_sim_id, status=ReportStatus.PENDING, @@ -554,6 +574,362 @@ def _trigger_economy_comparison( fn.spawn(job_id=job_id, traceparent=traceparent) +# Entity types by country +UK_ENTITIES = ["person", "benunit", "household"] +US_ENTITIES = ["person", "tax_unit", "spm_unit", "family", "marital_unit", "household"] + + +def _compute_entity_diff( + baseline_list: list[dict], + reform_list: list[dict], +) -> list[dict]: + """Compute per-variable diffs for a list of entity instances.""" + entity_impact = [] + + for b_entity, r_entity in zip(baseline_list, reform_list): + entity_diff = {} + for key in b_entity: + if key in r_entity: + baseline_val = b_entity[key] + reform_val = r_entity[key] + if isinstance(baseline_val, (int, float)) and isinstance( + reform_val, (int, float) + ): + entity_diff[key] = { + "baseline": baseline_val, + "reform": reform_val, + "change": reform_val - baseline_val, + } + entity_impact.append(entity_diff) + + return entity_impact + + +def _compute_household_impact( + baseline_result: dict, + reform_result: dict, + country: str, +) -> dict[str, Any]: + """Compute difference between baseline and reform for all entity types.""" + entities = UK_ENTITIES if country == "uk" else US_ENTITIES + + impact: dict[str, Any] = {} + + for entity in entities: + if entity in baseline_result and entity in reform_result: + impact[entity] = _compute_entity_diff( + baseline_result[entity], + reform_result[entity], + ) + + return impact + + +def _ensure_list(value: Any) -> list: + """Ensure value is a list; wrap dict in list if needed.""" + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _run_household_simulation(simulation_id: UUID, session: Session) -> None: + """Run a single household simulation and store result.""" + from policyengine_api.api.household import ( + _calculate_household_uk, + _calculate_household_us, + ) + + simulation = session.get(Simulation, simulation_id) + if not simulation: + raise ValueError(f"Simulation {simulation_id} not found") + + household = session.get(Household, simulation.household_id) + if not household: + raise ValueError(f"Household {simulation.household_id} not found") + + # Update status + simulation.status = SimulationStatus.RUNNING + simulation.started_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + try: + # Get policy if set + policy_data = None + if simulation.policy_id: + policy = session.get(Policy, simulation.policy_id) + if policy and policy.parameter_values: + policy_data = {} + for pv in policy.parameter_values: + if pv.parameter: + param_name = pv.parameter.name + policy_data[param_name] = { + "value": pv.value_json.get("value") + if isinstance(pv.value_json, dict) + else pv.value_json, + "start_date": pv.start_date.isoformat() + if pv.start_date + else None, + "end_date": pv.end_date.isoformat() + if pv.end_date + else None, + } + + # Extract household data with list conversion + data = household.household_data + people = data.get("people", []) + + # Run calculation based on country + if household.tax_benefit_model_name == "policyengine_uk": + result = _calculate_household_uk( + people=people, + benunit=_ensure_list(data.get("benunit")), + household=_ensure_list(data.get("household")), + year=household.year, + policy_data=policy_data, + ) + else: + result = _calculate_household_us( + people=people, + marital_unit=_ensure_list(data.get("marital_unit")), + family=_ensure_list(data.get("family")), + spm_unit=_ensure_list(data.get("spm_unit")), + tax_unit=_ensure_list(data.get("tax_unit")), + household=_ensure_list(data.get("household")), + year=household.year, + policy_data=policy_data, + ) + + # Store result + simulation.household_result = result + simulation.status = SimulationStatus.COMPLETED + simulation.completed_at = datetime.now(timezone.utc) + + except Exception as e: + simulation.status = SimulationStatus.FAILED + simulation.error_message = str(e) + simulation.completed_at = datetime.now(timezone.utc) + + session.add(simulation) + session.commit() + + +def _trigger_household_report(report_id: UUID, session: Session) -> None: + """Trigger household simulation(s) for a report.""" + report = session.get(Report, report_id) + if not report: + raise ValueError(f"Report {report_id} not found") + + # Update report status + report.status = ReportStatus.RUNNING + session.add(report) + session.commit() + + try: + # Run baseline + baseline_sim = session.get(Simulation, report.baseline_simulation_id) + if baseline_sim and baseline_sim.status == SimulationStatus.PENDING: + _run_household_simulation(baseline_sim.id, session) + + # Run reform if exists + if report.reform_simulation_id: + reform_sim = session.get(Simulation, report.reform_simulation_id) + if reform_sim and reform_sim.status == SimulationStatus.PENDING: + _run_household_simulation(reform_sim.id, session) + + # Update report status + report.status = ReportStatus.COMPLETED + except Exception as e: + report.status = ReportStatus.FAILED + report.error_message = str(e) + + session.add(report) + session.commit() + + +# Household impact request/response schemas +class HouseholdImpactRequest(BaseModel): + """Request for household impact analysis.""" + + household_id: UUID = Field(description="ID of the household to analyze") + policy_id: UUID | None = Field( + default=None, + description="Reform policy ID. If None, runs single calculation under current law.", + ) + dynamic_id: UUID | None = Field( + default=None, description="Optional behavioural response specification ID" + ) + + +class HouseholdSimulationInfo(BaseModel): + """Info about a household simulation.""" + + id: UUID + status: SimulationStatus + error_message: str | None = None + + +class HouseholdImpactResponse(BaseModel): + """Response for household impact analysis.""" + + report_id: UUID + report_type: str + status: ReportStatus + baseline_simulation: HouseholdSimulationInfo | None = None + reform_simulation: HouseholdSimulationInfo | None = None + baseline_result: dict | None = None + reform_result: dict | None = None + impact: dict | None = None + error_message: str | None = None + + +def _build_household_response( + report: Report, + baseline_sim: Simulation, + reform_sim: Simulation | None, + session: Session, +) -> HouseholdImpactResponse: + """Build response including computed impact for comparisons.""" + baseline_result = baseline_sim.household_result if baseline_sim else None + reform_result = reform_sim.household_result if reform_sim else None + + # Compute impact if comparison and both complete + impact = None + if reform_sim and baseline_result and reform_result: + # Determine country from household + household = session.get(Household, baseline_sim.household_id) + if household: + country = ( + "uk" if household.tax_benefit_model_name == "policyengine_uk" else "us" + ) + impact = _compute_household_impact(baseline_result, reform_result, country) + + return HouseholdImpactResponse( + report_id=report.id, + report_type=report.report_type or "household_single", + status=report.status, + baseline_simulation=HouseholdSimulationInfo( + id=baseline_sim.id, + status=baseline_sim.status, + error_message=baseline_sim.error_message, + ) + if baseline_sim + else None, + reform_simulation=HouseholdSimulationInfo( + id=reform_sim.id, + status=reform_sim.status, + error_message=reform_sim.error_message, + ) + if reform_sim + else None, + baseline_result=baseline_result, + reform_result=reform_result, + impact=impact, + error_message=report.error_message, + ) + + +@router.post("/household-impact", response_model=HouseholdImpactResponse) +def household_impact( + request: HouseholdImpactRequest, + session: Session = Depends(get_session), +) -> HouseholdImpactResponse: + """Run household impact analysis. + + If policy_id is None: single run under current law. + If policy_id is set: comparison (baseline vs reform). + + This is a synchronous operation for household calculations. + """ + # Validate household exists + household = session.get(Household, request.household_id) + if not household: + raise HTTPException( + status_code=404, detail=f"Household {request.household_id} not found" + ) + + # Validate policy if provided + if request.policy_id: + policy = session.get(Policy, request.policy_id) + if not policy: + raise HTTPException( + status_code=404, detail=f"Policy {request.policy_id} not found" + ) + + # Get model version from household's tax_benefit_model_name + model_version = _get_model_version(household.tax_benefit_model_name, session) + + # Create baseline simulation + baseline_sim = _get_or_create_simulation( + simulation_type=SimulationType.HOUSEHOLD, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=request.dynamic_id, + session=session, + household_id=request.household_id, + ) + + # Create reform simulation if policy_id provided + reform_sim = None + if request.policy_id: + reform_sim = _get_or_create_simulation( + simulation_type=SimulationType.HOUSEHOLD, + model_version_id=model_version.id, + policy_id=request.policy_id, + dynamic_id=request.dynamic_id, + session=session, + household_id=request.household_id, + ) + + # Determine report type + report_type = "household_comparison" if request.policy_id else "household_single" + + # Create report + label = f"Household impact: {household.tax_benefit_model_name}" + report = _get_or_create_report( + baseline_sim_id=baseline_sim.id, + reform_sim_id=reform_sim.id if reform_sim else None, + label=label, + report_type=report_type, + session=session, + ) + + # Trigger compute if pending + if report.status == ReportStatus.PENDING: + with logfire.span("trigger_household_report", job_id=str(report.id)): + _trigger_household_report(report.id, session) + + return _build_household_response(report, baseline_sim, reform_sim, session) + + +@router.get("/household-impact/{report_id}", response_model=HouseholdImpactResponse) +def get_household_impact( + report_id: UUID, + session: Session = Depends(get_session), +) -> HouseholdImpactResponse: + """Get household impact analysis status and results.""" + report = session.get(Report, report_id) + if not report: + raise HTTPException(status_code=404, detail=f"Report {report_id} not found") + + if not report.baseline_simulation_id: + raise HTTPException( + status_code=500, detail="Report missing baseline simulation ID" + ) + + baseline_sim = session.get(Simulation, report.baseline_simulation_id) + if not baseline_sim: + raise HTTPException(status_code=500, detail="Baseline simulation data missing") + + reform_sim = None + if report.reform_simulation_id: + reform_sim = session.get(Simulation, report.reform_simulation_id) + + return _build_household_response(report, baseline_sim, reform_sim, session) + + @router.post("/economic-impact", response_model=EconomicImpactResponse) def economic_impact( request: EconomicImpactRequest, @@ -580,19 +956,21 @@ def economic_impact( # Get or create simulations baseline_sim = _get_or_create_simulation( - dataset_id=request.dataset_id, + simulation_type=SimulationType.ECONOMY, model_version_id=model_version.id, policy_id=None, dynamic_id=request.dynamic_id, session=session, + dataset_id=request.dataset_id, ) reform_sim = _get_or_create_simulation( - dataset_id=request.dataset_id, + simulation_type=SimulationType.ECONOMY, model_version_id=model_version.id, policy_id=request.policy_id, dynamic_id=request.dynamic_id, session=session, + dataset_id=request.dataset_id, ) # Get or create report @@ -600,7 +978,9 @@ def economic_impact( if request.policy_id: label += f" (policy {request.policy_id})" - report = _get_or_create_report(baseline_sim.id, reform_sim.id, label, session) + report = _get_or_create_report( + baseline_sim.id, reform_sim.id, label, "economy_comparison", session + ) # Trigger computation if report is pending if report.status == ReportStatus.PENDING: diff --git a/src/policyengine_api/models/__init__.py b/src/policyengine_api/models/__init__.py index 546c538..c49b457 100644 --- a/src/policyengine_api/models/__init__.py +++ b/src/policyengine_api/models/__init__.py @@ -36,7 +36,13 @@ ProgramStatisticsRead, ) from .report import Report, ReportCreate, ReportRead, ReportStatus -from .simulation import Simulation, SimulationCreate, SimulationRead, SimulationStatus +from .simulation import ( + Simulation, + SimulationCreate, + SimulationRead, + SimulationStatus, + SimulationType, +) from .tax_benefit_model import ( TaxBenefitModel, TaxBenefitModelCreate, @@ -112,6 +118,7 @@ "SimulationCreate", "SimulationRead", "SimulationStatus", + "SimulationType", "TaxBenefitModel", "TaxBenefitModelCreate", "TaxBenefitModelRead", diff --git a/src/policyengine_api/models/report.py b/src/policyengine_api/models/report.py index ee1b678..bc2cd40 100644 --- a/src/policyengine_api/models/report.py +++ b/src/policyengine_api/models/report.py @@ -19,6 +19,7 @@ class ReportBase(SQLModel): label: str description: str | None = None + report_type: str | None = None user_id: UUID | None = Field(default=None, foreign_key="users.id") markdown: str | None = Field(default=None, sa_column=Column(Text)) parent_report_id: UUID | None = Field(default=None, foreign_key="reports.id") diff --git a/src/policyengine_api/models/simulation.py b/src/policyengine_api/models/simulation.py index b23141e..985db3e 100644 --- a/src/policyengine_api/models/simulation.py +++ b/src/policyengine_api/models/simulation.py @@ -1,13 +1,16 @@ from datetime import datetime, timezone from enum import Enum -from typing import TYPE_CHECKING -from uuid import UUID, uuid4 +from typing import TYPE_CHECKING, Any +from sqlalchemy import Column +from sqlalchemy.dialects.postgresql import JSON from sqlmodel import Field, Relationship, SQLModel +from uuid import UUID, uuid4 if TYPE_CHECKING: from .dataset import Dataset from .dynamic import Dynamic + from .household import Household from .policy import Policy from .tax_benefit_model_version import TaxBenefitModelVersion @@ -21,10 +24,19 @@ class SimulationStatus(str, Enum): FAILED = "failed" +class SimulationType(str, Enum): + """Type of simulation.""" + + HOUSEHOLD = "household" + ECONOMY = "economy" + + class SimulationBase(SQLModel): """Base simulation fields.""" - dataset_id: UUID = Field(foreign_key="datasets.id") + simulation_type: SimulationType = SimulationType.ECONOMY + dataset_id: UUID | None = Field(default=None, foreign_key="datasets.id") + household_id: UUID | None = Field(default=None, foreign_key="households.id") policy_id: UUID | None = Field(default=None, foreign_key="policies.id") dynamic_id: UUID | None = Field(default=None, foreign_key="dynamics.id") tax_benefit_model_version_id: UUID = Field( @@ -45,6 +57,9 @@ class Simulation(SimulationBase, table=True): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) started_at: datetime | None = None completed_at: datetime | None = None + household_result: dict[str, Any] | None = Field( + default=None, sa_column=Column(JSON) + ) # Relationships dataset: "Dataset" = Relationship( @@ -53,6 +68,12 @@ class Simulation(SimulationBase, table=True): "primaryjoin": "Simulation.dataset_id==Dataset.id", } ) + household: "Household" = Relationship( + sa_relationship_kwargs={ + "foreign_keys": "[Simulation.household_id]", + "primaryjoin": "Simulation.household_id==Household.id", + } + ) policy: "Policy" = Relationship() dynamic: "Dynamic" = Relationship() tax_benefit_model_version: "TaxBenefitModelVersion" = Relationship() @@ -78,3 +99,4 @@ class SimulationRead(SimulationBase): updated_at: datetime started_at: datetime | None completed_at: datetime | None + household_result: dict[str, Any] | None = None diff --git a/supabase/migrations/20260203000002_simulation_household_support.sql b/supabase/migrations/20260203000002_simulation_household_support.sql new file mode 100644 index 0000000..6813f07 --- /dev/null +++ b/supabase/migrations/20260203000002_simulation_household_support.sql @@ -0,0 +1,16 @@ +-- Add simulation_type as TEXT (SQLModel enum maps to text) +ALTER TABLE simulations ADD COLUMN simulation_type TEXT NOT NULL DEFAULT 'economy'; + +-- Make dataset_id nullable (was required) +ALTER TABLE simulations ALTER COLUMN dataset_id DROP NOT NULL; + +-- Add household support columns +ALTER TABLE simulations ADD COLUMN household_id UUID REFERENCES households(id); +ALTER TABLE simulations ADD COLUMN household_result JSONB; + +-- Indexes +CREATE INDEX idx_simulations_household ON simulations (household_id); +CREATE INDEX idx_simulations_type ON simulations (simulation_type); + +-- Add report_type to reports +ALTER TABLE reports ADD COLUMN report_type TEXT; diff --git a/test_fixtures/fixtures_analysis.py b/test_fixtures/fixtures_analysis.py new file mode 100644 index 0000000..d56b702 --- /dev/null +++ b/test_fixtures/fixtures_analysis.py @@ -0,0 +1,164 @@ +"""Fixtures and helpers for analysis endpoint tests.""" + +from uuid import UUID + +from sqlmodel import Session + +from policyengine_api.models import ( + Household, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) + + +def create_tax_benefit_model( + session: Session, + name: str = "policyengine-uk", + description: str = "UK tax benefit model", +) -> TaxBenefitModel: + """Create and persist a TaxBenefitModel record.""" + model = TaxBenefitModel( + name=name, + description=description, + ) + session.add(model) + session.commit() + session.refresh(model) + return model + + +def create_model_version( + session: Session, + model_id: UUID, + version: str = "1.0.0", + description: str = "Test version", +) -> TaxBenefitModelVersion: + """Create and persist a TaxBenefitModelVersion record.""" + model_version = TaxBenefitModelVersion( + model_id=model_id, + version=version, + description=description, + ) + session.add(model_version) + session.commit() + session.refresh(model_version) + return model_version + + +def create_parameter( + session: Session, + model_version_id: UUID, + name: str = "test_parameter", + label: str = "Test Parameter", + description: str = "A test parameter", +) -> Parameter: + """Create and persist a Parameter record.""" + param = Parameter( + tax_benefit_model_version_id=model_version_id, + name=name, + label=label, + description=description, + ) + session.add(param) + session.commit() + session.refresh(param) + return param + + +def create_policy( + session: Session, + model_version_id: UUID, + name: str = "Test Policy", + description: str = "A test policy", +) -> Policy: + """Create and persist a Policy record.""" + policy = Policy( + tax_benefit_model_version_id=model_version_id, + name=name, + description=description, + ) + session.add(policy) + session.commit() + session.refresh(policy) + return policy + + +def create_policy_with_parameter_value( + session: Session, + model_version_id: UUID, + parameter_id: UUID, + value: float, + name: str = "Test Policy", +) -> Policy: + """Create a Policy with an associated ParameterValue.""" + policy = create_policy(session, model_version_id, name=name) + + param_value = ParameterValue( + policy_id=policy.id, + parameter_id=parameter_id, + value_json={"value": value}, + ) + session.add(param_value) + session.commit() + session.refresh(policy) + return policy + + +def create_household_for_analysis( + session: Session, + tax_benefit_model_name: str = "policyengine_uk", + year: int = 2024, + label: str = "Test household for analysis", +) -> Household: + """Create a household suitable for analysis testing.""" + if tax_benefit_model_name == "policyengine_uk": + household_data = { + "people": [{"age": 30, "employment_income": 35000}], + "benunit": {}, + "household": {"region": "LONDON"}, + } + else: + household_data = { + "people": [{"age": 30, "employment_income": 50000}], + "tax_unit": {"state_code": "CA"}, + "family": {}, + "spm_unit": {}, + "marital_unit": {}, + "household": {"state_fips": 6}, + } + + record = Household( + tax_benefit_model_name=tax_benefit_model_name, + year=year, + label=label, + household_data=household_data, + ) + session.add(record) + session.commit() + session.refresh(record) + return record + + +def setup_uk_model_and_version( + session: Session, +) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: + """Create UK model and version for testing.""" + model = create_tax_benefit_model( + session, name="policyengine-uk", description="UK model" + ) + version = create_model_version(session, model.id) + return model, version + + +def setup_us_model_and_version( + session: Session, +) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: + """Create US model and version for testing.""" + model = create_tax_benefit_model( + session, name="policyengine-us", description="US model" + ) + version = create_model_version(session, model.id) + return model, version diff --git a/tests/test_analysis_household_impact.py b/tests/test_analysis_household_impact.py new file mode 100644 index 0000000..e8a614b --- /dev/null +++ b/tests/test_analysis_household_impact.py @@ -0,0 +1,297 @@ +"""Tests for household impact analysis endpoints.""" + +from uuid import uuid4 + +import pytest + +from test_fixtures.fixtures_analysis import ( + create_household_for_analysis, + create_policy, + setup_uk_model_and_version, + setup_us_model_and_version, +) +from policyengine_api.models import Report, ReportStatus, Simulation, SimulationType + + +# --------------------------------------------------------------------------- +# Validation tests (no database required beyond session fixture) +# --------------------------------------------------------------------------- + + +class TestHouseholdImpactValidation: + """Tests for request validation.""" + + def test_missing_household_id(self, client): + """Test that missing household_id returns 422.""" + response = client.post( + "/analysis/household-impact", + json={}, + ) + assert response.status_code == 422 + + def test_invalid_uuid(self, client): + """Test that invalid UUID returns 422.""" + response = client.post( + "/analysis/household-impact", + json={ + "household_id": "not-a-uuid", + }, + ) + assert response.status_code == 422 + + +# --------------------------------------------------------------------------- +# 404 tests +# --------------------------------------------------------------------------- + + +class TestHouseholdImpactNotFound: + """Tests for 404 responses.""" + + def test_household_not_found(self, client, session): + """Test that non-existent household returns 404.""" + # Need model for the model version lookup + setup_uk_model_and_version(session) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(uuid4()), + }, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"].lower() + + def test_policy_not_found(self, client, session): + """Test that non-existent policy returns 404.""" + setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + "policy_id": str(uuid4()), + }, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"].lower() + + def test_get_report_not_found(self, client): + """Test that GET with non-existent report_id returns 404.""" + response = client.get(f"/analysis/household-impact/{uuid4()}") + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# Record creation tests +# --------------------------------------------------------------------------- + + +class TestHouseholdImpactRecordCreation: + """Tests for correct record creation.""" + + def test_single_run_creates_one_simulation(self, client, session): + """Single run (no policy_id) creates one simulation.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + }, + ) + # May fail during calculation since policyengine not available, + # but should create records + data = response.json() + assert "report_id" in data + assert data["report_type"] == "household_single" + assert data["baseline_simulation"] is not None + assert data["reform_simulation"] is None + + def test_comparison_creates_two_simulations(self, client, session): + """Comparison (with policy_id) creates two simulations.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + policy = create_policy(session, version.id) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + "policy_id": str(policy.id), + }, + ) + data = response.json() + assert "report_id" in data + assert data["report_type"] == "household_comparison" + assert data["baseline_simulation"] is not None + assert data["reform_simulation"] is not None + + def test_simulation_type_is_household(self, client, session): + """Created simulations have simulation_type=HOUSEHOLD.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + }, + ) + data = response.json() + + # Check simulation in database + sim_id = data["baseline_simulation"]["id"] + sim = session.get(Simulation, sim_id) + assert sim is not None + assert sim.simulation_type == SimulationType.HOUSEHOLD + assert sim.household_id == household.id + assert sim.dataset_id is None + + def test_report_links_simulations(self, client, session): + """Report correctly links baseline and reform simulations.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + policy = create_policy(session, version.id) + + response = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + "policy_id": str(policy.id), + }, + ) + data = response.json() + + # Check report in database + report = session.get(Report, data["report_id"]) + assert report is not None + assert report.baseline_simulation_id == data["baseline_simulation"]["id"] + assert report.reform_simulation_id == data["reform_simulation"]["id"] + assert report.report_type == "household_comparison" + + +# --------------------------------------------------------------------------- +# Deduplication tests +# --------------------------------------------------------------------------- + + +class TestHouseholdImpactDeduplication: + """Tests for simulation/report deduplication.""" + + def test_same_request_returns_same_simulation(self, client, session): + """Same household + same parameters returns same simulation ID.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + + # First request + response1 = client.post( + "/analysis/household-impact", + json={"household_id": str(household.id)}, + ) + data1 = response1.json() + + # Second request with same parameters + response2 = client.post( + "/analysis/household-impact", + json={"household_id": str(household.id)}, + ) + data2 = response2.json() + + # Should return same IDs + assert data1["report_id"] == data2["report_id"] + assert data1["baseline_simulation"]["id"] == data2["baseline_simulation"]["id"] + + def test_different_policy_creates_different_simulation(self, client, session): + """Different policy creates different simulation.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + policy1 = create_policy(session, version.id, name="Policy 1") + policy2 = create_policy(session, version.id, name="Policy 2") + + # Request with policy1 + response1 = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + "policy_id": str(policy1.id), + }, + ) + data1 = response1.json() + + # Request with policy2 + response2 = client.post( + "/analysis/household-impact", + json={ + "household_id": str(household.id), + "policy_id": str(policy2.id), + }, + ) + data2 = response2.json() + + # Reports should be different + assert data1["report_id"] != data2["report_id"] + # Reform simulations should be different + assert ( + data1["reform_simulation"]["id"] != data2["reform_simulation"]["id"] + ) + # Baseline simulations should be the same (same household, no policy) + assert ( + data1["baseline_simulation"]["id"] == data2["baseline_simulation"]["id"] + ) + + +# --------------------------------------------------------------------------- +# GET endpoint tests +# --------------------------------------------------------------------------- + + +class TestGetHouseholdImpact: + """Tests for GET /analysis/household-impact/{report_id}.""" + + def test_get_returns_report_data(self, client, session): + """GET returns report with simulation info.""" + _, version = setup_uk_model_and_version(session) + household = create_household_for_analysis(session) + + # Create report via POST + post_response = client.post( + "/analysis/household-impact", + json={"household_id": str(household.id)}, + ) + report_id = post_response.json()["report_id"] + + # GET the report + get_response = client.get(f"/analysis/household-impact/{report_id}") + assert get_response.status_code == 200 + + data = get_response.json() + assert data["report_id"] == report_id + assert data["report_type"] == "household_single" + assert data["baseline_simulation"] is not None + + +# --------------------------------------------------------------------------- +# US household tests +# --------------------------------------------------------------------------- + + +class TestUSHouseholdImpact: + """Tests specific to US households.""" + + def test_us_household_creates_simulation(self, client, session): + """US household creates simulation with correct model.""" + _, version = setup_us_model_and_version(session) + household = create_household_for_analysis( + session, tax_benefit_model_name="policyengine_us" + ) + + response = client.post( + "/analysis/household-impact", + json={"household_id": str(household.id)}, + ) + data = response.json() + assert "report_id" in data + assert data["baseline_simulation"] is not None From 3c28466a9623f3266a3430a910a8968383e99ad3 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 3 Feb 2026 22:34:31 +0300 Subject: [PATCH 04/87] fix: Improve code quality --- src/policyengine_api/api/__init__.py | 2 + src/policyengine_api/api/analysis.py | 361 +--------- .../api/household_analysis.py | 640 ++++++++++++++++++ 3 files changed, 643 insertions(+), 360 deletions(-) create mode 100644 src/policyengine_api/api/household_analysis.py diff --git a/src/policyengine_api/api/__init__.py b/src/policyengine_api/api/__init__.py index 92f5ea5..c3e0353 100644 --- a/src/policyengine_api/api/__init__.py +++ b/src/policyengine_api/api/__init__.py @@ -9,6 +9,7 @@ datasets, dynamics, household, + household_analysis, households, outputs, parameter_values, @@ -35,6 +36,7 @@ api_router.include_router(tax_benefit_model_versions.router) api_router.include_router(change_aggregates.router) api_router.include_router(household.router) +api_router.include_router(household_analysis.router) api_router.include_router(households.router) api_router.include_router(analysis.router) api_router.include_router(agent.router) diff --git a/src/policyengine_api/api/analysis.py b/src/policyengine_api/api/analysis.py index b1ab584..10e6fc5 100644 --- a/src/policyengine_api/api/analysis.py +++ b/src/policyengine_api/api/analysis.py @@ -16,8 +16,7 @@ """ import math -from datetime import datetime, timezone -from typing import Any, Literal +from typing import Literal from uuid import UUID, uuid5 import logfire @@ -30,8 +29,6 @@ Dataset, DecileImpact, DecileImpactRead, - Household, - Policy, ProgramStatistics, ProgramStatisticsRead, Report, @@ -574,362 +571,6 @@ def _trigger_economy_comparison( fn.spawn(job_id=job_id, traceparent=traceparent) -# Entity types by country -UK_ENTITIES = ["person", "benunit", "household"] -US_ENTITIES = ["person", "tax_unit", "spm_unit", "family", "marital_unit", "household"] - - -def _compute_entity_diff( - baseline_list: list[dict], - reform_list: list[dict], -) -> list[dict]: - """Compute per-variable diffs for a list of entity instances.""" - entity_impact = [] - - for b_entity, r_entity in zip(baseline_list, reform_list): - entity_diff = {} - for key in b_entity: - if key in r_entity: - baseline_val = b_entity[key] - reform_val = r_entity[key] - if isinstance(baseline_val, (int, float)) and isinstance( - reform_val, (int, float) - ): - entity_diff[key] = { - "baseline": baseline_val, - "reform": reform_val, - "change": reform_val - baseline_val, - } - entity_impact.append(entity_diff) - - return entity_impact - - -def _compute_household_impact( - baseline_result: dict, - reform_result: dict, - country: str, -) -> dict[str, Any]: - """Compute difference between baseline and reform for all entity types.""" - entities = UK_ENTITIES if country == "uk" else US_ENTITIES - - impact: dict[str, Any] = {} - - for entity in entities: - if entity in baseline_result and entity in reform_result: - impact[entity] = _compute_entity_diff( - baseline_result[entity], - reform_result[entity], - ) - - return impact - - -def _ensure_list(value: Any) -> list: - """Ensure value is a list; wrap dict in list if needed.""" - if value is None: - return [] - if isinstance(value, list): - return value - return [value] - - -def _run_household_simulation(simulation_id: UUID, session: Session) -> None: - """Run a single household simulation and store result.""" - from policyengine_api.api.household import ( - _calculate_household_uk, - _calculate_household_us, - ) - - simulation = session.get(Simulation, simulation_id) - if not simulation: - raise ValueError(f"Simulation {simulation_id} not found") - - household = session.get(Household, simulation.household_id) - if not household: - raise ValueError(f"Household {simulation.household_id} not found") - - # Update status - simulation.status = SimulationStatus.RUNNING - simulation.started_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - - try: - # Get policy if set - policy_data = None - if simulation.policy_id: - policy = session.get(Policy, simulation.policy_id) - if policy and policy.parameter_values: - policy_data = {} - for pv in policy.parameter_values: - if pv.parameter: - param_name = pv.parameter.name - policy_data[param_name] = { - "value": pv.value_json.get("value") - if isinstance(pv.value_json, dict) - else pv.value_json, - "start_date": pv.start_date.isoformat() - if pv.start_date - else None, - "end_date": pv.end_date.isoformat() - if pv.end_date - else None, - } - - # Extract household data with list conversion - data = household.household_data - people = data.get("people", []) - - # Run calculation based on country - if household.tax_benefit_model_name == "policyengine_uk": - result = _calculate_household_uk( - people=people, - benunit=_ensure_list(data.get("benunit")), - household=_ensure_list(data.get("household")), - year=household.year, - policy_data=policy_data, - ) - else: - result = _calculate_household_us( - people=people, - marital_unit=_ensure_list(data.get("marital_unit")), - family=_ensure_list(data.get("family")), - spm_unit=_ensure_list(data.get("spm_unit")), - tax_unit=_ensure_list(data.get("tax_unit")), - household=_ensure_list(data.get("household")), - year=household.year, - policy_data=policy_data, - ) - - # Store result - simulation.household_result = result - simulation.status = SimulationStatus.COMPLETED - simulation.completed_at = datetime.now(timezone.utc) - - except Exception as e: - simulation.status = SimulationStatus.FAILED - simulation.error_message = str(e) - simulation.completed_at = datetime.now(timezone.utc) - - session.add(simulation) - session.commit() - - -def _trigger_household_report(report_id: UUID, session: Session) -> None: - """Trigger household simulation(s) for a report.""" - report = session.get(Report, report_id) - if not report: - raise ValueError(f"Report {report_id} not found") - - # Update report status - report.status = ReportStatus.RUNNING - session.add(report) - session.commit() - - try: - # Run baseline - baseline_sim = session.get(Simulation, report.baseline_simulation_id) - if baseline_sim and baseline_sim.status == SimulationStatus.PENDING: - _run_household_simulation(baseline_sim.id, session) - - # Run reform if exists - if report.reform_simulation_id: - reform_sim = session.get(Simulation, report.reform_simulation_id) - if reform_sim and reform_sim.status == SimulationStatus.PENDING: - _run_household_simulation(reform_sim.id, session) - - # Update report status - report.status = ReportStatus.COMPLETED - except Exception as e: - report.status = ReportStatus.FAILED - report.error_message = str(e) - - session.add(report) - session.commit() - - -# Household impact request/response schemas -class HouseholdImpactRequest(BaseModel): - """Request for household impact analysis.""" - - household_id: UUID = Field(description="ID of the household to analyze") - policy_id: UUID | None = Field( - default=None, - description="Reform policy ID. If None, runs single calculation under current law.", - ) - dynamic_id: UUID | None = Field( - default=None, description="Optional behavioural response specification ID" - ) - - -class HouseholdSimulationInfo(BaseModel): - """Info about a household simulation.""" - - id: UUID - status: SimulationStatus - error_message: str | None = None - - -class HouseholdImpactResponse(BaseModel): - """Response for household impact analysis.""" - - report_id: UUID - report_type: str - status: ReportStatus - baseline_simulation: HouseholdSimulationInfo | None = None - reform_simulation: HouseholdSimulationInfo | None = None - baseline_result: dict | None = None - reform_result: dict | None = None - impact: dict | None = None - error_message: str | None = None - - -def _build_household_response( - report: Report, - baseline_sim: Simulation, - reform_sim: Simulation | None, - session: Session, -) -> HouseholdImpactResponse: - """Build response including computed impact for comparisons.""" - baseline_result = baseline_sim.household_result if baseline_sim else None - reform_result = reform_sim.household_result if reform_sim else None - - # Compute impact if comparison and both complete - impact = None - if reform_sim and baseline_result and reform_result: - # Determine country from household - household = session.get(Household, baseline_sim.household_id) - if household: - country = ( - "uk" if household.tax_benefit_model_name == "policyengine_uk" else "us" - ) - impact = _compute_household_impact(baseline_result, reform_result, country) - - return HouseholdImpactResponse( - report_id=report.id, - report_type=report.report_type or "household_single", - status=report.status, - baseline_simulation=HouseholdSimulationInfo( - id=baseline_sim.id, - status=baseline_sim.status, - error_message=baseline_sim.error_message, - ) - if baseline_sim - else None, - reform_simulation=HouseholdSimulationInfo( - id=reform_sim.id, - status=reform_sim.status, - error_message=reform_sim.error_message, - ) - if reform_sim - else None, - baseline_result=baseline_result, - reform_result=reform_result, - impact=impact, - error_message=report.error_message, - ) - - -@router.post("/household-impact", response_model=HouseholdImpactResponse) -def household_impact( - request: HouseholdImpactRequest, - session: Session = Depends(get_session), -) -> HouseholdImpactResponse: - """Run household impact analysis. - - If policy_id is None: single run under current law. - If policy_id is set: comparison (baseline vs reform). - - This is a synchronous operation for household calculations. - """ - # Validate household exists - household = session.get(Household, request.household_id) - if not household: - raise HTTPException( - status_code=404, detail=f"Household {request.household_id} not found" - ) - - # Validate policy if provided - if request.policy_id: - policy = session.get(Policy, request.policy_id) - if not policy: - raise HTTPException( - status_code=404, detail=f"Policy {request.policy_id} not found" - ) - - # Get model version from household's tax_benefit_model_name - model_version = _get_model_version(household.tax_benefit_model_name, session) - - # Create baseline simulation - baseline_sim = _get_or_create_simulation( - simulation_type=SimulationType.HOUSEHOLD, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=request.dynamic_id, - session=session, - household_id=request.household_id, - ) - - # Create reform simulation if policy_id provided - reform_sim = None - if request.policy_id: - reform_sim = _get_or_create_simulation( - simulation_type=SimulationType.HOUSEHOLD, - model_version_id=model_version.id, - policy_id=request.policy_id, - dynamic_id=request.dynamic_id, - session=session, - household_id=request.household_id, - ) - - # Determine report type - report_type = "household_comparison" if request.policy_id else "household_single" - - # Create report - label = f"Household impact: {household.tax_benefit_model_name}" - report = _get_or_create_report( - baseline_sim_id=baseline_sim.id, - reform_sim_id=reform_sim.id if reform_sim else None, - label=label, - report_type=report_type, - session=session, - ) - - # Trigger compute if pending - if report.status == ReportStatus.PENDING: - with logfire.span("trigger_household_report", job_id=str(report.id)): - _trigger_household_report(report.id, session) - - return _build_household_response(report, baseline_sim, reform_sim, session) - - -@router.get("/household-impact/{report_id}", response_model=HouseholdImpactResponse) -def get_household_impact( - report_id: UUID, - session: Session = Depends(get_session), -) -> HouseholdImpactResponse: - """Get household impact analysis status and results.""" - report = session.get(Report, report_id) - if not report: - raise HTTPException(status_code=404, detail=f"Report {report_id} not found") - - if not report.baseline_simulation_id: - raise HTTPException( - status_code=500, detail="Report missing baseline simulation ID" - ) - - baseline_sim = session.get(Simulation, report.baseline_simulation_id) - if not baseline_sim: - raise HTTPException(status_code=500, detail="Baseline simulation data missing") - - reform_sim = None - if report.reform_simulation_id: - reform_sim = session.get(Simulation, report.reform_simulation_id) - - return _build_household_response(report, baseline_sim, reform_sim, session) - - @router.post("/economic-impact", response_model=EconomicImpactResponse) def economic_impact( request: EconomicImpactRequest, diff --git a/src/policyengine_api/api/household_analysis.py b/src/policyengine_api/api/household_analysis.py new file mode 100644 index 0000000..29ea89e --- /dev/null +++ b/src/policyengine_api/api/household_analysis.py @@ -0,0 +1,640 @@ +"""Household impact analysis endpoints. + +Use these endpoints to analyze household-level effects of policy reforms. +Supports single runs (current law) and comparisons (baseline vs reform). + +WORKFLOW: +1. Create a stored household: POST /households +2. Optionally create a reform policy: POST /policies +3. Run analysis: POST /analysis/household-impact +4. Results are synchronous - the response includes computed values +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Protocol +from uuid import UUID + +import logfire +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlmodel import Session + +from policyengine_api.models import ( + Household, + Policy, + Report, + ReportStatus, + Simulation, + SimulationStatus, + SimulationType, +) +from policyengine_api.services.database import get_session + +from .analysis import ( + _get_model_version, + _get_or_create_report, + _get_or_create_simulation, +) + +router = APIRouter(prefix="/analysis", tags=["analysis"]) + + +# ============================================================================= +# Country Strategy Pattern +# ============================================================================= + + +@dataclass(frozen=True) +class CountryConfig: + """Configuration for a country's household calculation.""" + + name: str + entity_types: tuple[str, ...] + + +UK_CONFIG = CountryConfig( + name="uk", + entity_types=("person", "benunit", "household"), +) + +US_CONFIG = CountryConfig( + name="us", + entity_types=("person", "tax_unit", "spm_unit", "family", "marital_unit", "household"), +) + + +def get_country_config(tax_benefit_model_name: str) -> CountryConfig: + """Get country configuration from model name.""" + if tax_benefit_model_name == "policyengine_uk": + return UK_CONFIG + return US_CONFIG + + +class HouseholdCalculator(Protocol): + """Protocol for country-specific household calculators.""" + + def __call__( + self, + household_data: dict[str, Any], + year: int, + policy_data: dict | None, + ) -> dict: ... + + +def calculate_uk_household( + household_data: dict[str, Any], + year: int, + policy_data: dict | None, +) -> dict: + """Calculate UK household using the existing implementation.""" + from policyengine_api.api.household import _calculate_household_uk + + return _calculate_household_uk( + people=household_data.get("people", []), + benunit=_ensure_list(household_data.get("benunit")), + household=_ensure_list(household_data.get("household")), + year=year, + policy_data=policy_data, + ) + + +def calculate_us_household( + household_data: dict[str, Any], + year: int, + policy_data: dict | None, +) -> dict: + """Calculate US household using the existing implementation.""" + from policyengine_api.api.household import _calculate_household_us + + return _calculate_household_us( + people=household_data.get("people", []), + marital_unit=_ensure_list(household_data.get("marital_unit")), + family=_ensure_list(household_data.get("family")), + spm_unit=_ensure_list(household_data.get("spm_unit")), + tax_unit=_ensure_list(household_data.get("tax_unit")), + household=_ensure_list(household_data.get("household")), + year=year, + policy_data=policy_data, + ) + + +def get_calculator(tax_benefit_model_name: str) -> HouseholdCalculator: + """Get the appropriate calculator for a country.""" + if tax_benefit_model_name == "policyengine_uk": + return calculate_uk_household + return calculate_us_household + + +# ============================================================================= +# Data Transformation Helpers +# ============================================================================= + + +def _ensure_list(value: Any) -> list: + """Ensure value is a list; wrap dict in list if needed.""" + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _extract_policy_data(policy: Policy | None) -> dict | None: + """Extract policy data from a Policy model into calculation format.""" + if not policy or not policy.parameter_values: + return None + + policy_data = {} + for pv in policy.parameter_values: + if not pv.parameter: + continue + + policy_data[pv.parameter.name] = { + "value": _extract_value(pv.value_json), + "start_date": _format_date(pv.start_date), + "end_date": _format_date(pv.end_date), + } + + return policy_data if policy_data else None + + +def _extract_value(value_json: Any) -> Any: + """Extract the actual value from value_json.""" + if isinstance(value_json, dict): + return value_json.get("value") + return value_json + + +def _format_date(date: Any) -> str | None: + """Format a date for the policy data structure.""" + if date is None: + return None + if hasattr(date, "isoformat"): + return date.isoformat() + return str(date) + + +# ============================================================================= +# Impact Computation +# ============================================================================= + + +def compute_variable_diff(baseline_val: Any, reform_val: Any) -> dict | None: + """Compute diff for a single variable if both are numeric.""" + if not isinstance(baseline_val, (int, float)): + return None + if not isinstance(reform_val, (int, float)): + return None + + return { + "baseline": baseline_val, + "reform": reform_val, + "change": reform_val - baseline_val, + } + + +def compute_entity_diff(baseline_entity: dict, reform_entity: dict) -> dict: + """Compute per-variable diffs for a single entity instance.""" + entity_diff = {} + + for key, baseline_val in baseline_entity.items(): + reform_val = reform_entity.get(key) + if reform_val is None: + continue + + diff = compute_variable_diff(baseline_val, reform_val) + if diff is not None: + entity_diff[key] = diff + + return entity_diff + + +def compute_entity_list_diff( + baseline_list: list[dict], + reform_list: list[dict], +) -> list[dict]: + """Compute diffs for a list of entity instances.""" + return [ + compute_entity_diff(b_entity, r_entity) + for b_entity, r_entity in zip(baseline_list, reform_list) + ] + + +def compute_household_impact( + baseline_result: dict, + reform_result: dict, + config: CountryConfig, +) -> dict[str, Any]: + """Compute difference between baseline and reform for all entity types.""" + impact: dict[str, Any] = {} + + for entity in config.entity_types: + baseline_entities = baseline_result.get(entity) + reform_entities = reform_result.get(entity) + + if baseline_entities is None or reform_entities is None: + continue + + impact[entity] = compute_entity_list_diff(baseline_entities, reform_entities) + + return impact + + +# ============================================================================= +# Simulation Execution +# ============================================================================= + + +def mark_simulation_running(simulation: Simulation, session: Session) -> None: + """Mark a simulation as running.""" + simulation.status = SimulationStatus.RUNNING + simulation.started_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + +def mark_simulation_completed( + simulation: Simulation, + result: dict, + session: Session, +) -> None: + """Mark a simulation as completed with result.""" + simulation.household_result = result + simulation.status = SimulationStatus.COMPLETED + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + +def mark_simulation_failed( + simulation: Simulation, + error: Exception, + session: Session, +) -> None: + """Mark a simulation as failed with error.""" + simulation.status = SimulationStatus.FAILED + simulation.error_message = str(error) + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + +def run_household_simulation(simulation_id: UUID, session: Session) -> None: + """Run a single household simulation and store result.""" + simulation = _load_simulation(simulation_id, session) + household = _load_household(simulation.household_id, session) + policy_data = _load_policy_data(simulation.policy_id, session) + + mark_simulation_running(simulation, session) + + try: + calculator = get_calculator(household.tax_benefit_model_name) + result = calculator(household.household_data, household.year, policy_data) + mark_simulation_completed(simulation, result, session) + except Exception as e: + mark_simulation_failed(simulation, e, session) + + +def _load_simulation(simulation_id: UUID, session: Session) -> Simulation: + """Load simulation or raise error.""" + simulation = session.get(Simulation, simulation_id) + if not simulation: + raise ValueError(f"Simulation {simulation_id} not found") + return simulation + + +def _load_household(household_id: UUID | None, session: Session) -> Household: + """Load household or raise error.""" + if not household_id: + raise ValueError("Simulation has no household_id") + + household = session.get(Household, household_id) + if not household: + raise ValueError(f"Household {household_id} not found") + return household + + +def _load_policy_data(policy_id: UUID | None, session: Session) -> dict | None: + """Load and extract policy data if policy_id is set.""" + if not policy_id: + return None + + policy = session.get(Policy, policy_id) + return _extract_policy_data(policy) + + +# ============================================================================= +# Report Orchestration +# ============================================================================= + + +def trigger_household_report(report_id: UUID, session: Session) -> None: + """Trigger household simulation(s) for a report.""" + report = _load_report(report_id, session) + _mark_report_running(report, session) + + try: + _run_report_simulations(report, session) + _mark_report_completed(report, session) + except Exception as e: + _mark_report_failed(report, e, session) + + +def _load_report(report_id: UUID, session: Session) -> Report: + """Load report or raise error.""" + report = session.get(Report, report_id) + if not report: + raise ValueError(f"Report {report_id} not found") + return report + + +def _mark_report_running(report: Report, session: Session) -> None: + """Mark report as running.""" + report.status = ReportStatus.RUNNING + session.add(report) + session.commit() + + +def _mark_report_completed(report: Report, session: Session) -> None: + """Mark report as completed.""" + report.status = ReportStatus.COMPLETED + session.add(report) + session.commit() + + +def _mark_report_failed(report: Report, error: Exception, session: Session) -> None: + """Mark report as failed.""" + report.status = ReportStatus.FAILED + report.error_message = str(error) + session.add(report) + session.commit() + + +def _run_report_simulations(report: Report, session: Session) -> None: + """Run all pending simulations for a report.""" + _run_simulation_if_pending(report.baseline_simulation_id, session) + + if report.reform_simulation_id: + _run_simulation_if_pending(report.reform_simulation_id, session) + + +def _run_simulation_if_pending(simulation_id: UUID | None, session: Session) -> None: + """Run simulation if it exists and is pending.""" + if not simulation_id: + return + + simulation = session.get(Simulation, simulation_id) + if simulation and simulation.status == SimulationStatus.PENDING: + run_household_simulation(simulation.id, session) + + +# ============================================================================= +# Request/Response Schemas +# ============================================================================= + + +class HouseholdImpactRequest(BaseModel): + """Request for household impact analysis.""" + + household_id: UUID = Field(description="ID of the household to analyze") + policy_id: UUID | None = Field( + default=None, + description="Reform policy ID. If None, runs single calculation under current law.", + ) + dynamic_id: UUID | None = Field( + default=None, + description="Optional behavioural response specification ID", + ) + + +class HouseholdSimulationInfo(BaseModel): + """Info about a household simulation.""" + + id: UUID + status: SimulationStatus + error_message: str | None = None + + +class HouseholdImpactResponse(BaseModel): + """Response for household impact analysis.""" + + report_id: UUID + report_type: str + status: ReportStatus + baseline_simulation: HouseholdSimulationInfo | None = None + reform_simulation: HouseholdSimulationInfo | None = None + baseline_result: dict | None = None + reform_result: dict | None = None + impact: dict | None = None + error_message: str | None = None + + +# ============================================================================= +# Response Building +# ============================================================================= + + +def build_simulation_info(simulation: Simulation | None) -> HouseholdSimulationInfo | None: + """Build simulation info from a simulation.""" + if not simulation: + return None + + return HouseholdSimulationInfo( + id=simulation.id, + status=simulation.status, + error_message=simulation.error_message, + ) + + +def build_household_response( + report: Report, + baseline_sim: Simulation, + reform_sim: Simulation | None, + session: Session, +) -> HouseholdImpactResponse: + """Build response including computed impact for comparisons.""" + baseline_result = baseline_sim.household_result + reform_result = reform_sim.household_result if reform_sim else None + + impact = _compute_impact_if_comparison( + baseline_sim, reform_sim, baseline_result, reform_result, session + ) + + return HouseholdImpactResponse( + report_id=report.id, + report_type=report.report_type or "household_single", + status=report.status, + baseline_simulation=build_simulation_info(baseline_sim), + reform_simulation=build_simulation_info(reform_sim), + baseline_result=baseline_result, + reform_result=reform_result, + impact=impact, + error_message=report.error_message, + ) + + +def _compute_impact_if_comparison( + baseline_sim: Simulation, + reform_sim: Simulation | None, + baseline_result: dict | None, + reform_result: dict | None, + session: Session, +) -> dict | None: + """Compute impact only if this is a comparison with both results.""" + if not reform_sim: + return None + if not baseline_result or not reform_result: + return None + + household = session.get(Household, baseline_sim.household_id) + if not household: + return None + + config = get_country_config(household.tax_benefit_model_name) + return compute_household_impact(baseline_result, reform_result, config) + + +# ============================================================================= +# Validation Helpers +# ============================================================================= + + +def validate_household_exists(household_id: UUID, session: Session) -> Household: + """Validate household exists and return it.""" + household = session.get(Household, household_id) + if not household: + raise HTTPException( + status_code=404, + detail=f"Household {household_id} not found", + ) + return household + + +def validate_policy_exists(policy_id: UUID | None, session: Session) -> None: + """Validate policy exists if provided.""" + if not policy_id: + return + + policy = session.get(Policy, policy_id) + if not policy: + raise HTTPException( + status_code=404, + detail=f"Policy {policy_id} not found", + ) + + +# ============================================================================= +# Endpoints +# ============================================================================= + + +@router.post("/household-impact", response_model=HouseholdImpactResponse) +def household_impact( + request: HouseholdImpactRequest, + session: Session = Depends(get_session), +) -> HouseholdImpactResponse: + """Run household impact analysis. + + If policy_id is None: single run under current law. + If policy_id is set: comparison (baseline vs reform). + + This is a synchronous operation for household calculations. + """ + household = validate_household_exists(request.household_id, session) + validate_policy_exists(request.policy_id, session) + + model_version = _get_model_version(household.tax_benefit_model_name, session) + + baseline_sim = _create_baseline_simulation( + household, model_version.id, request.dynamic_id, session + ) + reform_sim = _create_reform_simulation( + household, model_version.id, request.policy_id, request.dynamic_id, session + ) + + report_type = "household_comparison" if request.policy_id else "household_single" + report = _get_or_create_report( + baseline_sim_id=baseline_sim.id, + reform_sim_id=reform_sim.id if reform_sim else None, + label=f"Household impact: {household.tax_benefit_model_name}", + report_type=report_type, + session=session, + ) + + if report.status == ReportStatus.PENDING: + with logfire.span("trigger_household_report", job_id=str(report.id)): + trigger_household_report(report.id, session) + + return build_household_response(report, baseline_sim, reform_sim, session) + + +@router.get("/household-impact/{report_id}", response_model=HouseholdImpactResponse) +def get_household_impact( + report_id: UUID, + session: Session = Depends(get_session), +) -> HouseholdImpactResponse: + """Get household impact analysis status and results.""" + report = session.get(Report, report_id) + if not report: + raise HTTPException(status_code=404, detail=f"Report {report_id} not found") + + if not report.baseline_simulation_id: + raise HTTPException( + status_code=500, + detail="Report missing baseline simulation ID", + ) + + baseline_sim = session.get(Simulation, report.baseline_simulation_id) + if not baseline_sim: + raise HTTPException(status_code=500, detail="Baseline simulation data missing") + + reform_sim = None + if report.reform_simulation_id: + reform_sim = session.get(Simulation, report.reform_simulation_id) + + return build_household_response(report, baseline_sim, reform_sim, session) + + +# ============================================================================= +# Simulation Creation Helpers +# ============================================================================= + + +def _create_baseline_simulation( + household: Household, + model_version_id: UUID, + dynamic_id: UUID | None, + session: Session, +) -> Simulation: + """Create baseline simulation (current law, no policy).""" + return _get_or_create_simulation( + simulation_type=SimulationType.HOUSEHOLD, + model_version_id=model_version_id, + policy_id=None, + dynamic_id=dynamic_id, + session=session, + household_id=household.id, + ) + + +def _create_reform_simulation( + household: Household, + model_version_id: UUID, + policy_id: UUID | None, + dynamic_id: UUID | None, + session: Session, +) -> Simulation | None: + """Create reform simulation if policy_id is provided.""" + if not policy_id: + return None + + return _get_or_create_simulation( + simulation_type=SimulationType.HOUSEHOLD, + model_version_id=model_version_id, + policy_id=policy_id, + dynamic_id=dynamic_id, + session=session, + household_id=household.id, + ) From 78f87804a30dd3c56419fb4c80842f153d1cceac Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 3 Feb 2026 22:59:28 +0300 Subject: [PATCH 05/87] test: Add tests --- test_fixtures/fixtures_analysis.py | 164 --------- test_fixtures/fixtures_household_analysis.py | 366 +++++++++++++++++++ tests/test_analysis_household_impact.py | 245 ++++++++++++- 3 files changed, 603 insertions(+), 172 deletions(-) delete mode 100644 test_fixtures/fixtures_analysis.py create mode 100644 test_fixtures/fixtures_household_analysis.py diff --git a/test_fixtures/fixtures_analysis.py b/test_fixtures/fixtures_analysis.py deleted file mode 100644 index d56b702..0000000 --- a/test_fixtures/fixtures_analysis.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Fixtures and helpers for analysis endpoint tests.""" - -from uuid import UUID - -from sqlmodel import Session - -from policyengine_api.models import ( - Household, - Parameter, - ParameterValue, - Policy, - TaxBenefitModel, - TaxBenefitModelVersion, -) - - -def create_tax_benefit_model( - session: Session, - name: str = "policyengine-uk", - description: str = "UK tax benefit model", -) -> TaxBenefitModel: - """Create and persist a TaxBenefitModel record.""" - model = TaxBenefitModel( - name=name, - description=description, - ) - session.add(model) - session.commit() - session.refresh(model) - return model - - -def create_model_version( - session: Session, - model_id: UUID, - version: str = "1.0.0", - description: str = "Test version", -) -> TaxBenefitModelVersion: - """Create and persist a TaxBenefitModelVersion record.""" - model_version = TaxBenefitModelVersion( - model_id=model_id, - version=version, - description=description, - ) - session.add(model_version) - session.commit() - session.refresh(model_version) - return model_version - - -def create_parameter( - session: Session, - model_version_id: UUID, - name: str = "test_parameter", - label: str = "Test Parameter", - description: str = "A test parameter", -) -> Parameter: - """Create and persist a Parameter record.""" - param = Parameter( - tax_benefit_model_version_id=model_version_id, - name=name, - label=label, - description=description, - ) - session.add(param) - session.commit() - session.refresh(param) - return param - - -def create_policy( - session: Session, - model_version_id: UUID, - name: str = "Test Policy", - description: str = "A test policy", -) -> Policy: - """Create and persist a Policy record.""" - policy = Policy( - tax_benefit_model_version_id=model_version_id, - name=name, - description=description, - ) - session.add(policy) - session.commit() - session.refresh(policy) - return policy - - -def create_policy_with_parameter_value( - session: Session, - model_version_id: UUID, - parameter_id: UUID, - value: float, - name: str = "Test Policy", -) -> Policy: - """Create a Policy with an associated ParameterValue.""" - policy = create_policy(session, model_version_id, name=name) - - param_value = ParameterValue( - policy_id=policy.id, - parameter_id=parameter_id, - value_json={"value": value}, - ) - session.add(param_value) - session.commit() - session.refresh(policy) - return policy - - -def create_household_for_analysis( - session: Session, - tax_benefit_model_name: str = "policyengine_uk", - year: int = 2024, - label: str = "Test household for analysis", -) -> Household: - """Create a household suitable for analysis testing.""" - if tax_benefit_model_name == "policyengine_uk": - household_data = { - "people": [{"age": 30, "employment_income": 35000}], - "benunit": {}, - "household": {"region": "LONDON"}, - } - else: - household_data = { - "people": [{"age": 30, "employment_income": 50000}], - "tax_unit": {"state_code": "CA"}, - "family": {}, - "spm_unit": {}, - "marital_unit": {}, - "household": {"state_fips": 6}, - } - - record = Household( - tax_benefit_model_name=tax_benefit_model_name, - year=year, - label=label, - household_data=household_data, - ) - session.add(record) - session.commit() - session.refresh(record) - return record - - -def setup_uk_model_and_version( - session: Session, -) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: - """Create UK model and version for testing.""" - model = create_tax_benefit_model( - session, name="policyengine-uk", description="UK model" - ) - version = create_model_version(session, model.id) - return model, version - - -def setup_us_model_and_version( - session: Session, -) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: - """Create US model and version for testing.""" - model = create_tax_benefit_model( - session, name="policyengine-us", description="US model" - ) - version = create_model_version(session, model.id) - return model, version diff --git a/test_fixtures/fixtures_household_analysis.py b/test_fixtures/fixtures_household_analysis.py new file mode 100644 index 0000000..573930a --- /dev/null +++ b/test_fixtures/fixtures_household_analysis.py @@ -0,0 +1,366 @@ +"""Fixtures and helpers for household analysis endpoint tests.""" + +from typing import Any +from unittest.mock import patch +from uuid import UUID + +import pytest +from sqlmodel import Session + +from policyengine_api.models import ( + Household, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) + + +# ============================================================================= +# Sample Calculation Results +# ============================================================================= + + +SAMPLE_UK_BASELINE_RESULT: dict[str, Any] = { + "person": [ + { + "age": 30, + "employment_income": 35000.0, + "income_tax": 4500.0, + "national_insurance": 2800.0, + "net_income": 27700.0, + } + ], + "benunit": [ + { + "universal_credit": 0.0, + "child_benefit": 0.0, + } + ], + "household": [ + { + "region": "LONDON", + "council_tax": 1500.0, + } + ], +} + + +SAMPLE_UK_REFORM_RESULT: dict[str, Any] = { + "person": [ + { + "age": 30, + "employment_income": 35000.0, + "income_tax": 4000.0, + "national_insurance": 2800.0, + "net_income": 28200.0, + } + ], + "benunit": [ + { + "universal_credit": 0.0, + "child_benefit": 0.0, + } + ], + "household": [ + { + "region": "LONDON", + "council_tax": 1500.0, + } + ], +} + + +SAMPLE_US_BASELINE_RESULT: dict[str, Any] = { + "person": [ + { + "age": 30, + "employment_income": 50000.0, + "income_tax": 6000.0, + "fica": 3825.0, + "net_income": 40175.0, + } + ], + "tax_unit": [ + { + "state_code": "CA", + "state_income_tax": 2500.0, + } + ], + "spm_unit": [{"snap": 0.0}], + "family": [{}], + "marital_unit": [{}], + "household": [{"state_fips": 6}], +} + + +SAMPLE_US_REFORM_RESULT: dict[str, Any] = { + "person": [ + { + "age": 30, + "employment_income": 50000.0, + "income_tax": 5500.0, + "fica": 3825.0, + "net_income": 40675.0, + } + ], + "tax_unit": [ + { + "state_code": "CA", + "state_income_tax": 2500.0, + } + ], + "spm_unit": [{"snap": 0.0}], + "family": [{}], + "marital_unit": [{}], + "household": [{"state_fips": 6}], +} + + +# ============================================================================= +# Mock Calculator Functions +# ============================================================================= + + +def mock_calculate_uk_household( + household_data: dict[str, Any], + year: int, + policy_data: dict | None, +) -> dict: + """Mock UK calculator that returns sample results.""" + if policy_data: + return SAMPLE_UK_REFORM_RESULT + return SAMPLE_UK_BASELINE_RESULT + + +def mock_calculate_us_household( + household_data: dict[str, Any], + year: int, + policy_data: dict | None, +) -> dict: + """Mock US calculator that returns sample results.""" + if policy_data: + return SAMPLE_US_REFORM_RESULT + return SAMPLE_US_BASELINE_RESULT + + +def mock_calculate_household_failing( + household_data: dict[str, Any], + year: int, + policy_data: dict | None, +) -> dict: + """Mock calculator that raises an exception.""" + raise RuntimeError("Calculation failed") + + +# ============================================================================= +# Pytest Fixtures for Mocking +# ============================================================================= + + +@pytest.fixture +def mock_uk_calculator(): + """Fixture that patches UK calculator with mock.""" + with patch( + "policyengine_api.api.household_analysis.calculate_uk_household", + side_effect=mock_calculate_uk_household, + ) as mock: + yield mock + + +@pytest.fixture +def mock_us_calculator(): + """Fixture that patches US calculator with mock.""" + with patch( + "policyengine_api.api.household_analysis.calculate_us_household", + side_effect=mock_calculate_us_household, + ) as mock: + yield mock + + +@pytest.fixture +def mock_calculators(): + """Fixture that patches both UK and US calculators.""" + with ( + patch( + "policyengine_api.api.household_analysis.calculate_uk_household", + side_effect=mock_calculate_uk_household, + ) as uk_mock, + patch( + "policyengine_api.api.household_analysis.calculate_us_household", + side_effect=mock_calculate_us_household, + ) as us_mock, + ): + yield {"uk": uk_mock, "us": us_mock} + + +@pytest.fixture +def mock_failing_calculator(): + """Fixture that patches calculators to fail.""" + with ( + patch( + "policyengine_api.api.household_analysis.calculate_uk_household", + side_effect=mock_calculate_household_failing, + ), + patch( + "policyengine_api.api.household_analysis.calculate_us_household", + side_effect=mock_calculate_household_failing, + ), + ): + yield + + +# ============================================================================= +# Database Factory Functions +# ============================================================================= + + +def create_tax_benefit_model( + session: Session, + name: str = "policyengine-uk", + description: str = "UK tax benefit model", +) -> TaxBenefitModel: + """Create and persist a TaxBenefitModel record.""" + model = TaxBenefitModel( + name=name, + description=description, + ) + session.add(model) + session.commit() + session.refresh(model) + return model + + +def create_model_version( + session: Session, + model_id: UUID, + version: str = "1.0.0", + description: str = "Test version", +) -> TaxBenefitModelVersion: + """Create and persist a TaxBenefitModelVersion record.""" + model_version = TaxBenefitModelVersion( + model_id=model_id, + version=version, + description=description, + ) + session.add(model_version) + session.commit() + session.refresh(model_version) + return model_version + + +def create_parameter( + session: Session, + model_version_id: UUID, + name: str = "test_parameter", + label: str = "Test Parameter", + description: str = "A test parameter", +) -> Parameter: + """Create and persist a Parameter record.""" + param = Parameter( + tax_benefit_model_version_id=model_version_id, + name=name, + label=label, + description=description, + ) + session.add(param) + session.commit() + session.refresh(param) + return param + + +def create_policy( + session: Session, + model_version_id: UUID, + name: str = "Test Policy", + description: str = "A test policy", +) -> Policy: + """Create and persist a Policy record.""" + policy = Policy( + tax_benefit_model_version_id=model_version_id, + name=name, + description=description, + ) + session.add(policy) + session.commit() + session.refresh(policy) + return policy + + +def create_policy_with_parameter_value( + session: Session, + model_version_id: UUID, + parameter_id: UUID, + value: float, + name: str = "Test Policy", +) -> Policy: + """Create a Policy with an associated ParameterValue.""" + policy = create_policy(session, model_version_id, name=name) + + param_value = ParameterValue( + policy_id=policy.id, + parameter_id=parameter_id, + value_json={"value": value}, + ) + session.add(param_value) + session.commit() + session.refresh(policy) + return policy + + +def create_household_for_analysis( + session: Session, + tax_benefit_model_name: str = "policyengine_uk", + year: int = 2024, + label: str = "Test household for analysis", +) -> Household: + """Create a household suitable for analysis testing.""" + if tax_benefit_model_name == "policyengine_uk": + household_data = { + "people": [{"age": 30, "employment_income": 35000}], + "benunit": {}, + "household": {"region": "LONDON"}, + } + else: + household_data = { + "people": [{"age": 30, "employment_income": 50000}], + "tax_unit": {"state_code": "CA"}, + "family": {}, + "spm_unit": {}, + "marital_unit": {}, + "household": {"state_fips": 6}, + } + + record = Household( + tax_benefit_model_name=tax_benefit_model_name, + year=year, + label=label, + household_data=household_data, + ) + session.add(record) + session.commit() + session.refresh(record) + return record + + +def setup_uk_model_and_version( + session: Session, +) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: + """Create UK model and version for testing.""" + model = create_tax_benefit_model( + session, name="policyengine-uk", description="UK model" + ) + version = create_model_version(session, model.id) + return model, version + + +def setup_us_model_and_version( + session: Session, +) -> tuple[TaxBenefitModel, TaxBenefitModelVersion]: + """Create US model and version for testing.""" + model = create_tax_benefit_model( + session, name="policyengine-us", description="US model" + ) + version = create_model_version(session, model.id) + return model, version diff --git a/tests/test_analysis_household_impact.py b/tests/test_analysis_household_impact.py index e8a614b..23465c7 100644 --- a/tests/test_analysis_household_impact.py +++ b/tests/test_analysis_household_impact.py @@ -1,18 +1,247 @@ """Tests for household impact analysis endpoints.""" -from uuid import uuid4 +from datetime import date +from uuid import UUID, uuid4 import pytest -from test_fixtures.fixtures_analysis import ( +from test_fixtures.fixtures_household_analysis import ( + SAMPLE_UK_BASELINE_RESULT, + SAMPLE_UK_REFORM_RESULT, + SAMPLE_US_BASELINE_RESULT, + SAMPLE_US_REFORM_RESULT, create_household_for_analysis, create_policy, setup_uk_model_and_version, setup_us_model_and_version, ) +from policyengine_api.api.household_analysis import ( + UK_CONFIG, + US_CONFIG, + _ensure_list, + _extract_value, + _format_date, + compute_entity_diff, + compute_entity_list_diff, + compute_household_impact, + compute_variable_diff, + get_calculator, + get_country_config, +) from policyengine_api.models import Report, ReportStatus, Simulation, SimulationType +# --------------------------------------------------------------------------- +# Unit tests for helper functions +# --------------------------------------------------------------------------- + + +class TestEnsureList: + """Tests for _ensure_list helper.""" + + def test_none_returns_empty_list(self): + assert _ensure_list(None) == [] + + def test_list_returns_same_list(self): + input_list = [1, 2, 3] + assert _ensure_list(input_list) == input_list + + def test_dict_wrapped_in_list(self): + input_dict = {"key": "value"} + result = _ensure_list(input_dict) + assert result == [input_dict] + + def test_empty_list_returns_empty_list(self): + assert _ensure_list([]) == [] + + +class TestExtractValue: + """Tests for _extract_value helper.""" + + def test_dict_with_value_key(self): + assert _extract_value({"value": 100}) == 100 + + def test_dict_without_value_key(self): + assert _extract_value({"other": 100}) is None + + def test_non_dict_returns_as_is(self): + assert _extract_value(100) == 100 + assert _extract_value("string") == "string" + assert _extract_value([1, 2]) == [1, 2] + + +class TestFormatDate: + """Tests for _format_date helper.""" + + def test_none_returns_none(self): + assert _format_date(None) is None + + def test_date_object_formatted(self): + d = date(2024, 1, 15) + assert _format_date(d) == "2024-01-15" + + def test_string_returns_string(self): + assert _format_date("2024-01-15") == "2024-01-15" + + +class TestComputeVariableDiff: + """Tests for compute_variable_diff helper.""" + + def test_numeric_values_return_diff(self): + result = compute_variable_diff(100, 150) + assert result == {"baseline": 100, "reform": 150, "change": 50} + + def test_negative_change(self): + result = compute_variable_diff(150, 100) + assert result == {"baseline": 150, "reform": 100, "change": -50} + + def test_float_values(self): + result = compute_variable_diff(100.5, 200.5) + assert result == {"baseline": 100.5, "reform": 200.5, "change": 100.0} + + def test_non_numeric_baseline_returns_none(self): + assert compute_variable_diff("string", 100) is None + + def test_non_numeric_reform_returns_none(self): + assert compute_variable_diff(100, "string") is None + + def test_both_non_numeric_returns_none(self): + assert compute_variable_diff("a", "b") is None + + +class TestComputeEntityDiff: + """Tests for compute_entity_diff helper.""" + + def test_computes_diff_for_numeric_keys(self): + baseline = {"income": 1000, "tax": 200, "name": "John"} + reform = {"income": 1000, "tax": 150, "name": "John"} + result = compute_entity_diff(baseline, reform) + + assert "income" in result + assert result["income"]["change"] == 0 + assert "tax" in result + assert result["tax"]["change"] == -50 + assert "name" not in result + + def test_missing_key_in_reform_skipped(self): + baseline = {"income": 1000, "tax": 200} + reform = {"income": 1000} + result = compute_entity_diff(baseline, reform) + + assert "income" in result + assert "tax" not in result + + def test_empty_entities(self): + assert compute_entity_diff({}, {}) == {} + + +class TestComputeEntityListDiff: + """Tests for compute_entity_list_diff helper.""" + + def test_computes_diff_for_each_pair(self): + baseline_list = [{"income": 100}, {"income": 200}] + reform_list = [{"income": 120}, {"income": 180}] + result = compute_entity_list_diff(baseline_list, reform_list) + + assert len(result) == 2 + assert result[0]["income"]["change"] == 20 + assert result[1]["income"]["change"] == -20 + + def test_empty_lists(self): + assert compute_entity_list_diff([], []) == [] + + +class TestComputeHouseholdImpact: + """Tests for compute_household_impact helper.""" + + def test_uk_household_impact(self): + result = compute_household_impact( + SAMPLE_UK_BASELINE_RESULT, + SAMPLE_UK_REFORM_RESULT, + UK_CONFIG, + ) + + assert "person" in result + assert "benunit" in result + assert "household" in result + + # Check person income_tax changed + person_diff = result["person"][0] + assert "income_tax" in person_diff + assert person_diff["income_tax"]["baseline"] == 4500.0 + assert person_diff["income_tax"]["reform"] == 4000.0 + assert person_diff["income_tax"]["change"] == -500.0 + + def test_us_household_impact(self): + result = compute_household_impact( + SAMPLE_US_BASELINE_RESULT, + SAMPLE_US_REFORM_RESULT, + US_CONFIG, + ) + + assert "person" in result + assert "tax_unit" in result + assert "spm_unit" in result + assert "family" in result + assert "marital_unit" in result + assert "household" in result + + # Check person income_tax changed + person_diff = result["person"][0] + assert person_diff["income_tax"]["change"] == -500.0 + + def test_missing_entity_skipped(self): + baseline = {"person": [{"income": 100}]} + reform = {"person": [{"income": 120}]} + result = compute_household_impact(baseline, reform, UK_CONFIG) + + assert "person" in result + assert "benunit" not in result + assert "household" not in result + + +class TestGetCountryConfig: + """Tests for get_country_config helper.""" + + def test_uk_model_returns_uk_config(self): + config = get_country_config("policyengine_uk") + assert config == UK_CONFIG + assert config.name == "uk" + assert "benunit" in config.entity_types + + def test_us_model_returns_us_config(self): + config = get_country_config("policyengine_us") + assert config == US_CONFIG + assert config.name == "us" + assert "tax_unit" in config.entity_types + + def test_unknown_model_defaults_to_us(self): + config = get_country_config("unknown_model") + assert config == US_CONFIG + + +class TestGetCalculator: + """Tests for get_calculator helper.""" + + def test_uk_model_returns_uk_calculator(self): + from policyengine_api.api.household_analysis import calculate_uk_household + + calc = get_calculator("policyengine_uk") + assert calc == calculate_uk_household + + def test_us_model_returns_us_calculator(self): + from policyengine_api.api.household_analysis import calculate_us_household + + calc = get_calculator("policyengine_us") + assert calc == calculate_us_household + + def test_unknown_model_defaults_to_us(self): + from policyengine_api.api.household_analysis import calculate_us_household + + calc = get_calculator("unknown_model") + assert calc == calculate_us_household + + # --------------------------------------------------------------------------- # Validation tests (no database required beyond session fixture) # --------------------------------------------------------------------------- @@ -142,8 +371,8 @@ def test_simulation_type_is_household(self, client, session): ) data = response.json() - # Check simulation in database - sim_id = data["baseline_simulation"]["id"] + # Check simulation in database (convert string to UUID for query) + sim_id = UUID(data["baseline_simulation"]["id"]) sim = session.get(Simulation, sim_id) assert sim is not None assert sim.simulation_type == SimulationType.HOUSEHOLD @@ -165,11 +394,11 @@ def test_report_links_simulations(self, client, session): ) data = response.json() - # Check report in database - report = session.get(Report, data["report_id"]) + # Check report in database (convert string to UUID for query) + report = session.get(Report, UUID(data["report_id"])) assert report is not None - assert report.baseline_simulation_id == data["baseline_simulation"]["id"] - assert report.reform_simulation_id == data["reform_simulation"]["id"] + assert report.baseline_simulation_id == UUID(data["baseline_simulation"]["id"]) + assert report.reform_simulation_id == UUID(data["reform_simulation"]["id"]) assert report.report_type == "household_comparison" From 6f90fbef77b57c5b669fff0ec3517b4c8760ed6b Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 4 Feb 2026 03:12:48 +0300 Subject: [PATCH 06/87] feat: Use Alembic for db migrations --- .claude/skills/database-migrations.md | 301 +++++++++ CLAUDE.md | 16 +- alembic.ini | 145 +++++ alembic/README | 1 + alembic/env.py | 87 +++ alembic/script.py.mako | 28 + ...7ac554f4aa_add_parameter_values_indexes.py | 52 ++ .../20260204_d6e30d3b834d_initial_schema.py | 599 ++++++++++++++++++ pyproject.toml | 1 + scripts/init.py | 155 +++-- scripts/seed_nevada.py | 128 ++++ src/policyengine_api/config/settings.py | 17 +- supabase/.temp/cli-latest | 2 +- ...229000000_add_parameter_values_indexes.sql | 0 .../20260103000000_add_poverty_inequality.sql | 0 .../20260111000000_add_aggregate_status.sql | 0 .../20260203000000_create_households.sql | 0 ...001_create_user_household_associations.sql | 0 ...203000002_simulation_household_support.sql | 0 uv.lock | 28 + 20 files changed, 1514 insertions(+), 46 deletions(-) create mode 100644 .claude/skills/database-migrations.md create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py create mode 100644 alembic/versions/20260204_d6e30d3b834d_initial_schema.py create mode 100644 scripts/seed_nevada.py rename supabase/{migrations => migrations_archived}/20251229000000_add_parameter_values_indexes.sql (100%) rename supabase/{migrations => migrations_archived}/20260103000000_add_poverty_inequality.sql (100%) rename supabase/{migrations => migrations_archived}/20260111000000_add_aggregate_status.sql (100%) rename supabase/{migrations => migrations_archived}/20260203000000_create_households.sql (100%) rename supabase/{migrations => migrations_archived}/20260203000001_create_user_household_associations.sql (100%) rename supabase/{migrations => migrations_archived}/20260203000002_simulation_household_support.sql (100%) diff --git a/.claude/skills/database-migrations.md b/.claude/skills/database-migrations.md new file mode 100644 index 0000000..fedbef8 --- /dev/null +++ b/.claude/skills/database-migrations.md @@ -0,0 +1,301 @@ +# Database Migration Guidelines + +## Overview + +This project uses **Alembic** for database migrations with **SQLModel** models. Alembic is the industry-standard migration tool for SQLAlchemy/SQLModel projects. + +**CRITICAL**: SQL migrations are the single source of truth for database schema. All table creation and schema changes MUST go through Alembic migrations. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ SQLModel Models (src/policyengine_api/models/) │ +│ - Define Python classes │ +│ - Used for ORM queries │ +│ - NOT the source of truth for schema │ +└─────────────────────────────────────────────────────────────┘ + │ + │ alembic revision --autogenerate + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Alembic Migrations (alembic/versions/) │ +│ - Create/alter tables │ +│ - Add indexes, constraints │ +│ - SOURCE OF TRUTH for schema │ +└─────────────────────────────────────────────────────────────┘ + │ + │ alembic upgrade head + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ PostgreSQL Database (Supabase) │ +│ - Actual schema │ +│ - Tracked by alembic_version table │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Essential Rules + +### 1. NEVER use SQLModel.metadata.create_all() for schema creation + +The old pattern of using `SQLModel.metadata.create_all()` is deprecated. All tables are created via Alembic migrations. + +### 2. Every schema change requires a migration + +When you modify a SQLModel model (add column, change type, add index), you MUST: +1. Update the model in `src/policyengine_api/models/` +2. Generate a migration: `uv run alembic revision --autogenerate -m "Description"` +3. **Read and verify the generated migration** (see below) +4. Apply it: `uv run alembic upgrade head` + +### 3. ALWAYS verify auto-generated migrations before applying + +**This is critical for AI agents.** After running `alembic revision --autogenerate`, you MUST: + +1. **Read the generated migration file** in `alembic/versions/` +2. **Verify the `upgrade()` function** contains the expected changes: + - Correct table/column names + - Correct column types (e.g., `sa.String()`, `sa.Uuid()`, `sa.Integer()`) + - Proper foreign key references + - Appropriate nullable settings +3. **Verify the `downgrade()` function** properly reverses the changes +4. **Check for Alembic autogenerate limitations:** + - It may miss renamed columns (shows as drop + add instead) + - It may not detect some index changes + - It doesn't handle data migrations +5. **Edit the migration if needed** before applying + +Example verification: +```python +# Generated migration - verify this looks correct: +def upgrade() -> None: + op.add_column('users', sa.Column('phone', sa.String(), nullable=True)) + +def downgrade() -> None: + op.drop_column('users', 'phone') +``` + +**Never blindly apply a migration without reading it first.** + +### 4. Migrations must be self-contained + +Each migration should: +- Create tables it needs (never assume they exist from Python) +- Include both `upgrade()` and `downgrade()` functions +- Be idempotent where possible (use `IF NOT EXISTS` patterns) + +### 5. Never use conditional logic based on table existence + +Migrations should NOT check if tables exist. Instead: +- Ensure migrations run in the correct order (use `down_revision`) +- The initial migration creates all base tables +- Subsequent migrations build on that foundation + +## Common Commands + +```bash +# Apply all pending migrations +uv run alembic upgrade head + +# Generate migration from model changes +uv run alembic revision --autogenerate -m "Add users email index" + +# Create empty migration (for manual SQL) +uv run alembic revision -m "Add custom index" + +# Check current migration state +uv run alembic current + +# Show migration history +uv run alembic history + +# Downgrade one revision +uv run alembic downgrade -1 + +# Downgrade to specific revision +uv run alembic downgrade +``` + +## Local Development Workflow + +```bash +# 1. Start Supabase +supabase start + +# 2. Initialize database (runs migrations + applies RLS policies) +uv run python scripts/init.py + +# 3. Seed data +uv run python scripts/seed.py +``` + +### Reset database (DESTRUCTIVE) + +```bash +uv run python scripts/init.py --reset +``` + +## Adding a New Model + +1. Create the model in `src/policyengine_api/models/` + +```python +# src/policyengine_api/models/my_model.py +from sqlmodel import SQLModel, Field +from uuid import UUID, uuid4 + +class MyModel(SQLModel, table=True): + __tablename__ = "my_models" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + name: str +``` + +2. Export in `__init__.py`: + +```python +# src/policyengine_api/models/__init__.py +from .my_model import MyModel +``` + +3. Generate migration: + +```bash +uv run alembic revision --autogenerate -m "Add my_models table" +``` + +4. Review the generated migration in `alembic/versions/` + +5. Apply the migration: + +```bash +uv run alembic upgrade head +``` + +6. Update `scripts/init.py` to include the table in RLS policies if needed. + +## Adding an Index + +1. Generate a migration: + +```bash +uv run alembic revision -m "Add index on users.email" +``` + +2. Edit the migration: + +```python +def upgrade() -> None: + op.create_index("idx_users_email", "users", ["email"]) + +def downgrade() -> None: + op.drop_index("idx_users_email", "users") +``` + +3. Apply: + +```bash +uv run alembic upgrade head +``` + +## Production Considerations + +### Applying migrations to production + +1. Migrations are automatically applied when deploying +2. Always test migrations locally first +3. For data migrations, consider running during low-traffic periods + +### Transitioning production from old system to Alembic + +Production databases that were created before Alembic (using the old `SQLModel.metadata.create_all()` approach or raw Supabase migrations) need special handling. Running `alembic upgrade head` would fail because the tables already exist. + +**The solution: `alembic stamp`** + +The `alembic stamp` command marks a migration as "already applied" without actually running it. This tells Alembic "the database is already at this state, start tracking from here." + +**How it works:** + +1. `alembic stamp ` inserts a row into the `alembic_version` table with the specified revision ID +2. Alembic now thinks that migration (and all migrations before it) have been applied +3. Future migrations will run normally starting from that point + +**Step-by-step production transition:** + +```bash +# 1. Connect to production database +# (set SUPABASE_DB_URL or other connection env vars) + +# 2. Check if alembic_version table exists +# If not, Alembic will create it automatically + +# 3. Verify production schema matches the initial migration +# Compare tables/columns in production against alembic/versions/20260204_d6e30d3b834d_initial_schema.py + +# 4. Stamp the initial migration as applied +uv run alembic stamp d6e30d3b834d + +# 5. If production also has the indexes from the second migration, stamp that too +uv run alembic stamp a17ac554f4aa + +# 6. Verify the stamp worked +uv run alembic current +# Should show: a17ac554f4aa (head) + +# 7. From now on, new migrations will apply normally +uv run alembic upgrade head +``` + +**Handling partially applied migrations:** + +If production has some but not all changes from a migration: + +1. Manually apply the missing changes via SQL +2. Then stamp that migration as complete +3. Or: create a new migration that only adds the missing pieces + +**After stamping:** + +- All future schema changes go through Alembic migrations +- Developers generate migrations with `alembic revision --autogenerate` +- Deployments run `alembic upgrade head` to apply pending migrations +- The `alembic_version` table tracks what's been applied + +## File Structure + +``` +alembic/ +├── env.py # Alembic configuration (imports models, sets DB URL) +├── script.py.mako # Template for new migrations +├── versions/ # Migration files +│ ├── 20260204_d6e30d3b834d_initial_schema.py +│ └── 20260204_a17ac554f4aa_add_parameter_values_indexes.py +alembic.ini # Alembic settings + +supabase/ +├── migrations/ # Supabase-specific migrations (storage only) +│ ├── 20241119000000_storage_bucket.sql +│ └── 20241121000000_storage_policies.sql +└── migrations_archived/ # Old table migrations (now in Alembic) +``` + +## Troubleshooting + +### "Target database is not up to date" + +Run `alembic upgrade head` to apply pending migrations. + +### "Can't locate revision" + +The alembic_version table has a revision that doesn't exist in your migrations folder. This can happen if someone deleted a migration file. Fix by stamping to a known revision: + +```bash +alembic stamp head # If tables are current +alembic stamp d6e30d3b834d # If at initial schema +``` + +### "Table already exists" + +The migration is trying to create a table that already exists. Options: +1. If this is a fresh setup, drop and recreate: `uv run python scripts/init.py --reset` +2. If in production, stamp the migration as applied: `alembic stamp ` diff --git a/CLAUDE.md b/CLAUDE.md index 2df55fc..d6fb240 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,21 @@ Use `gh` CLI for GitHub operations to ensure Actions run correctly. ## Database -`make init` resets tables and storage. `make seed` populates UK/US models with variables, parameters, and datasets. +This project uses **Alembic** for database migrations. See `.claude/skills/database-migrations.md` for detailed guidelines. + +**Key rules:** +- All schema changes go through Alembic migrations (never use `SQLModel.metadata.create_all()`) +- After modifying a model: `uv run alembic revision --autogenerate -m "Description"` +- Apply migrations: `uv run alembic upgrade head` + +**Local development:** +```bash +supabase start # Start local Supabase +uv run python scripts/init.py # Run migrations + apply RLS policies +uv run python scripts/seed.py # Seed data +``` + +`scripts/init.py --reset` drops and recreates everything (destructive). ## Modal sandbox + Claude Code CLI gotchas diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..ed54635 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,145 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names +# Prepend with date for easier chronological ordering +file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL - This is overridden by env.py which reads from application settings. +# The placeholder below is only used if env.py doesn't set it. +sqlalchemy.url = postgresql://placeholder:placeholder@localhost/placeholder + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# NOTE: ruff is in dev dependencies, so this hook only works when dev deps are installed +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..f930498 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,87 @@ +"""Alembic environment configuration for SQLModel migrations. + +This module configures Alembic to: +1. Use the database URL from application settings +2. Import all SQLModel models for autogenerate support +3. Run migrations in both offline and online modes +""" + +import sys +from logging.config import fileConfig +from pathlib import Path + +from sqlalchemy import engine_from_config, pool +from sqlmodel import SQLModel + +from alembic import context + +# Add src to path so we can import policyengine_api +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +# Import all models to register them with SQLModel.metadata +# This is required for autogenerate to detect model changes +from policyengine_api import models # noqa: F401 +from policyengine_api.config.settings import settings + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Override sqlalchemy.url with the actual database URL from settings +config.set_main_option("sqlalchemy.url", settings.database_url) + +# Interpret the config file for Python logging. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# SQLModel metadata for autogenerate support +# This allows Alembic to detect changes in your SQLModel models +target_metadata = SQLModel.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py b/alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py new file mode 100644 index 0000000..e1967c2 --- /dev/null +++ b/alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py @@ -0,0 +1,52 @@ +"""Add parameter_values indexes + +Revision ID: a17ac554f4aa +Revises: d6e30d3b834d +Create Date: 2026-02-04 02:20:00.000000 + +This migration adds performance indexes to the parameter_values table +for optimizing common query patterns. +""" + +from typing import Sequence, Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a17ac554f4aa" +down_revision: Union[str, Sequence[str], None] = "d6e30d3b834d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add performance indexes to parameter_values.""" + # Composite index for the most common query pattern (filtering by both) + op.create_index( + "idx_parameter_values_parameter_policy", + "parameter_values", + ["parameter_id", "policy_id"], + ) + + # Single index on policy_id for filtering by policy alone + op.create_index( + "idx_parameter_values_policy", + "parameter_values", + ["policy_id"], + ) + + # Partial index for baseline values (policy_id IS NULL) + # This optimizes the common "get current law values" query + op.create_index( + "idx_parameter_values_baseline", + "parameter_values", + ["parameter_id"], + postgresql_where="policy_id IS NULL", + ) + + +def downgrade() -> None: + """Remove parameter_values indexes.""" + op.drop_index("idx_parameter_values_baseline", "parameter_values") + op.drop_index("idx_parameter_values_policy", "parameter_values") + op.drop_index("idx_parameter_values_parameter_policy", "parameter_values") diff --git a/alembic/versions/20260204_d6e30d3b834d_initial_schema.py b/alembic/versions/20260204_d6e30d3b834d_initial_schema.py new file mode 100644 index 0000000..d4de071 --- /dev/null +++ b/alembic/versions/20260204_d6e30d3b834d_initial_schema.py @@ -0,0 +1,599 @@ +"""Initial schema + +Revision ID: d6e30d3b834d +Revises: +Create Date: 2026-02-04 02:15:03.471607 + +This migration creates all base tables for the PolicyEngine API. +Tables are organized by dependency tier to ensure proper creation order. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "d6e30d3b834d" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create all tables.""" + # ======================================================================== + # TIER 1: Tables with no foreign key dependencies + # ======================================================================== + + # Tax benefit models (e.g., "uk", "us") + op.create_table( + "tax_benefit_models", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # Users + op.create_table( + "users", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("first_name", sa.String(), nullable=False), + sa.Column("last_name", sa.String(), nullable=False), + sa.Column("email", sa.String(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + ) + op.create_index("ix_users_email", "users", ["email"]) + + # Policies (reform definitions) + op.create_table( + "policies", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # Dynamics (behavioral response definitions) + op.create_table( + "dynamics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # ======================================================================== + # TIER 2: Tables depending on tier 1 + # ======================================================================== + + # Tax benefit model versions + op.create_table( + "tax_benefit_model_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("model_id", sa.Uuid(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["model_id"], ["tax_benefit_models.id"]), + ) + + # Datasets (h5 files in storage) + op.create_table( + "datasets", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("filepath", sa.String(), nullable=False), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("is_output_dataset", sa.Boolean(), nullable=False, default=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), + ) + + # ======================================================================== + # TIER 3: Tables depending on tier 2 + # ======================================================================== + + # Parameters (tax-benefit system parameters) + op.create_table( + "parameters", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("label", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("data_type", sa.String(), nullable=True), + sa.Column("unit", sa.String(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + ) + + # Variables (tax-benefit system variables) + op.create_table( + "variables", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("data_type", sa.String(), nullable=True), + sa.Column("possible_values", sa.JSON(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + ) + + # Dataset versions + op.create_table( + "dataset_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=False), + sa.Column("dataset_id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), + sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), + ) + + # Households (stored household definitions) + op.create_table( + "households", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_name", sa.String(), nullable=False), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("label", sa.String(), nullable=True), + sa.Column("household_data", sa.JSON(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "idx_households_model_name", "households", ["tax_benefit_model_name"] + ) + op.create_index("idx_households_year", "households", ["year"]) + + # ======================================================================== + # TIER 4: Tables depending on tier 3 + # ======================================================================== + + # Parameter values (policy/dynamic parameter modifications) + op.create_table( + "parameter_values", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("parameter_id", sa.Uuid(), nullable=False), + sa.Column("value_json", sa.JSON(), nullable=True), + sa.Column("start_date", sa.DateTime(timezone=True), nullable=False), + sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["parameter_id"], ["parameters.id"]), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + ) + + # Simulations (economy or household calculations) + op.create_table( + "simulations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_type", sa.String(), nullable=False, default="economy"), + sa.Column("dataset_id", sa.Uuid(), nullable=True), + sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column("output_dataset_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("household_result", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), + sa.ForeignKeyConstraint(["household_id"], ["households.id"]), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + sa.ForeignKeyConstraint(["output_dataset_id"], ["datasets.id"]), + ) + + # User-household associations + op.create_table( + "user_household_associations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("household_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["household_id"], ["households.id"], ondelete="CASCADE"), + sa.UniqueConstraint("user_id", "household_id"), + ) + op.create_index( + "idx_user_household_user", "user_household_associations", ["user_id"] + ) + op.create_index( + "idx_user_household_household", "user_household_associations", ["household_id"] + ) + + # Household jobs (async household calculations) + op.create_table( + "household_jobs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_name", sa.String(), nullable=False), + sa.Column("request_data", sa.JSON(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + ) + + # ======================================================================== + # TIER 5: Tables depending on simulations + # ======================================================================== + + # Reports (analysis reports) + op.create_table( + "reports", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("label", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("report_type", sa.String(), nullable=True), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("markdown", sa.Text(), nullable=True), + sa.Column("parent_report_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=True), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["parent_report_id"], ["reports.id"]), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + ) + + # Aggregates (single-simulation aggregate outputs) + op.create_table( + "aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("variable", sa.String(), nullable=False), + sa.Column("aggregate_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("filter_config", sa.JSON(), nullable=False, default={}), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Change aggregates (baseline vs reform comparison) + op.create_table( + "change_aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("variable", sa.String(), nullable=False), + sa.Column("aggregate_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("filter_config", sa.JSON(), nullable=False, default={}), + sa.Column("change_geq", sa.Float(), nullable=True), + sa.Column("change_leq", sa.Float(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Decile impacts + op.create_table( + "decile_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("income_variable", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("decile", sa.Integer(), nullable=False), + sa.Column("quantiles", sa.Integer(), nullable=False, default=10), + sa.Column("baseline_mean", sa.Float(), nullable=True), + sa.Column("reform_mean", sa.Float(), nullable=True), + sa.Column("absolute_change", sa.Float(), nullable=True), + sa.Column("relative_change", sa.Float(), nullable=True), + sa.Column("count_better_off", sa.Float(), nullable=True), + sa.Column("count_worse_off", sa.Float(), nullable=True), + sa.Column("count_no_change", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Program statistics + op.create_table( + "program_statistics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("program_name", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False), + sa.Column("is_tax", sa.Boolean(), nullable=False, default=False), + sa.Column("baseline_total", sa.Float(), nullable=True), + sa.Column("reform_total", sa.Float(), nullable=True), + sa.Column("change", sa.Float(), nullable=True), + sa.Column("baseline_count", sa.Float(), nullable=True), + sa.Column("reform_count", sa.Float(), nullable=True), + sa.Column("winners", sa.Float(), nullable=True), + sa.Column("losers", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Poverty + op.create_table( + "poverty", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("poverty_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False, default="person"), + sa.Column("filter_variable", sa.String(), nullable=True), + sa.Column("headcount", sa.Float(), nullable=True), + sa.Column("total_population", sa.Float(), nullable=True), + sa.Column("rate", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["simulation_id"], ["simulations.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), + ) + op.create_index("idx_poverty_simulation_id", "poverty", ["simulation_id"]) + op.create_index("idx_poverty_report_id", "poverty", ["report_id"]) + + # Inequality + op.create_table( + "inequality", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("income_variable", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False, default="household"), + sa.Column("gini", sa.Float(), nullable=True), + sa.Column("top_10_share", sa.Float(), nullable=True), + sa.Column("top_1_share", sa.Float(), nullable=True), + sa.Column("bottom_50_share", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["simulation_id"], ["simulations.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), + ) + op.create_index("idx_inequality_simulation_id", "inequality", ["simulation_id"]) + op.create_index("idx_inequality_report_id", "inequality", ["report_id"]) + + +def downgrade() -> None: + """Drop all tables in reverse order.""" + # Tier 5 + op.drop_index("idx_inequality_report_id", "inequality") + op.drop_index("idx_inequality_simulation_id", "inequality") + op.drop_table("inequality") + op.drop_index("idx_poverty_report_id", "poverty") + op.drop_index("idx_poverty_simulation_id", "poverty") + op.drop_table("poverty") + op.drop_table("program_statistics") + op.drop_table("decile_impacts") + op.drop_table("change_aggregates") + op.drop_table("aggregates") + op.drop_table("reports") + + # Tier 4 + op.drop_table("household_jobs") + op.drop_index("idx_user_household_household", "user_household_associations") + op.drop_index("idx_user_household_user", "user_household_associations") + op.drop_table("user_household_associations") + op.drop_table("simulations") + op.drop_table("parameter_values") + + # Tier 3 + op.drop_index("idx_households_year", "households") + op.drop_index("idx_households_model_name", "households") + op.drop_table("households") + op.drop_table("dataset_versions") + op.drop_table("variables") + op.drop_table("parameters") + + # Tier 2 + op.drop_table("datasets") + op.drop_table("tax_benefit_model_versions") + + # Tier 1 + op.drop_table("dynamics") + op.drop_table("policies") + op.drop_index("ix_users_email", "users") + op.drop_table("users") + op.drop_table("tax_benefit_models") diff --git a/pyproject.toml b/pyproject.toml index 27eb310..1fe9093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "fastapi-mcp>=0.4.0", "modal>=0.68.0", "anthropic>=0.40.0", + "alembic>=1.13.0", ] [project.optional-dependencies] diff --git a/scripts/init.py b/scripts/init.py index cf7a04a..3aa925b 100644 --- a/scripts/init.py +++ b/scripts/init.py @@ -1,12 +1,19 @@ -"""Initialise Supabase: reset database, recreate tables, buckets, and permissions. +"""Initialise Supabase database with tables, buckets, and permissions. -This script performs a complete reset of the Supabase instance: -1. Drops and recreates the public schema (all tables) -2. Deletes and recreates the storage bucket -3. Creates all tables from SQLModel definitions -4. Applies RLS policies and storage permissions +This script can run in two modes: +1. Init mode (default): Creates tables via Alembic, applies RLS policies +2. Reset mode (--reset): Drops everything and recreates from scratch (DESTRUCTIVE) + +Usage: + uv run python scripts/init.py # Safe init (creates if not exists) + uv run python scripts/init.py --reset # Destructive reset (drops everything) + +For local development after `supabase start`, use init mode. +For production, use init mode to ensure tables and policies exist. +Reset mode should only be used when you need a completely fresh database. """ +import subprocess import sys from pathlib import Path @@ -14,16 +21,14 @@ from rich.console import Console from rich.panel import Panel -from sqlmodel import SQLModel, create_engine +from sqlmodel import create_engine -# Import all models to register them with SQLModel.metadata -from policyengine_api import models # noqa: F401 from policyengine_api.config.settings import settings from policyengine_api.services.storage import get_service_role_client console = Console() -MIGRATIONS_DIR = Path(__file__).parent.parent / "supabase" / "migrations" +PROJECT_ROOT = Path(__file__).parent.parent def reset_storage_bucket(): @@ -57,30 +62,61 @@ def reset_storage_bucket(): console.print(f"[yellow]⚠ Warning with storage bucket: {e}[/yellow]") +def ensure_storage_bucket(): + """Ensure storage bucket exists (non-destructive).""" + console.print("[bold blue]Ensuring storage bucket exists...") + + try: + supabase = get_service_role_client() + bucket_name = settings.storage_bucket + + # Try to get bucket info + try: + supabase.storage.get_bucket(bucket_name) + console.print(f"[green]✓[/green] Bucket '{bucket_name}' exists") + except Exception: + # Bucket doesn't exist, create it + supabase.storage.create_bucket(bucket_name, options={"public": True}) + console.print(f"[green]✓[/green] Created bucket '{bucket_name}'") + + except Exception as e: + console.print(f"[yellow]⚠ Warning with storage bucket: {e}[/yellow]") + + def reset_database(): - """Drop and recreate all tables.""" - console.print("[bold blue]Resetting database...") + """Drop and recreate the public schema (DESTRUCTIVE).""" + console.print("[bold red]Dropping database schema...") engine = create_engine(settings.database_url, echo=False) - # Drop and recreate public schema - console.print(" Dropping public schema...") with engine.begin() as conn: conn.exec_driver_sql("DROP SCHEMA public CASCADE") conn.exec_driver_sql("CREATE SCHEMA public") conn.exec_driver_sql("GRANT ALL ON SCHEMA public TO postgres") conn.exec_driver_sql("GRANT ALL ON SCHEMA public TO public") - # Create all tables from SQLModel - console.print(" Creating tables...") - SQLModel.metadata.create_all(engine) + console.print("[green]✓[/green] Schema dropped and recreated") + return engine - tables = list(SQLModel.metadata.tables.keys()) - console.print(f"[green]✓[/green] Created {len(tables)} tables:") - for table in sorted(tables): - console.print(f" {table}") - return engine +def run_alembic_migrations(): + """Run Alembic migrations to create/update tables.""" + console.print("[bold blue]Running Alembic migrations...") + + result = subprocess.run( + ["uv", "run", "alembic", "upgrade", "head"], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + console.print(f"[red]✗ Alembic migration failed:[/red]") + console.print(result.stderr) + raise RuntimeError("Alembic migration failed") + + console.print("[green]✓[/green] Alembic migrations complete") + console.print(result.stdout) def apply_storage_policies(engine): @@ -158,6 +194,10 @@ def apply_rls_policies(engine): "parameter_values", "users", "household_jobs", + "households", + "user_household_associations", + "poverty", + "inequality", ] # Read-only tables (public can read, only service role can write) @@ -178,6 +218,7 @@ def apply_rls_policies(engine): "dynamics", "reports", "household_jobs", + "households", ] # Read-only results tables @@ -186,6 +227,8 @@ def apply_rls_policies(engine): "change_aggregates", "decile_impacts", "program_statistics", + "poverty", + "inequality", ] sql_parts = [] @@ -230,6 +273,13 @@ def apply_rls_policies(engine): FOR SELECT TO anon, authenticated USING (true); """) + # User-household associations need special handling + sql_parts.append(""" + DROP POLICY IF EXISTS "Users can manage own associations" ON user_household_associations; + CREATE POLICY "Users can manage own associations" ON user_household_associations + FOR ALL TO anon, authenticated USING (true) WITH CHECK (true); + """) + sql = "\n".join(sql_parts) conn = engine.raw_connection() @@ -246,30 +296,53 @@ def apply_rls_policies(engine): def main(): - """Run full Supabase initialisation.""" - console.print( - Panel.fit( - "[bold red]⚠ WARNING: This will DELETE ALL DATA[/bold red]\n" - "This script resets the entire Supabase instance.", - title="Supabase init", + """Run Supabase initialisation.""" + reset_mode = "--reset" in sys.argv + + if reset_mode: + console.print( + Panel.fit( + "[bold red]⚠ WARNING: This will DELETE ALL DATA[/bold red]\n" + "This script will reset the entire Supabase instance.", + title="Supabase RESET", + ) ) - ) - # Confirm unless running non-interactively - if sys.stdin.isatty(): - response = console.input("\nType 'yes' to continue: ") - if response.lower() != "yes": - console.print("[yellow]Aborted[/yellow]") - return + # Confirm unless running non-interactively + if sys.stdin.isatty(): + response = console.input("\nType 'yes' to continue: ") + if response.lower() != "yes": + console.print("[yellow]Aborted[/yellow]") + return + + console.print() + + # Reset storage bucket + reset_storage_bucket() + console.print() + + # Drop database schema + engine = reset_database() + console.print() + else: + console.print( + Panel.fit( + "[bold blue]Initialising Supabase[/bold blue]\n" + "This will create tables if they don't exist (safe/idempotent).\n" + "Use [cyan]--reset[/cyan] flag to drop and recreate everything.", + title="Supabase init", + ) + ) + console.print() - console.print() + # Ensure storage bucket exists + ensure_storage_bucket() + console.print() - # Reset storage bucket - reset_storage_bucket() - console.print() + engine = create_engine(settings.database_url, echo=False) - # Reset database and create tables - engine = reset_database() + # Run Alembic migrations to create/update tables + run_alembic_migrations() console.print() # Apply storage policies diff --git a/scripts/seed_nevada.py b/scripts/seed_nevada.py new file mode 100644 index 0000000..0af2cb4 --- /dev/null +++ b/scripts/seed_nevada.py @@ -0,0 +1,128 @@ +"""Seed Nevada datasets into local Supabase. + +This script seeds pre-created Nevada state and congressional district datasets +into the local Supabase database for testing purposes. + +Usage: + uv run python scripts/seed_nevada.py +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from rich.console import Console +from sqlmodel import Session, create_engine, select + +from policyengine_api.config.settings import settings +from policyengine_api.models import Dataset, TaxBenefitModel +from policyengine_api.services.storage import upload_dataset_for_seeding + +console = Console() + +# Nevada datasets location +NEVADA_DATA_DIR = Path(__file__).parent.parent / "test_data" / "nevada_datasets" + + +def main(): + """Seed Nevada datasets.""" + console.print("[bold blue]Seeding Nevada datasets for testing...") + + engine = create_engine(settings.database_url, echo=False) + + with Session(engine) as session: + # Get or create US model + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not us_model: + console.print(" Creating US tax-benefit model...") + us_model = TaxBenefitModel( + name="policyengine-us", + description="US tax-benefit system model", + ) + session.add(us_model) + session.commit() + session.refresh(us_model) + console.print(" [green]✓[/green] Created policyengine-us model") + + # Seed state datasets + states_dir = NEVADA_DATA_DIR / "states" + if states_dir.exists(): + console.print("\n [bold]Nevada State Datasets:[/bold]") + for h5_file in sorted(states_dir.glob("*.h5")): + name = h5_file.stem # e.g., "NV_year_2024" + year = int(name.split("_")[-1]) + + # Check if already exists + existing = session.exec( + select(Dataset).where(Dataset.name == name) + ).first() + + if existing: + console.print(f" [yellow]⏭[/yellow] {name} (already exists)") + continue + + # Upload to storage + console.print(f" Uploading {name}...", end=" ") + try: + object_name = upload_dataset_for_seeding(str(h5_file)) + + # Create database record + db_dataset = Dataset( + name=name, + description=f"Nevada state dataset for year {year}", + filepath=object_name, + year=year, + tax_benefit_model_id=us_model.id, + ) + session.add(db_dataset) + session.commit() + console.print("[green]✓[/green]") + except Exception as e: + console.print(f"[red]✗ {e}[/red]") + + # Seed district datasets + districts_dir = NEVADA_DATA_DIR / "districts" + if districts_dir.exists(): + console.print("\n [bold]Nevada Congressional District Datasets:[/bold]") + for h5_file in sorted(districts_dir.glob("*.h5")): + name = h5_file.stem # e.g., "NV-01_year_2024" + year = int(name.split("_")[-1]) + district = name.split("_")[0] # e.g., "NV-01" + + # Check if already exists + existing = session.exec( + select(Dataset).where(Dataset.name == name) + ).first() + + if existing: + console.print(f" [yellow]⏭[/yellow] {name} (already exists)") + continue + + # Upload to storage + console.print(f" Uploading {name}...", end=" ") + try: + object_name = upload_dataset_for_seeding(str(h5_file)) + + # Create database record + db_dataset = Dataset( + name=name, + description=f"{district} congressional district dataset for year {year}", + filepath=object_name, + year=year, + tax_benefit_model_id=us_model.id, + ) + session.add(db_dataset) + session.commit() + console.print("[green]✓[/green]") + except Exception as e: + console.print(f"[red]✗ {e}[/red]") + + console.print("\n[bold green]✓ Nevada datasets seeded successfully![/bold green]") + + +if __name__ == "__main__": + main() diff --git a/src/policyengine_api/config/settings.py b/src/policyengine_api/config/settings.py index 76a1ab1..efba345 100644 --- a/src/policyengine_api/config/settings.py +++ b/src/policyengine_api/config/settings.py @@ -40,10 +40,21 @@ class Settings(BaseSettings): @property def database_url(self) -> str: - """Get database URL from Supabase.""" + """Get database URL from Supabase. + + For local development, the database runs on port 54322 (not 54321 which is the API). + Use supabase_db_url to override, or rely on the default local URL. + """ + if self.supabase_db_url: + return self.supabase_db_url + + # For local development, default to the standard Supabase local DB port + if "localhost" in self.supabase_url or "127.0.0.1" in self.supabase_url: + return "postgresql://postgres:postgres@127.0.0.1:54322/postgres" + + # For remote Supabase, construct URL from API URL (usually need supabase_db_url set) return ( - self.supabase_db_url - or self.supabase_url.replace( + self.supabase_url.replace( "http://", "postgresql://postgres:postgres@" ).replace("https://", "postgresql://postgres:postgres@") + "/postgres" diff --git a/supabase/.temp/cli-latest b/supabase/.temp/cli-latest index 8c68db7..1dd6178 100644 --- a/supabase/.temp/cli-latest +++ b/supabase/.temp/cli-latest @@ -1 +1 @@ -v2.67.1 \ No newline at end of file +v2.75.0 \ No newline at end of file diff --git a/supabase/migrations/20251229000000_add_parameter_values_indexes.sql b/supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql similarity index 100% rename from supabase/migrations/20251229000000_add_parameter_values_indexes.sql rename to supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql diff --git a/supabase/migrations/20260103000000_add_poverty_inequality.sql b/supabase/migrations_archived/20260103000000_add_poverty_inequality.sql similarity index 100% rename from supabase/migrations/20260103000000_add_poverty_inequality.sql rename to supabase/migrations_archived/20260103000000_add_poverty_inequality.sql diff --git a/supabase/migrations/20260111000000_add_aggregate_status.sql b/supabase/migrations_archived/20260111000000_add_aggregate_status.sql similarity index 100% rename from supabase/migrations/20260111000000_add_aggregate_status.sql rename to supabase/migrations_archived/20260111000000_add_aggregate_status.sql diff --git a/supabase/migrations/20260203000000_create_households.sql b/supabase/migrations_archived/20260203000000_create_households.sql similarity index 100% rename from supabase/migrations/20260203000000_create_households.sql rename to supabase/migrations_archived/20260203000000_create_households.sql diff --git a/supabase/migrations/20260203000001_create_user_household_associations.sql b/supabase/migrations_archived/20260203000001_create_user_household_associations.sql similarity index 100% rename from supabase/migrations/20260203000001_create_user_household_associations.sql rename to supabase/migrations_archived/20260203000001_create_user_household_associations.sql diff --git a/supabase/migrations/20260203000002_simulation_household_support.sql b/supabase/migrations_archived/20260203000002_simulation_household_support.sql similarity index 100% rename from supabase/migrations/20260203000002_simulation_household_support.sql rename to supabase/migrations_archived/20260203000002_simulation_household_support.sql diff --git a/uv.lock b/uv.lock index 094ebf8..466caf4 100644 --- a/uv.lock +++ b/uv.lock @@ -91,6 +91,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "alembic" +version = "1.18.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/41/ab8f624929847b49f84955c594b165855efd829b0c271e1a8cac694138e5/alembic-1.18.3.tar.gz", hash = "sha256:1212aa3778626f2b0f0aa6dd4e99a5f99b94bd25a0c1ac0bba3be65e081e50b0", size = 2052564, upload-time = "2026-01-29T20:24:15.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/8e/d79281f323e7469b060f15bd229e48d7cdd219559e67e71c013720a88340/alembic-1.18.3-py3-none-any.whl", hash = "sha256:12a0359bfc068a4ecbb9b3b02cf77856033abfdb59e4a5aca08b7eacd7b74ddd", size = 262282, upload-time = "2026-01-29T20:24:17.488Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -1057,6 +1071,18 @@ sqlalchemy = [ { name = "opentelemetry-instrumentation-sqlalchemy" }, ] +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1757,6 +1783,7 @@ name = "policyengine-api-v2" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "alembic" }, { name = "anthropic" }, { name = "boto3" }, { name = "fastapi" }, @@ -1793,6 +1820,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13.0" }, { name = "anthropic", specifier = ">=0.40.0" }, { name = "boto3", specifier = ">=1.41.1" }, { name = "fastapi", specifier = ">=0.115.0" }, From 0fb609b523fb7cddc5a0893b7ce60e61711571f2 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 4 Feb 2026 03:53:48 +0300 Subject: [PATCH 07/87] refactor: Make household impact async pattern match economic impact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace synchronous inline calculation with async trigger pattern - Add _trigger_household_impact() mirroring _trigger_economy_comparison() - Add _run_local_household_impact() for local execution (blocking) - Add _run_simulation_in_session() for running individual simulations - Update POST endpoint to trigger and return immediately - Add test script for manual end-to-end testing Note: Local execution blocks the request (same as economic impact). True async requires Modal functions (household_impact_uk/us). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/test_household_impact.py | 135 +++++++++++++++ .../api/household_analysis.py | 161 ++++++++++++------ 2 files changed, 247 insertions(+), 49 deletions(-) create mode 100644 scripts/test_household_impact.py diff --git a/scripts/test_household_impact.py b/scripts/test_household_impact.py new file mode 100644 index 0000000..81c85b0 --- /dev/null +++ b/scripts/test_household_impact.py @@ -0,0 +1,135 @@ +"""Test household impact analysis end-to-end. + +This script tests the async household impact analysis workflow: +1. Create a stored household +2. Run household impact analysis (returns immediately with report_id) +3. Poll until completed +4. Verify results + +Usage: + uv run python scripts/test_household_impact.py +""" + +import sys +import time + +import requests + +BASE_URL = "http://127.0.0.1:8000" + + +def main(): + print("=" * 60) + print("Testing Household Impact Analysis (Async)") + print("=" * 60) + + # Step 1: Create a US household + print("\n1. Creating US household...") + household_data = { + "tax_benefit_model_name": "policyengine_us", + "year": 2024, + "label": "Test household for impact analysis", + "people": [ + { + "age": 35, + "employment_income": 50000, + } + ], + "tax_unit": {}, + "family": {}, + "spm_unit": {}, + "marital_unit": {}, + "household": {"state_code": "NV"}, + } + + resp = requests.post(f"{BASE_URL}/households/", json=household_data) + if resp.status_code != 201: + print(f" FAILED: {resp.status_code} - {resp.text}") + sys.exit(1) + + household = resp.json() + household_id = household["id"] + print(f" Created household: {household_id}") + + # Step 2: Run household impact analysis + print("\n2. Starting household impact analysis...") + impact_request = { + "household_id": household_id, + "policy_id": None, # Single run under current law + } + + resp = requests.post(f"{BASE_URL}/analysis/household-impact", json=impact_request) + if resp.status_code != 200: + print(f" FAILED: {resp.status_code} - {resp.text}") + sys.exit(1) + + result = resp.json() + report_id = result["report_id"] + status = result["status"] + print(f" Report ID: {report_id}") + print(f" Initial status: {status}") + + # Step 3: Poll until completed + print("\n3. Polling for results...") + max_attempts = 30 + for attempt in range(max_attempts): + resp = requests.get(f"{BASE_URL}/analysis/household-impact/{report_id}") + if resp.status_code != 200: + print(f" FAILED: {resp.status_code} - {resp.text}") + sys.exit(1) + + result = resp.json() + status = result["status"].upper() # Normalize to uppercase + print(f" Attempt {attempt + 1}: status={status}") + + if status == "COMPLETED": + break + elif status == "FAILED": + print(f" FAILED: {result.get('error_message', 'Unknown error')}") + sys.exit(1) + + time.sleep(0.5) + else: + print(f" FAILED: Timed out after {max_attempts} attempts") + sys.exit(1) + + # Step 4: Verify results + print("\n4. Verifying results...") + baseline_result = result.get("baseline_result") + if not baseline_result: + print(" FAILED: No baseline result") + sys.exit(1) + + print(f" Baseline result keys: {list(baseline_result.keys())}") + + # Check for expected entity types + expected_entities = ["person", "tax_unit", "spm_unit", "family", "marital_unit", "household"] + for entity in expected_entities: + if entity in baseline_result: + print(f" ✓ {entity}: {len(baseline_result[entity])} entities") + else: + print(f" ✗ {entity}: missing") + + # Look for net_income in person output + if "person" in baseline_result and baseline_result["person"]: + person = baseline_result["person"][0] + if "household_net_income" in person: + print(f" household_net_income: ${person['household_net_income']:,.2f}") + elif "spm_unit_net_income" in person: + print(f" spm_unit_net_income: ${person['spm_unit_net_income']:,.2f}") + + # Step 5: Cleanup - delete household + print("\n5. Cleaning up...") + resp = requests.delete(f"{BASE_URL}/households/{household_id}") + if resp.status_code == 204: + print(f" Deleted household: {household_id}") + else: + print(f" Warning: Failed to delete household: {resp.status_code}") + + print("\n" + "=" * 60) + print("SUCCESS: Household impact analysis working correctly!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/src/policyengine_api/api/household_analysis.py b/src/policyengine_api/api/household_analysis.py index 29ea89e..981d968 100644 --- a/src/policyengine_api/api/household_analysis.py +++ b/src/policyengine_api/api/household_analysis.py @@ -6,11 +6,11 @@ WORKFLOW: 1. Create a stored household: POST /households 2. Optionally create a reform policy: POST /policies -3. Run analysis: POST /analysis/household-impact -4. Results are synchronous - the response includes computed values +3. Run analysis: POST /analysis/household-impact (returns report_id) +4. Poll GET /analysis/household-impact/{report_id} until status="completed" +5. Results include baseline_result, reform_result (if comparison), and impact diff """ -from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Protocol @@ -18,6 +18,7 @@ import logfire from fastapi import APIRouter, Depends, HTTPException +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from pydantic import BaseModel, Field from sqlmodel import Session @@ -38,6 +39,14 @@ _get_or_create_simulation, ) + +def get_traceparent() -> str | None: + """Get the current W3C traceparent header for distributed tracing.""" + carrier: dict[str, str] = {} + TraceContextTextMapPropagator().inject(carrier) + return carrier.get("traceparent") + + router = APIRouter(prefix="/analysis", tags=["analysis"]) @@ -61,7 +70,14 @@ class CountryConfig: US_CONFIG = CountryConfig( name="us", - entity_types=("person", "tax_unit", "spm_unit", "family", "marital_unit", "household"), + entity_types=( + "person", + "tax_unit", + "spm_unit", + "family", + "marital_unit", + "household", + ), ) @@ -326,68 +342,109 @@ def _load_policy_data(policy_id: UUID | None, session: Session) -> dict | None: # ============================================================================= -# Report Orchestration +# Report Orchestration (Async) # ============================================================================= -def trigger_household_report(report_id: UUID, session: Session) -> None: - """Trigger household simulation(s) for a report.""" - report = _load_report(report_id, session) - _mark_report_running(report, session) - - try: - _run_report_simulations(report, session) - _mark_report_completed(report, session) - except Exception as e: - _mark_report_failed(report, e, session) - +def _run_local_household_impact(report_id: str, session: Session) -> None: + """Run household impact analysis locally. -def _load_report(report_id: UUID, session: Session) -> Report: - """Load report or raise error.""" + NOTE: This runs synchronously and blocks the HTTP request when running + locally (agent_use_modal=False). This mirrors the economic impact behavior. + True async execution requires Modal. + """ report = session.get(Report, report_id) if not report: - raise ValueError(f"Report {report_id} not found") - return report - + return -def _mark_report_running(report: Report, session: Session) -> None: - """Mark report as running.""" report.status = ReportStatus.RUNNING session.add(report) session.commit() + try: + # Run baseline simulation + if report.baseline_simulation_id: + _run_simulation_in_session(report.baseline_simulation_id, session) + + # Run reform simulation if present + if report.reform_simulation_id: + _run_simulation_in_session(report.reform_simulation_id, session) -def _mark_report_completed(report: Report, session: Session) -> None: - """Mark report as completed.""" - report.status = ReportStatus.COMPLETED - session.add(report) - session.commit() + report.status = ReportStatus.COMPLETED + session.add(report) + session.commit() + except Exception as e: + report.status = ReportStatus.FAILED + report.error_message = str(e) + session.add(report) + session.commit() -def _mark_report_failed(report: Report, error: Exception, session: Session) -> None: - """Mark report as failed.""" - report.status = ReportStatus.FAILED - report.error_message = str(error) - session.add(report) +def _run_simulation_in_session(simulation_id: UUID, session: Session) -> None: + """Run a single household simulation within an existing session.""" + simulation = session.get(Simulation, simulation_id) + if not simulation or simulation.status != SimulationStatus.PENDING: + return + + household = session.get(Household, simulation.household_id) + if not household: + raise ValueError(f"Household {simulation.household_id} not found") + + policy_data = _load_policy_data(simulation.policy_id, session) + + simulation.status = SimulationStatus.RUNNING + simulation.started_at = datetime.now(timezone.utc) + session.add(simulation) session.commit() + try: + calculator = get_calculator(household.tax_benefit_model_name) + result = calculator(household.household_data, household.year, policy_data) -def _run_report_simulations(report: Report, session: Session) -> None: - """Run all pending simulations for a report.""" - _run_simulation_if_pending(report.baseline_simulation_id, session) + simulation.household_result = result + simulation.status = SimulationStatus.COMPLETED + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + except Exception as e: + simulation.status = SimulationStatus.FAILED + simulation.error_message = str(e) + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + raise - if report.reform_simulation_id: - _run_simulation_if_pending(report.reform_simulation_id, session) +def _trigger_household_impact( + report_id: str, tax_benefit_model_name: str, session: Session | None = None +) -> None: + """Trigger household impact calculation (local or Modal based on settings).""" + from policyengine_api.config import settings + + traceparent = get_traceparent() -def _run_simulation_if_pending(simulation_id: UUID | None, session: Session) -> None: - """Run simulation if it exists and is pending.""" - if not simulation_id: - return + if not settings.agent_use_modal and session is not None: + # Run locally (blocking - see _run_local_household_impact docstring) + _run_local_household_impact(report_id, session) + else: + # Use Modal + import modal - simulation = session.get(Simulation, simulation_id) - if simulation and simulation.status == SimulationStatus.PENDING: - run_household_simulation(simulation.id, session) + if tax_benefit_model_name == "policyengine_uk": + fn = modal.Function.from_name("policyengine", "household_impact_uk") + else: + fn = modal.Function.from_name("policyengine", "household_impact_us") + + fn.spawn(report_id=report_id, traceparent=traceparent) + + +# Legacy functions kept for compatibility +def _load_report(report_id: UUID, session: Session) -> Report: + """Load report or raise error.""" + report = session.get(Report, report_id) + if not report: + raise ValueError(f"Report {report_id} not found") + return report # ============================================================================= @@ -436,7 +493,9 @@ class HouseholdImpactResponse(BaseModel): # ============================================================================= -def build_simulation_info(simulation: Simulation | None) -> HouseholdSimulationInfo | None: +def build_simulation_info( + simulation: Simulation | None, +) -> HouseholdSimulationInfo | None: """Build simulation info from a simulation.""" if not simulation: return None @@ -540,7 +599,9 @@ def household_impact( If policy_id is None: single run under current law. If policy_id is set: comparison (baseline vs reform). - This is a synchronous operation for household calculations. + This is an async operation. The endpoint returns immediately with a report_id + and status="pending". Poll GET /analysis/household-impact/{report_id} until + status="completed" to get results. """ household = validate_household_exists(request.household_id, session) validate_policy_exists(request.policy_id, session) @@ -564,8 +625,10 @@ def household_impact( ) if report.status == ReportStatus.PENDING: - with logfire.span("trigger_household_report", job_id=str(report.id)): - trigger_household_report(report.id, session) + with logfire.span("trigger_household_impact", job_id=str(report.id)): + _trigger_household_impact( + str(report.id), household.tax_benefit_model_name, session + ) return build_household_response(report, baseline_sim, reform_sim, session) From dbb65345be565170049d59e1b1941dd0a8834db8 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 4 Feb 2026 20:06:01 +0300 Subject: [PATCH 08/87] fix: Break up Alembic; add smaller seed scripts --- ...ema.py => 20260204_0001_initial_schema.py} | 84 +-- .../20260204_0002_add_household_support.py | 170 +++++ ...0204_0003_add_parameter_values_indexes.py} | 8 +- scripts/seed.py | 107 +-- scripts/seed_common.py | 362 +++++++++ scripts/seed_policies.py | 143 ++++ scripts/seed_uk_datasets.py | 113 +++ scripts/seed_uk_model.py | 33 + scripts/seed_us_datasets.py | 108 +++ scripts/seed_us_model.py | 33 + src/policyengine_api/modal_app.py | 689 +++++++++++++++++- 11 files changed, 1723 insertions(+), 127 deletions(-) rename alembic/versions/{20260204_d6e30d3b834d_initial_schema.py => 20260204_0001_initial_schema.py} (87%) create mode 100644 alembic/versions/20260204_0002_add_household_support.py rename alembic/versions/{20260204_a17ac554f4aa_add_parameter_values_indexes.py => 20260204_0003_add_parameter_values_indexes.py} (90%) create mode 100644 scripts/seed_common.py create mode 100644 scripts/seed_policies.py create mode 100644 scripts/seed_uk_datasets.py create mode 100644 scripts/seed_uk_model.py create mode 100644 scripts/seed_us_datasets.py create mode 100644 scripts/seed_us_model.py diff --git a/alembic/versions/20260204_d6e30d3b834d_initial_schema.py b/alembic/versions/20260204_0001_initial_schema.py similarity index 87% rename from alembic/versions/20260204_d6e30d3b834d_initial_schema.py rename to alembic/versions/20260204_0001_initial_schema.py index d4de071..273124a 100644 --- a/alembic/versions/20260204_d6e30d3b834d_initial_schema.py +++ b/alembic/versions/20260204_0001_initial_schema.py @@ -1,11 +1,11 @@ -"""Initial schema +"""Initial schema (main branch state) -Revision ID: d6e30d3b834d +Revision ID: 0001_initial Revises: -Create Date: 2026-02-04 02:15:03.471607 +Create Date: 2026-02-04 -This migration creates all base tables for the PolicyEngine API. -Tables are organized by dependency tier to ensure proper creation order. +This migration creates all base tables for the PolicyEngine API as they +exist on the main branch, BEFORE the household CRUD changes. """ from typing import Sequence, Union @@ -14,14 +14,14 @@ from alembic import op # revision identifiers, used by Alembic. -revision: str = "d6e30d3b834d" +revision: str = "0001_initial" down_revision: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - """Create all tables.""" + """Create all tables as they exist on main branch.""" # ======================================================================== # TIER 1: Tables with no foreign key dependencies # ======================================================================== @@ -215,33 +215,6 @@ def upgrade() -> None: sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), ) - # Households (stored household definitions) - op.create_table( - "households", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_name", sa.String(), nullable=False), - sa.Column("year", sa.Integer(), nullable=False), - sa.Column("label", sa.String(), nullable=True), - sa.Column("household_data", sa.JSON(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - "idx_households_model_name", "households", ["tax_benefit_model_name"] - ) - op.create_index("idx_households_year", "households", ["year"]) - # ======================================================================== # TIER 4: Tables depending on tier 3 # ======================================================================== @@ -268,13 +241,11 @@ def upgrade() -> None: sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), ) - # Simulations (economy or household calculations) + # Simulations (economy calculations) - NOTE: No household support yet op.create_table( "simulations", sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_type", sa.String(), nullable=False, default="economy"), - sa.Column("dataset_id", sa.Uuid(), nullable=True), - sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("dataset_id", sa.Uuid(), nullable=False), # Required in main sa.Column("policy_id", sa.Uuid(), nullable=True), sa.Column("dynamic_id", sa.Uuid(), nullable=True), sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), @@ -295,10 +266,8 @@ def upgrade() -> None: ), sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("household_result", sa.JSON(), nullable=True), sa.PrimaryKeyConstraint("id"), sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), - sa.ForeignKeyConstraint(["household_id"], ["households.id"]), sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), sa.ForeignKeyConstraint( @@ -307,31 +276,7 @@ def upgrade() -> None: sa.ForeignKeyConstraint(["output_dataset_id"], ["datasets.id"]), ) - # User-household associations - op.create_table( - "user_household_associations", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=False), - sa.Column("household_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["household_id"], ["households.id"], ondelete="CASCADE"), - sa.UniqueConstraint("user_id", "household_id"), - ) - op.create_index( - "idx_user_household_user", "user_household_associations", ["user_id"] - ) - op.create_index( - "idx_user_household_household", "user_household_associations", ["household_id"] - ) - - # Household jobs (async household calculations) + # Household jobs (async household calculations) - legacy approach op.create_table( "household_jobs", sa.Column("id", sa.Uuid(), nullable=False), @@ -359,13 +304,12 @@ def upgrade() -> None: # TIER 5: Tables depending on simulations # ======================================================================== - # Reports (analysis reports) + # Reports (analysis reports) - NOTE: No report_type yet op.create_table( "reports", sa.Column("id", sa.Uuid(), nullable=False), sa.Column("label", sa.String(), nullable=False), sa.Column("description", sa.String(), nullable=True), - sa.Column("report_type", sa.String(), nullable=True), sa.Column("user_id", sa.Uuid(), nullable=True), sa.Column("markdown", sa.Text(), nullable=True), sa.Column("parent_report_id", sa.Uuid(), nullable=True), @@ -573,16 +517,10 @@ def downgrade() -> None: # Tier 4 op.drop_table("household_jobs") - op.drop_index("idx_user_household_household", "user_household_associations") - op.drop_index("idx_user_household_user", "user_household_associations") - op.drop_table("user_household_associations") op.drop_table("simulations") op.drop_table("parameter_values") # Tier 3 - op.drop_index("idx_households_year", "households") - op.drop_index("idx_households_model_name", "households") - op.drop_table("households") op.drop_table("dataset_versions") op.drop_table("variables") op.drop_table("parameters") diff --git a/alembic/versions/20260204_0002_add_household_support.py b/alembic/versions/20260204_0002_add_household_support.py new file mode 100644 index 0000000..beb00a0 --- /dev/null +++ b/alembic/versions/20260204_0002_add_household_support.py @@ -0,0 +1,170 @@ +"""Add household CRUD and impact analysis support + +Revision ID: 0002_household +Revises: 0001_initial +Create Date: 2026-02-04 + +This migration adds support for: +- Storing household definitions (households table) +- User-household associations for saved households +- Household-based simulations (adds household_id to simulations) +- Household impact reports (adds report_type to reports) +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0002_household" +down_revision: Union[str, Sequence[str], None] = "0001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add household support.""" + # ======================================================================== + # NEW TABLES + # ======================================================================== + + # Households (stored household definitions) + op.create_table( + "households", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_name", sa.String(), nullable=False), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("label", sa.String(), nullable=True), + sa.Column("household_data", sa.JSON(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "idx_households_model_name", "households", ["tax_benefit_model_name"] + ) + op.create_index("idx_households_year", "households", ["year"]) + + # User-household associations (many-to-many for saved households) + op.create_table( + "user_household_associations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("household_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["household_id"], ["households.id"], ondelete="CASCADE"), + sa.UniqueConstraint("user_id", "household_id"), + ) + op.create_index( + "idx_user_household_user", "user_household_associations", ["user_id"] + ) + op.create_index( + "idx_user_household_household", "user_household_associations", ["household_id"] + ) + + # ======================================================================== + # MODIFY SIMULATIONS TABLE + # ======================================================================== + + # Add simulation_type column (economy vs household) + op.add_column( + "simulations", + sa.Column( + "simulation_type", + sa.String(), + nullable=False, + server_default="economy", + ), + ) + + # Add household_id column (for household simulations) + op.add_column( + "simulations", + sa.Column("household_id", sa.Uuid(), nullable=True), + ) + op.create_foreign_key( + "fk_simulations_household_id", + "simulations", + "households", + ["household_id"], + ["id"], + ) + + # Add household_result column (stores household calculation results) + op.add_column( + "simulations", + sa.Column("household_result", sa.JSON(), nullable=True), + ) + + # Make dataset_id nullable (household simulations don't need a dataset) + op.alter_column( + "simulations", + "dataset_id", + existing_type=sa.Uuid(), + nullable=True, + ) + + # ======================================================================== + # MODIFY REPORTS TABLE + # ======================================================================== + + # Add report_type column (economy_comparison, household_impact, etc.) + op.add_column( + "reports", + sa.Column("report_type", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + """Remove household support.""" + # ======================================================================== + # REVERT REPORTS TABLE + # ======================================================================== + op.drop_column("reports", "report_type") + + # ======================================================================== + # REVERT SIMULATIONS TABLE + # ======================================================================== + + # Make dataset_id required again + op.alter_column( + "simulations", + "dataset_id", + existing_type=sa.Uuid(), + nullable=False, + ) + + # Remove household columns + op.drop_column("simulations", "household_result") + op.drop_constraint("fk_simulations_household_id", "simulations", type_="foreignkey") + op.drop_column("simulations", "household_id") + op.drop_column("simulations", "simulation_type") + + # ======================================================================== + # DROP NEW TABLES + # ======================================================================== + op.drop_index("idx_user_household_household", "user_household_associations") + op.drop_index("idx_user_household_user", "user_household_associations") + op.drop_table("user_household_associations") + + op.drop_index("idx_households_year", "households") + op.drop_index("idx_households_model_name", "households") + op.drop_table("households") diff --git a/alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py b/alembic/versions/20260204_0003_add_parameter_values_indexes.py similarity index 90% rename from alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py rename to alembic/versions/20260204_0003_add_parameter_values_indexes.py index e1967c2..53518cf 100644 --- a/alembic/versions/20260204_a17ac554f4aa_add_parameter_values_indexes.py +++ b/alembic/versions/20260204_0003_add_parameter_values_indexes.py @@ -1,7 +1,7 @@ """Add parameter_values indexes -Revision ID: a17ac554f4aa -Revises: d6e30d3b834d +Revision ID: 0003_param_idx +Revises: 0002_household Create Date: 2026-02-04 02:20:00.000000 This migration adds performance indexes to the parameter_values table @@ -13,8 +13,8 @@ from alembic import op # revision identifiers, used by Alembic. -revision: str = "a17ac554f4aa" -down_revision: Union[str, Sequence[str], None] = "d6e30d3b834d" +revision: str = "0003_param_idx" +down_revision: Union[str, Sequence[str], None] = "0002_household" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/scripts/seed.py b/scripts/seed.py index f3fbfa8..4274528 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -363,7 +363,7 @@ def seed_model(model_version, session, lite: bool = False) -> TaxBenefitModelVer return db_version -def seed_datasets(session, lite: bool = False): +def seed_datasets(session, lite: bool = False, skip_uk_datasets: bool = False): """Seed datasets and upload to S3.""" with logfire.span("seed_datasets"): mode_str = " (lite mode - 2026 only)" if lite else "" @@ -383,60 +383,64 @@ def seed_datasets(session, lite: bool = False): ) return - # UK datasets - console.print(" Creating UK datasets...") data_folder = str(Path(__file__).parent.parent / "data") - uk_datasets = ensure_uk_datasets(data_folder=data_folder) - - # In lite mode, only upload FRS 2026 - if lite: - uk_datasets = { - k: v for k, v in uk_datasets.items() if v.year == 2026 and "frs" in k - } - console.print(f" Lite mode: filtered to {len(uk_datasets)} dataset(s)") + # UK datasets uk_created = 0 uk_skipped = 0 - with logfire.span("seed_uk_datasets", count=len(uk_datasets)): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task("UK datasets", total=len(uk_datasets)) - for _, pe_dataset in uk_datasets.items(): - progress.update(task, description=f"UK: {pe_dataset.name}") - - # Check if dataset already exists - existing = session.exec( - select(Dataset).where(Dataset.name == pe_dataset.name) - ).first() - - if existing: - uk_skipped += 1 + if skip_uk_datasets: + console.print(" [yellow]Skipping UK datasets (--skip-uk-datasets)[/yellow]") + else: + console.print(" Creating UK datasets...") + uk_datasets = ensure_uk_datasets(data_folder=data_folder) + + # In lite mode, only upload FRS 2026 + if lite: + uk_datasets = { + k: v for k, v in uk_datasets.items() if v.year == 2026 and "frs" in k + } + console.print(f" Lite mode: filtered to {len(uk_datasets)} dataset(s)") + + with logfire.span("seed_uk_datasets", count=len(uk_datasets)): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("UK datasets", total=len(uk_datasets)) + for _, pe_dataset in uk_datasets.items(): + progress.update(task, description=f"UK: {pe_dataset.name}") + + # Check if dataset already exists + existing = session.exec( + select(Dataset).where(Dataset.name == pe_dataset.name) + ).first() + + if existing: + uk_skipped += 1 + progress.advance(task) + continue + + # Upload to S3 + object_name = upload_dataset_for_seeding(pe_dataset.filepath) + + # Create database record + db_dataset = Dataset( + name=pe_dataset.name, + description=pe_dataset.description, + filepath=object_name, + year=pe_dataset.year, + tax_benefit_model_id=uk_model.id, + ) + session.add(db_dataset) + session.commit() + uk_created += 1 progress.advance(task) - continue - - # Upload to S3 - object_name = upload_dataset_for_seeding(pe_dataset.filepath) - - # Create database record - db_dataset = Dataset( - name=pe_dataset.name, - description=pe_dataset.description, - filepath=object_name, - year=pe_dataset.year, - tax_benefit_model_id=uk_model.id, - ) - session.add(db_dataset) - session.commit() - uk_created += 1 - progress.advance(task) - console.print( - f" [green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped" - ) + console.print( + f" [green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped" + ) # US datasets console.print(" Creating US datasets...") @@ -622,6 +626,11 @@ def main(): action="store_true", help="Lite mode: skip US state parameters, only seed FRS 2026 and CPS 2026 datasets", ) + parser.add_argument( + "--skip-uk-datasets", + action="store_true", + help="Skip UK datasets (useful when HuggingFace token is not available)", + ) args = parser.parse_args() with logfire.span("database_seeding"): @@ -638,7 +647,7 @@ def main(): console.print(f"[green]✓[/green] US model seeded: {us_version.id}\n") # Seed datasets - seed_datasets(session, lite=args.lite) + seed_datasets(session, lite=args.lite, skip_uk_datasets=args.skip_uk_datasets) # Seed example policies seed_example_policies(session) diff --git a/scripts/seed_common.py b/scripts/seed_common.py new file mode 100644 index 0000000..4e4e2ec --- /dev/null +++ b/scripts/seed_common.py @@ -0,0 +1,362 @@ +"""Shared utilities for seed scripts.""" + +import io +import json +import logging +import math +import sys +import warnings +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import logfire +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn +from sqlmodel import Session, create_engine + +# Disable all SQLAlchemy and database logging BEFORE any imports +logging.basicConfig(level=logging.ERROR) +logging.getLogger("sqlalchemy").setLevel(logging.ERROR) +warnings.filterwarnings("ignore") + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from policyengine_api.config.settings import settings # noqa: E402 + +# Configure logfire +if settings.logfire_token: + logfire.configure( + token=settings.logfire_token, + environment=settings.logfire_environment, + console=False, + ) + +console = Console() + + +def get_session(): + """Get database session with logging disabled.""" + engine = create_engine(settings.database_url, echo=False) + return Session(engine) + + +def bulk_insert(session, table: str, columns: list[str], rows: list[dict]): + """Fast bulk insert using PostgreSQL COPY via StringIO.""" + if not rows: + return + + # Get raw psycopg2 connection + connection = session.connection() + raw_conn = connection.connection.dbapi_connection + cursor = raw_conn.cursor() + + # Build CSV-like data in memory + output = io.StringIO() + for row in rows: + values = [] + for col in columns: + val = row[col] + if val is None: + values.append("\\N") + elif isinstance(val, str): + # Escape special characters for COPY + val = ( + val.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n") + ) + values.append(val) + else: + values.append(str(val)) + output.write("\t".join(values) + "\n") + + output.seek(0) + + # COPY is the fastest way to bulk load PostgreSQL + cursor.copy_from(output, table, columns=columns, null="\\N") + session.commit() + + +def seed_model(model_version, session, lite: bool = False): + """Seed a tax-benefit model with its variables and parameters. + + Returns the TaxBenefitModelVersion that was created or found. + """ + from policyengine_api.models import ( + TaxBenefitModel, + TaxBenefitModelVersion, + ) + from sqlmodel import select + + with logfire.span( + "seed_model", + model=model_version.model.id, + version=model_version.version, + ): + # Create or get the model + console.print(f"[bold blue]Seeding {model_version.model.id}...") + + existing_model = session.exec( + select(TaxBenefitModel).where( + TaxBenefitModel.name == model_version.model.id + ) + ).first() + + if existing_model: + db_model = existing_model + console.print(f" Using existing model: {db_model.id}") + else: + db_model = TaxBenefitModel( + name=model_version.model.id, + description=model_version.model.description, + ) + session.add(db_model) + session.commit() + session.refresh(db_model) + console.print(f" Created model: {db_model.id}") + + # Create model version + existing_version = session.exec( + select(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == db_model.id, + TaxBenefitModelVersion.version == model_version.version, + ) + ).first() + + if existing_version: + console.print( + f" Model version {model_version.version} already exists, skipping" + ) + return existing_version + + db_version = TaxBenefitModelVersion( + model_id=db_model.id, + version=model_version.version, + description=f"Version {model_version.version}", + ) + session.add(db_version) + session.commit() + session.refresh(db_version) + console.print(f" Created version: {db_version.version}") + + # Add variables + with logfire.span("add_variables", count=len(model_version.variables)): + var_rows = [] + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task( + f"Preparing {len(model_version.variables)} variables", + total=len(model_version.variables), + ) + for var in model_version.variables: + var_rows.append( + { + "id": uuid4(), + "name": var.name, + "entity": var.entity, + "description": var.description or "", + "data_type": var.data_type.__name__ + if hasattr(var.data_type, "__name__") + else str(var.data_type), + "possible_values": None, + "tax_benefit_model_version_id": db_version.id, + "created_at": datetime.now(timezone.utc), + } + ) + progress.advance(task) + + console.print(f" Inserting {len(var_rows)} variables...") + bulk_insert( + session, + "variables", + [ + "id", + "name", + "entity", + "description", + "data_type", + "possible_values", + "tax_benefit_model_version_id", + "created_at", + ], + var_rows, + ) + + console.print( + f" [green]✓[/green] Added {len(model_version.variables)} variables" + ) + + # Add parameters (only user-facing ones: those with labels) + # Deduplicate by name - keep first occurrence + # + # WHY DEDUPLICATION IS NEEDED: + # The policyengine package can provide multiple parameter entries with the same + # name. This happens because parameters can have multiple bracket entries or + # state-specific variants that share the same base name. We keep only the first + # occurrence to avoid database unique constraint violations and reduce redundancy. + # + # In lite mode, exclude US state parameters (gov.states.*) + seen_names = set() + parameters_to_add = [] + skipped_state_params = 0 + skipped_no_label = 0 + skipped_duplicate = 0 + + for p in model_version.parameters: + if p.label is None: + skipped_no_label += 1 + continue + if p.name in seen_names: + skipped_duplicate += 1 + continue + # In lite mode, skip state-level parameters for faster seeding + if lite and p.name.startswith("gov.states."): + skipped_state_params += 1 + continue + parameters_to_add.append(p) + seen_names.add(p.name) + + console.print(f" Parameter filtering:") + console.print(f" - Total from source: {len(model_version.parameters)}") + console.print(f" - Skipped (no label): {skipped_no_label}") + console.print(f" - Skipped (duplicate name): {skipped_duplicate}") + if lite and skipped_state_params > 0: + console.print(f" - Skipped (state params, lite mode): {skipped_state_params}") + console.print(f" - To add: {len(parameters_to_add)}") + + with logfire.span("add_parameters", count=len(parameters_to_add)): + # Build list of parameter dicts for bulk insert + param_rows = [] + param_names = [] # Track (pe_id, name, generated_uuid) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task( + f"Preparing {len(parameters_to_add)} parameters", + total=len(parameters_to_add), + ) + for param in parameters_to_add: + param_uuid = uuid4() + param_rows.append( + { + "id": param_uuid, + "name": param.name, + "label": param.label if hasattr(param, "label") else None, + "description": param.description or "", + "data_type": param.data_type.__name__ + if hasattr(param.data_type, "__name__") + else str(param.data_type), + "unit": param.unit, + "tax_benefit_model_version_id": db_version.id, + "created_at": datetime.now(timezone.utc), + } + ) + param_names.append((param.id, param.name, param_uuid)) + progress.advance(task) + + console.print(f" Inserting {len(param_rows)} parameters...") + bulk_insert( + session, + "parameters", + [ + "id", + "name", + "label", + "description", + "data_type", + "unit", + "tax_benefit_model_version_id", + "created_at", + ], + param_rows, + ) + + # Build param_id_map from pre-generated UUIDs + param_id_map = {pe_id: db_uuid for pe_id, name, db_uuid in param_names} + + console.print( + f" [green]✓[/green] Added {len(parameters_to_add)} parameters" + ) + + # Add parameter values + # Filter to only include values for parameters we added + parameter_values_to_add = [ + pv + for pv in model_version.parameter_values + if pv.parameter.id in param_id_map + ] + console.print(f" Found {len(parameter_values_to_add)} parameter values to add") + + with logfire.span("add_parameter_values", count=len(parameter_values_to_add)): + pv_rows = [] + skipped = 0 + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task( + f"Preparing {len(parameter_values_to_add)} parameter values", + total=len(parameter_values_to_add), + ) + for pv in parameter_values_to_add: + # Handle Infinity values - skip them as they can't be stored in JSON + if isinstance(pv.value, float) and ( + math.isinf(pv.value) or math.isnan(pv.value) + ): + skipped += 1 + progress.advance(task) + continue + + # Source data has dates swapped (start > end), fix ordering + # Only swap if both dates are set, otherwise keep original + if pv.start_date and pv.end_date: + start = pv.end_date # Swap: source end is our start + end = pv.start_date # Swap: source start is our end + else: + start = pv.start_date + end = pv.end_date + pv_rows.append( + { + "id": uuid4(), + "parameter_id": param_id_map[pv.parameter.id], + "value_json": json.dumps(pv.value), + "start_date": start, + "end_date": end, + "policy_id": None, + "dynamic_id": None, + "created_at": datetime.now(timezone.utc), + } + ) + progress.advance(task) + + console.print(f" Inserting {len(pv_rows)} parameter values...") + bulk_insert( + session, + "parameter_values", + [ + "id", + "parameter_id", + "value_json", + "start_date", + "end_date", + "policy_id", + "dynamic_id", + "created_at", + ], + pv_rows, + ) + + console.print( + f" [green]✓[/green] Added {len(pv_rows)} parameter values" + + (f" (skipped {skipped} invalid)" if skipped else "") + ) + + return db_version diff --git a/scripts/seed_policies.py b/scripts/seed_policies.py new file mode 100644 index 0000000..e57b964 --- /dev/null +++ b/scripts/seed_policies.py @@ -0,0 +1,143 @@ +"""Seed example policy reforms for UK and US.""" + +import time +from datetime import datetime, timezone + +import logfire +from sqlmodel import select + +from seed_common import console, get_session + + +def main(): + from policyengine_api.models import ( + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + ) + + console.print("[bold green]Seeding example policies...[/bold green]\n") + + start = time.time() + with get_session() as session: + with logfire.span("seed_example_policies"): + # Get model versions + uk_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") + ).first() + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not uk_model or not us_model: + console.print( + "[red]Error: UK or US model not found. Run seed_*_model.py first.[/red]" + ) + return + + uk_version = session.exec( + select(TaxBenefitModelVersion) + .where(TaxBenefitModelVersion.model_id == uk_model.id) + .order_by(TaxBenefitModelVersion.created_at.desc()) + ).first() + + us_version = session.exec( + select(TaxBenefitModelVersion) + .where(TaxBenefitModelVersion.model_id == us_model.id) + .order_by(TaxBenefitModelVersion.created_at.desc()) + ).first() + + # UK example policy: raise basic rate to 22p + uk_policy_name = "UK basic rate 22p" + existing_uk_policy = session.exec( + select(Policy).where(Policy.name == uk_policy_name) + ).first() + + if existing_uk_policy: + console.print(f" Policy '{uk_policy_name}' already exists, skipping") + else: + # Find the basic rate parameter + uk_basic_rate_param = session.exec( + select(Parameter).where( + Parameter.name == "gov.hmrc.income_tax.rates.uk[0].rate", + Parameter.tax_benefit_model_version_id == uk_version.id, + ) + ).first() + + if uk_basic_rate_param: + uk_policy = Policy( + name=uk_policy_name, + description="Raise the UK income tax basic rate from 20p to 22p", + ) + session.add(uk_policy) + session.commit() + session.refresh(uk_policy) + + # Add parameter value (22% = 0.22) + uk_param_value = ParameterValue( + parameter_id=uk_basic_rate_param.id, + value_json={"value": 0.22}, + start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), + end_date=None, + policy_id=uk_policy.id, + ) + session.add(uk_param_value) + session.commit() + console.print(f" [green]✓[/green] Created UK policy: {uk_policy_name}") + else: + console.print( + " [yellow]Warning: UK basic rate parameter not found[/yellow]" + ) + + # US example policy: raise first bracket rate to 12% + us_policy_name = "US 12% lowest bracket" + existing_us_policy = session.exec( + select(Policy).where(Policy.name == us_policy_name) + ).first() + + if existing_us_policy: + console.print(f" Policy '{us_policy_name}' already exists, skipping") + else: + # Find the first bracket rate parameter + us_first_bracket_param = session.exec( + select(Parameter).where( + Parameter.name == "gov.irs.income.bracket.rates.1", + Parameter.tax_benefit_model_version_id == us_version.id, + ) + ).first() + + if us_first_bracket_param: + us_policy = Policy( + name=us_policy_name, + description="Raise US federal income tax lowest bracket to 12%", + ) + session.add(us_policy) + session.commit() + session.refresh(us_policy) + + # Add parameter value (12% = 0.12) + us_param_value = ParameterValue( + parameter_id=us_first_bracket_param.id, + value_json={"value": 0.12}, + start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), + end_date=None, + policy_id=us_policy.id, + ) + session.add(us_param_value) + session.commit() + console.print(f" [green]✓[/green] Created US policy: {us_policy_name}") + else: + console.print( + " [yellow]Warning: US first bracket parameter not found[/yellow]" + ) + + console.print("[green]✓[/green] Example policies seeded") + + elapsed = time.time() - start + console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_uk_datasets.py b/scripts/seed_uk_datasets.py new file mode 100644 index 0000000..1754454 --- /dev/null +++ b/scripts/seed_uk_datasets.py @@ -0,0 +1,113 @@ +"""Seed UK datasets (FRS) and upload to S3. + +NOTE: Requires HUGGING_FACE_TOKEN environment variable to be set, +as UK FRS datasets are hosted on a private HuggingFace repository. +""" + +import argparse +import time +from pathlib import Path + +import logfire +from rich.progress import Progress, SpinnerColumn, TextColumn +from sqlmodel import select + +from seed_common import console, get_session + + +def main(): + parser = argparse.ArgumentParser(description="Seed UK datasets") + parser.add_argument( + "--lite", + action="store_true", + help="Lite mode: only seed FRS 2026", + ) + args = parser.parse_args() + + # Import here to avoid slow import at module level + from policyengine.tax_benefit_models.uk.datasets import ( + ensure_datasets as ensure_uk_datasets, + ) + + from policyengine_api.models import Dataset, TaxBenefitModel + from policyengine_api.services.storage import upload_dataset_for_seeding + + console.print("[bold green]Seeding UK datasets...[/bold green]\n") + console.print("[yellow]Note: Requires HUGGING_FACE_TOKEN environment variable[/yellow]\n") + + start = time.time() + with get_session() as session: + # Get UK model + uk_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") + ).first() + + if not uk_model: + console.print("[red]Error: UK model not found. Run seed_uk_model.py first.[/red]") + return + + data_folder = str(Path(__file__).parent.parent / "data") + console.print(f" Data folder: {data_folder}") + + # Get datasets + console.print(" Loading UK datasets from policyengine package...") + ds_start = time.time() + uk_datasets = ensure_uk_datasets(data_folder=data_folder) + console.print(f" Loaded {len(uk_datasets)} datasets in {time.time() - ds_start:.1f}s") + + # In lite mode, only upload FRS 2026 + if args.lite: + uk_datasets = { + k: v for k, v in uk_datasets.items() if v.year == 2026 and "frs" in k + } + console.print(f" Lite mode: filtered to {len(uk_datasets)} dataset(s)") + + created = 0 + skipped = 0 + + with logfire.span("seed_uk_datasets", count=len(uk_datasets)): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("UK datasets", total=len(uk_datasets)) + for name, pe_dataset in uk_datasets.items(): + progress.update(task, description=f"UK: {pe_dataset.name}") + + # Check if dataset already exists + existing = session.exec( + select(Dataset).where(Dataset.name == pe_dataset.name) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Upload to S3 + upload_start = time.time() + object_name = upload_dataset_for_seeding(pe_dataset.filepath) + console.print(f" Uploaded {pe_dataset.name} in {time.time() - upload_start:.1f}s") + + # Create database record + db_dataset = Dataset( + name=pe_dataset.name, + description=pe_dataset.description, + filepath=object_name, + year=pe_dataset.year, + tax_benefit_model_id=uk_model.id, + ) + session.add(db_dataset) + session.commit() + created += 1 + progress.advance(task) + + console.print(f"[green]✓[/green] UK datasets: {created} created, {skipped} skipped") + + elapsed = time.time() - start + console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_uk_model.py b/scripts/seed_uk_model.py new file mode 100644 index 0000000..07543bf --- /dev/null +++ b/scripts/seed_uk_model.py @@ -0,0 +1,33 @@ +"""Seed UK model (variables, parameters, parameter values).""" + +import argparse +import time + +from seed_common import console, get_session, seed_model + + +def main(): + parser = argparse.ArgumentParser(description="Seed UK model") + parser.add_argument( + "--lite", + action="store_true", + help="Lite mode: skip state parameters", + ) + args = parser.parse_args() + + # Import here to avoid slow import at module level + from policyengine.tax_benefit_models.uk import uk_latest + + console.print("[bold green]Seeding UK model...[/bold green]\n") + + start = time.time() + with get_session() as session: + version = seed_model(uk_latest, session, lite=args.lite) + console.print(f"[green]✓[/green] UK model seeded: {version.id}") + + elapsed = time.time() - start + console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_us_datasets.py b/scripts/seed_us_datasets.py new file mode 100644 index 0000000..abf1995 --- /dev/null +++ b/scripts/seed_us_datasets.py @@ -0,0 +1,108 @@ +"""Seed US datasets (CPS) and upload to S3.""" + +import argparse +import time +from pathlib import Path + +import logfire +from rich.progress import Progress, SpinnerColumn, TextColumn +from sqlmodel import select + +from seed_common import console, get_session + + +def main(): + parser = argparse.ArgumentParser(description="Seed US datasets") + parser.add_argument( + "--lite", + action="store_true", + help="Lite mode: only seed CPS 2026", + ) + args = parser.parse_args() + + # Import here to avoid slow import at module level + from policyengine.tax_benefit_models.us.datasets import ( + ensure_datasets as ensure_us_datasets, + ) + + from policyengine_api.models import Dataset, TaxBenefitModel + from policyengine_api.services.storage import upload_dataset_for_seeding + + console.print("[bold green]Seeding US datasets...[/bold green]\n") + + start = time.time() + with get_session() as session: + # Get US model + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not us_model: + console.print("[red]Error: US model not found. Run seed_us_model.py first.[/red]") + return + + data_folder = str(Path(__file__).parent.parent / "data") + console.print(f" Data folder: {data_folder}") + + # Get datasets + console.print(" Loading US datasets from policyengine package...") + ds_start = time.time() + us_datasets = ensure_us_datasets(data_folder=data_folder) + console.print(f" Loaded {len(us_datasets)} datasets in {time.time() - ds_start:.1f}s") + + # In lite mode, only upload CPS 2026 + if args.lite: + us_datasets = { + k: v for k, v in us_datasets.items() if v.year == 2026 and "cps" in k + } + console.print(f" Lite mode: filtered to {len(us_datasets)} dataset(s)") + + created = 0 + skipped = 0 + + with logfire.span("seed_us_datasets", count=len(us_datasets)): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("US datasets", total=len(us_datasets)) + for name, pe_dataset in us_datasets.items(): + progress.update(task, description=f"US: {pe_dataset.name}") + + # Check if dataset already exists + existing = session.exec( + select(Dataset).where(Dataset.name == pe_dataset.name) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Upload to S3 + upload_start = time.time() + object_name = upload_dataset_for_seeding(pe_dataset.filepath) + console.print(f" Uploaded {pe_dataset.name} in {time.time() - upload_start:.1f}s") + + # Create database record + db_dataset = Dataset( + name=pe_dataset.name, + description=pe_dataset.description, + filepath=object_name, + year=pe_dataset.year, + tax_benefit_model_id=us_model.id, + ) + session.add(db_dataset) + session.commit() + created += 1 + progress.advance(task) + + console.print(f"[green]✓[/green] US datasets: {created} created, {skipped} skipped") + + elapsed = time.time() - start + console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_us_model.py b/scripts/seed_us_model.py new file mode 100644 index 0000000..ce8a829 --- /dev/null +++ b/scripts/seed_us_model.py @@ -0,0 +1,33 @@ +"""Seed US model (variables, parameters, parameter values).""" + +import argparse +import time + +from seed_common import console, get_session, seed_model + + +def main(): + parser = argparse.ArgumentParser(description="Seed US model") + parser.add_argument( + "--lite", + action="store_true", + help="Lite mode: skip state parameters", + ) + args = parser.parse_args() + + # Import here to avoid slow import at module level + from policyengine.tax_benefit_models.us import us_latest + + console.print("[bold green]Seeding US model...[/bold green]\n") + + start = time.time() + with get_session() as session: + version = seed_model(us_latest, session, lite=args.lite) + console.print(f"[green]✓[/green] US model seeded: {version.id}") + + elapsed = time.time() - start + console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() diff --git a/src/policyengine_api/modal_app.py b/src/policyengine_api/modal_app.py index 1aa8119..14083cf 100644 --- a/src/policyengine_api/modal_app.py +++ b/src/policyengine_api/modal_app.py @@ -7,7 +7,8 @@ Function naming follows the API hierarchy: - simulate_household_*: Single household calculation (/simulate/household) - simulate_economy_*: Single economy simulation (/simulate/economy) -- economy_comparison_*: Full economy comparison analysis (/analysis/compare/economy) +- economy_comparison_*: Full economy comparison analysis (/analysis/economic-impact) +- household_impact_*: Household impact analysis (/analysis/household-impact) Deploy with: modal deploy src/policyengine_api/modal_app.py """ @@ -2516,3 +2517,689 @@ def compute_change_aggregate_us( raise finally: logfire.force_flush() + + +# ============================================================================= +# Household Impact Functions +# ============================================================================= + + +@app.function( + image=uk_image, + secrets=[db_secrets, logfire_secrets], + memory=2048, + cpu=2, + timeout=300, +) +def household_impact_uk(report_id: str, traceparent: str | None = None) -> None: + """Run UK household impact analysis and write results to database.""" + import logfire + + configure_logfire("policyengine-modal-uk", traceparent) + + try: + with logfire.span("household_impact_uk", report_id=report_id): + from datetime import datetime, timezone + from uuid import UUID + + from sqlmodel import Session, create_engine + + database_url = get_database_url() + engine = create_engine(database_url) + + try: + from policyengine_api.models import ( + Household, + Report, + ReportStatus, + Simulation, + SimulationStatus, + ) + + with Session(engine) as session: + # Load report + report = session.get(Report, UUID(report_id)) + if not report: + raise ValueError(f"Report {report_id} not found") + + # Mark as running + report.status = ReportStatus.RUNNING + session.add(report) + session.commit() + + # Run baseline simulation + if report.baseline_simulation_id: + _run_household_simulation_uk( + report.baseline_simulation_id, session + ) + + # Run reform simulation if present + if report.reform_simulation_id: + _run_household_simulation_uk( + report.reform_simulation_id, session + ) + + # Mark report as completed + report.status = ReportStatus.COMPLETED + session.add(report) + session.commit() + + except Exception as e: + logfire.error( + "UK household impact failed", report_id=report_id, error=str(e) + ) + try: + from sqlmodel import text + + with Session(engine) as session: + session.execute( + text( + "UPDATE reports SET status = 'FAILED', error_message = :error " + "WHERE id = :report_id" + ), + {"report_id": report_id, "error": str(e)[:1000]}, + ) + session.commit() + except Exception as db_error: + logfire.error("Failed to update DB", error=str(db_error)) + raise + finally: + logfire.force_flush() + + +def _run_household_simulation_uk(simulation_id, session) -> None: + """Run a single UK household simulation.""" + from datetime import datetime, timezone + + from policyengine_api.models import ( + Household, + Simulation, + SimulationStatus, + ) + + simulation = session.get(Simulation, simulation_id) + if not simulation or simulation.status != SimulationStatus.PENDING: + return + + household = session.get(Household, simulation.household_id) + if not household: + raise ValueError(f"Household {simulation.household_id} not found") + + # Mark as running + simulation.status = SimulationStatus.RUNNING + simulation.started_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + try: + # Get policy data if present + policy_data = _get_household_policy_data(simulation.policy_id, session) + + # Run calculation + result = _calculate_uk_household( + household.household_data, + household.year, + policy_data, + ) + + # Store result + simulation.household_result = result + simulation.status = SimulationStatus.COMPLETED + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + except Exception as e: + simulation.status = SimulationStatus.FAILED + simulation.error_message = str(e) + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + raise + + +def _calculate_uk_household( + household_data: dict, year: int, policy_data: dict | None +) -> dict: + """Calculate UK household and return result dict.""" + import tempfile + from pathlib import Path + + import pandas as pd + from microdf import MicroDataFrame + from policyengine.core import Simulation + from policyengine.tax_benefit_models.uk import uk_latest + from policyengine.tax_benefit_models.uk.datasets import ( + PolicyEngineUKDataset, + UKYearData, + ) + + people = household_data.get("people", []) + benunit = household_data.get("benunit", []) + hh = household_data.get("household", []) + + # Ensure lists + if isinstance(benunit, dict): + benunit = [benunit] + if isinstance(hh, dict): + hh = [hh] + + n_people = len(people) + n_benunits = max(1, len(benunit) if benunit else 1) + n_households = max(1, len(hh) if hh else 1) + + # Build person data + person_data = { + "person_id": list(range(n_people)), + "person_benunit_id": [0] * n_people, + "person_household_id": [0] * n_people, + "person_weight": [1.0] * n_people, + } + for i, person in enumerate(people): + for key, value in person.items(): + if key not in person_data: + person_data[key] = [0.0] * n_people + person_data[key][i] = value + + # Build benunit data + benunit_data = { + "benunit_id": list(range(n_benunits)), + "benunit_weight": [1.0] * n_benunits, + } + for i, bu in enumerate(benunit if benunit else [{}]): + for key, value in bu.items(): + if key not in benunit_data: + benunit_data[key] = [0.0] * n_benunits + benunit_data[key][i] = value + + # Build household data + household_df_data = { + "household_id": list(range(n_households)), + "household_weight": [1.0] * n_households, + "region": ["LONDON"] * n_households, + "tenure_type": ["RENT_PRIVATELY"] * n_households, + "council_tax": [0.0] * n_households, + "rent": [0.0] * n_households, + } + for i, h in enumerate(hh if hh else [{}]): + for key, value in h.items(): + if key not in household_df_data: + household_df_data[key] = [0.0] * n_households + household_df_data[key][i] = value + + # Create MicroDataFrames + person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") + benunit_df = MicroDataFrame(pd.DataFrame(benunit_data), weights="benunit_weight") + household_df = MicroDataFrame( + pd.DataFrame(household_df_data), weights="household_weight" + ) + + # Create temporary dataset + tmpdir = tempfile.mkdtemp() + filepath = str(Path(tmpdir) / "household_calc.h5") + + dataset = PolicyEngineUKDataset( + name="Household calculation", + description="Household(s) for calculation", + person=person_df, + benunit=benunit_df, + household=household_df, + filepath=filepath, + year_data_class=UKYearData, + ) + dataset.save() + + # Build policy if provided + policy = None + if policy_data: + from policyengine.core.policy import ParameterValue, Policy + + pe_param_values = [] + param_lookup = {p.name: p for p in uk_latest.parameters} + for pv in policy_data.get("parameter_values", []): + param_name = pv.get("parameter_name") + if param_name and param_name in param_lookup: + pe_pv = ParameterValue( + parameter=param_lookup[param_name], + value=pv.get("value"), + start_date=pv.get("start_date"), + end_date=pv.get("end_date"), + ) + pe_param_values.append(pe_pv) + + if pe_param_values: + policy = Policy( + name=policy_data.get("name", "Reform"), + description=policy_data.get("description", ""), + parameter_values=pe_param_values, + ) + + # Run simulation + sim = Simulation( + dataset=dataset, + tax_benefit_model_version=uk_latest, + policy=policy, + ) + sim.ensure() + + # Extract results + result = {"person": [], "benunit": [], "household": []} + + for i in range(n_people): + person_result = {} + for var in sim.output_dataset.person.columns: + val = sim.output_dataset.person[var].iloc[i] + person_result[var] = float(val) if hasattr(val, "item") else val + result["person"].append(person_result) + + for i in range(n_benunits): + benunit_result = {} + for var in sim.output_dataset.benunit.columns: + val = sim.output_dataset.benunit[var].iloc[i] + benunit_result[var] = float(val) if hasattr(val, "item") else val + result["benunit"].append(benunit_result) + + for i in range(n_households): + household_result = {} + for var in sim.output_dataset.household.columns: + val = sim.output_dataset.household[var].iloc[i] + household_result[var] = float(val) if hasattr(val, "item") else val + result["household"].append(household_result) + + return result + + +@app.function( + image=us_image, + secrets=[db_secrets, logfire_secrets], + memory=2048, + cpu=2, + timeout=300, +) +def household_impact_us(report_id: str, traceparent: str | None = None) -> None: + """Run US household impact analysis and write results to database.""" + import logfire + + configure_logfire("policyengine-modal-us", traceparent) + + try: + with logfire.span("household_impact_us", report_id=report_id): + from datetime import datetime, timezone + from uuid import UUID + + from sqlmodel import Session, create_engine + + database_url = get_database_url() + engine = create_engine(database_url) + + try: + from policyengine_api.models import ( + Household, + Report, + ReportStatus, + Simulation, + SimulationStatus, + ) + + with Session(engine) as session: + # Load report + report = session.get(Report, UUID(report_id)) + if not report: + raise ValueError(f"Report {report_id} not found") + + # Mark as running + report.status = ReportStatus.RUNNING + session.add(report) + session.commit() + + # Run baseline simulation + if report.baseline_simulation_id: + _run_household_simulation_us( + report.baseline_simulation_id, session + ) + + # Run reform simulation if present + if report.reform_simulation_id: + _run_household_simulation_us( + report.reform_simulation_id, session + ) + + # Mark report as completed + report.status = ReportStatus.COMPLETED + session.add(report) + session.commit() + + except Exception as e: + logfire.error( + "US household impact failed", report_id=report_id, error=str(e) + ) + try: + from sqlmodel import text + + with Session(engine) as session: + session.execute( + text( + "UPDATE reports SET status = 'FAILED', error_message = :error " + "WHERE id = :report_id" + ), + {"report_id": report_id, "error": str(e)[:1000]}, + ) + session.commit() + except Exception as db_error: + logfire.error("Failed to update DB", error=str(db_error)) + raise + finally: + logfire.force_flush() + + +def _run_household_simulation_us(simulation_id, session) -> None: + """Run a single US household simulation.""" + from datetime import datetime, timezone + + from policyengine_api.models import ( + Household, + Simulation, + SimulationStatus, + ) + + simulation = session.get(Simulation, simulation_id) + if not simulation or simulation.status != SimulationStatus.PENDING: + return + + household = session.get(Household, simulation.household_id) + if not household: + raise ValueError(f"Household {simulation.household_id} not found") + + # Mark as running + simulation.status = SimulationStatus.RUNNING + simulation.started_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + + try: + # Get policy data if present + policy_data = _get_household_policy_data(simulation.policy_id, session) + + # Run calculation + result = _calculate_us_household( + household.household_data, + household.year, + policy_data, + ) + + # Store result + simulation.household_result = result + simulation.status = SimulationStatus.COMPLETED + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + except Exception as e: + simulation.status = SimulationStatus.FAILED + simulation.error_message = str(e) + simulation.completed_at = datetime.now(timezone.utc) + session.add(simulation) + session.commit() + raise + + +def _calculate_us_household( + household_data: dict, year: int, policy_data: dict | None +) -> dict: + """Calculate US household and return result dict.""" + import tempfile + from pathlib import Path + + import pandas as pd + from microdf import MicroDataFrame + from policyengine.core import Simulation + from policyengine.tax_benefit_models.us import us_latest + from policyengine.tax_benefit_models.us.datasets import ( + PolicyEngineUSDataset, + USYearData, + ) + + people = household_data.get("people", []) + tax_unit = household_data.get("tax_unit", []) + family = household_data.get("family", []) + spm_unit = household_data.get("spm_unit", []) + marital_unit = household_data.get("marital_unit", []) + hh = household_data.get("household", []) + + # Ensure lists + if isinstance(tax_unit, dict): + tax_unit = [tax_unit] + if isinstance(family, dict): + family = [family] + if isinstance(spm_unit, dict): + spm_unit = [spm_unit] + if isinstance(marital_unit, dict): + marital_unit = [marital_unit] + if isinstance(hh, dict): + hh = [hh] + + n_people = len(people) + n_tax_units = max(1, len(tax_unit) if tax_unit else 1) + n_families = max(1, len(family) if family else 1) + n_spm_units = max(1, len(spm_unit) if spm_unit else 1) + n_marital_units = max(1, len(marital_unit) if marital_unit else 1) + n_households = max(1, len(hh) if hh else 1) + + # Build person data + person_data = { + "person_id": list(range(n_people)), + "person_tax_unit_id": [0] * n_people, + "person_family_id": [0] * n_people, + "person_spm_unit_id": [0] * n_people, + "person_marital_unit_id": [0] * n_people, + "person_household_id": [0] * n_people, + "person_weight": [1.0] * n_people, + } + for i, person in enumerate(people): + for key, value in person.items(): + if key not in person_data: + person_data[key] = [0.0] * n_people + person_data[key][i] = value + + # Build tax_unit data + tax_unit_data = { + "tax_unit_id": list(range(n_tax_units)), + "tax_unit_weight": [1.0] * n_tax_units, + } + for i, tu in enumerate(tax_unit if tax_unit else [{}]): + for key, value in tu.items(): + if key not in tax_unit_data: + tax_unit_data[key] = [0.0] * n_tax_units + tax_unit_data[key][i] = value + + # Build family data + family_data = { + "family_id": list(range(n_families)), + "family_weight": [1.0] * n_families, + } + for i, fam in enumerate(family if family else [{}]): + for key, value in fam.items(): + if key not in family_data: + family_data[key] = [0.0] * n_families + family_data[key][i] = value + + # Build spm_unit data + spm_unit_data = { + "spm_unit_id": list(range(n_spm_units)), + "spm_unit_weight": [1.0] * n_spm_units, + } + for i, spm in enumerate(spm_unit if spm_unit else [{}]): + for key, value in spm.items(): + if key not in spm_unit_data: + spm_unit_data[key] = [0.0] * n_spm_units + spm_unit_data[key][i] = value + + # Build marital_unit data + marital_unit_data = { + "marital_unit_id": list(range(n_marital_units)), + "marital_unit_weight": [1.0] * n_marital_units, + } + for i, mu in enumerate(marital_unit if marital_unit else [{}]): + for key, value in mu.items(): + if key not in marital_unit_data: + marital_unit_data[key] = [0.0] * n_marital_units + marital_unit_data[key][i] = value + + # Build household data + household_df_data = { + "household_id": list(range(n_households)), + "household_weight": [1.0] * n_households, + } + for i, h in enumerate(hh if hh else [{}]): + for key, value in h.items(): + if key not in household_df_data: + household_df_data[key] = [0.0] * n_households + household_df_data[key][i] = value + + # Create MicroDataFrames + person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") + tax_unit_df = MicroDataFrame( + pd.DataFrame(tax_unit_data), weights="tax_unit_weight" + ) + family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") + spm_unit_df = MicroDataFrame( + pd.DataFrame(spm_unit_data), weights="spm_unit_weight" + ) + marital_unit_df = MicroDataFrame( + pd.DataFrame(marital_unit_data), weights="marital_unit_weight" + ) + household_df = MicroDataFrame( + pd.DataFrame(household_df_data), weights="household_weight" + ) + + # Create temporary dataset + tmpdir = tempfile.mkdtemp() + filepath = str(Path(tmpdir) / "household_calc.h5") + + dataset = PolicyEngineUSDataset( + name="Household calculation", + description="Household(s) for calculation", + person=person_df, + tax_unit=tax_unit_df, + family=family_df, + spm_unit=spm_unit_df, + marital_unit=marital_unit_df, + household=household_df, + filepath=filepath, + year_data_class=USYearData, + ) + dataset.save() + + # Build policy if provided + policy = None + if policy_data: + from policyengine.core.policy import ParameterValue, Policy + + pe_param_values = [] + param_lookup = {p.name: p for p in us_latest.parameters} + for pv in policy_data.get("parameter_values", []): + param_name = pv.get("parameter_name") + if param_name and param_name in param_lookup: + pe_pv = ParameterValue( + parameter=param_lookup[param_name], + value=pv.get("value"), + start_date=pv.get("start_date"), + end_date=pv.get("end_date"), + ) + pe_param_values.append(pe_pv) + + if pe_param_values: + policy = Policy( + name=policy_data.get("name", "Reform"), + description=policy_data.get("description", ""), + parameter_values=pe_param_values, + ) + + # Run simulation + sim = Simulation( + dataset=dataset, + tax_benefit_model_version=us_latest, + policy=policy, + ) + sim.ensure() + + # Extract results + result = { + "person": [], + "tax_unit": [], + "family": [], + "spm_unit": [], + "marital_unit": [], + "household": [], + } + + for i in range(n_people): + person_result = {} + for var in sim.output_dataset.person.columns: + val = sim.output_dataset.person[var].iloc[i] + person_result[var] = float(val) if hasattr(val, "item") else val + result["person"].append(person_result) + + for i in range(n_tax_units): + tu_result = {} + for var in sim.output_dataset.tax_unit.columns: + val = sim.output_dataset.tax_unit[var].iloc[i] + tu_result[var] = float(val) if hasattr(val, "item") else val + result["tax_unit"].append(tu_result) + + for i in range(n_families): + fam_result = {} + for var in sim.output_dataset.family.columns: + val = sim.output_dataset.family[var].iloc[i] + fam_result[var] = float(val) if hasattr(val, "item") else val + result["family"].append(fam_result) + + for i in range(n_spm_units): + spm_result = {} + for var in sim.output_dataset.spm_unit.columns: + val = sim.output_dataset.spm_unit[var].iloc[i] + spm_result[var] = float(val) if hasattr(val, "item") else val + result["spm_unit"].append(spm_result) + + for i in range(n_marital_units): + mu_result = {} + for var in sim.output_dataset.marital_unit.columns: + val = sim.output_dataset.marital_unit[var].iloc[i] + mu_result[var] = float(val) if hasattr(val, "item") else val + result["marital_unit"].append(mu_result) + + for i in range(n_households): + hh_result = {} + for var in sim.output_dataset.household.columns: + val = sim.output_dataset.household[var].iloc[i] + hh_result[var] = float(val) if hasattr(val, "item") else val + result["household"].append(hh_result) + + return result + + +def _get_household_policy_data(policy_id, session) -> dict | None: + """Get policy data for household calculation.""" + if policy_id is None: + return None + + from policyengine_api.models import Policy + + db_policy = session.get(Policy, policy_id) + if not db_policy: + return None + + return { + "name": db_policy.name, + "description": db_policy.description, + "parameter_values": [ + { + "parameter_name": pv.parameter.name if pv.parameter else None, + "value": pv.value_json.get("value") + if isinstance(pv.value_json, dict) + else pv.value_json, + "start_date": pv.start_date.isoformat() if pv.start_date else None, + "end_date": pv.end_date.isoformat() if pv.end_date else None, + } + for pv in db_policy.parameter_values + if pv.parameter + ], + } From cf8511f31f526e0e46c6baaf0ba841bcef8886c0 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 5 Feb 2026 17:28:24 +0300 Subject: [PATCH 09/87] test: Tests --- scripts/seed_common.py | 11 +- scripts/test_economy_simulation.py | 277 +++++++++++ scripts/test_household_scenarios.py | 344 +++++++++++++ src/policyengine_api/api/household.py | 388 +++++++++------ .../api/household_analysis.py | 33 +- src/policyengine_api/modal_app.py | 464 ++++++++++++++++-- test_fixtures/fixtures_policy_reform.py | 282 +++++++++++ tests/test_policy_reform.py | 327 ++++++++++++ 8 files changed, 1929 insertions(+), 197 deletions(-) create mode 100644 scripts/test_economy_simulation.py create mode 100644 scripts/test_household_scenarios.py create mode 100644 test_fixtures/fixtures_policy_reform.py create mode 100644 tests/test_policy_reform.py diff --git a/scripts/seed_common.py b/scripts/seed_common.py index 4e4e2ec..f6d7ab6 100644 --- a/scripts/seed_common.py +++ b/scripts/seed_common.py @@ -189,8 +189,7 @@ def seed_model(model_version, session, lite: bool = False): f" [green]✓[/green] Added {len(model_version.variables)} variables" ) - # Add parameters (only user-facing ones: those with labels) - # Deduplicate by name - keep first occurrence + # Add parameters - deduplicate by name (keep first occurrence) # # WHY DEDUPLICATION IS NEEDED: # The policyengine package can provide multiple parameter entries with the same @@ -198,17 +197,16 @@ def seed_model(model_version, session, lite: bool = False): # state-specific variants that share the same base name. We keep only the first # occurrence to avoid database unique constraint violations and reduce redundancy. # + # NOTE: We do NOT filter by label. Parameters without labels (bracket params, + # breakdown params) are still valid and needed for policy analysis. + # # In lite mode, exclude US state parameters (gov.states.*) seen_names = set() parameters_to_add = [] skipped_state_params = 0 - skipped_no_label = 0 skipped_duplicate = 0 for p in model_version.parameters: - if p.label is None: - skipped_no_label += 1 - continue if p.name in seen_names: skipped_duplicate += 1 continue @@ -221,7 +219,6 @@ def seed_model(model_version, session, lite: bool = False): console.print(f" Parameter filtering:") console.print(f" - Total from source: {len(model_version.parameters)}") - console.print(f" - Skipped (no label): {skipped_no_label}") console.print(f" - Skipped (duplicate name): {skipped_duplicate}") if lite and skipped_state_params > 0: console.print(f" - Skipped (state params, lite mode): {skipped_state_params}") diff --git a/scripts/test_economy_simulation.py b/scripts/test_economy_simulation.py new file mode 100644 index 0000000..3845fc4 --- /dev/null +++ b/scripts/test_economy_simulation.py @@ -0,0 +1,277 @@ +"""Test economy-wide simulation following the exact flow from modal_app.py. + +This script mimics the economy-wide simulation code path as closely as possible +to verify whether policy reforms are being applied correctly. +""" + +import tempfile +from datetime import datetime +from pathlib import Path + +import pandas as pd +from microdf import MicroDataFrame + +# Import exactly as modal_app.py does +from policyengine.core import Simulation as PESimulation +from policyengine.core.policy import ParameterValue as PEParameterValue +from policyengine.core.policy import Policy as PEPolicy +from policyengine.tax_benefit_models.us import us_latest +from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset, USYearData + + +def create_test_dataset(year: int) -> PolicyEngineUSDataset: + """Create a small test dataset similar to what would be loaded from storage. + + Uses the same structure as economy-wide datasets but with just a few households. + """ + # Create 3 test households with different income levels + # Each household has 2 adults + 2 children (to test CTC) + n_households = 3 + n_people = n_households * 4 # 4 people per household + + # Person data + person_data = { + "person_id": list(range(n_people)), + "person_household_id": [i // 4 for i in range(n_people)], + "person_marital_unit_id": [], + "person_family_id": [i // 4 for i in range(n_people)], + "person_spm_unit_id": [i // 4 for i in range(n_people)], + "person_tax_unit_id": [i // 4 for i in range(n_people)], + "person_weight": [1000.0] * n_people, # Weight for population scaling + "age": [], + "employment_income": [], + } + + # Build person details + marital_unit_counter = 0 + for hh in range(n_households): + base_income = 10000 + (hh * 20000) # 10k, 30k, 50k + # Adult 1 + person_data["age"].append(35) + person_data["employment_income"].append(base_income) + person_data["person_marital_unit_id"].append(marital_unit_counter) + # Adult 2 + person_data["age"].append(33) + person_data["employment_income"].append(0) + person_data["person_marital_unit_id"].append(marital_unit_counter) + marital_unit_counter += 1 + # Child 1 + person_data["age"].append(5) + person_data["employment_income"].append(0) + person_data["person_marital_unit_id"].append(marital_unit_counter) + marital_unit_counter += 1 + # Child 2 + person_data["age"].append(3) + person_data["employment_income"].append(0) + person_data["person_marital_unit_id"].append(marital_unit_counter) + marital_unit_counter += 1 + + n_marital_units = marital_unit_counter + + # Entity data + household_data = { + "household_id": list(range(n_households)), + "household_weight": [1000.0] * n_households, + "state_fips": [48] * n_households, # Texas + } + + marital_unit_data = { + "marital_unit_id": list(range(n_marital_units)), + "marital_unit_weight": [1000.0] * n_marital_units, + } + + family_data = { + "family_id": list(range(n_households)), + "family_weight": [1000.0] * n_households, + } + + spm_unit_data = { + "spm_unit_id": list(range(n_households)), + "spm_unit_weight": [1000.0] * n_households, + } + + tax_unit_data = { + "tax_unit_id": list(range(n_households)), + "tax_unit_weight": [1000.0] * n_households, + } + + # Create MicroDataFrames (same as economy datasets) + person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") + household_df = MicroDataFrame(pd.DataFrame(household_data), weights="household_weight") + marital_unit_df = MicroDataFrame(pd.DataFrame(marital_unit_data), weights="marital_unit_weight") + family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") + spm_unit_df = MicroDataFrame(pd.DataFrame(spm_unit_data), weights="spm_unit_weight") + tax_unit_df = MicroDataFrame(pd.DataFrame(tax_unit_data), weights="tax_unit_weight") + + # Create dataset file + tmpdir = tempfile.mkdtemp() + filepath = str(Path(tmpdir) / "test_economy.h5") + + return PolicyEngineUSDataset( + name="Test Economy Dataset", + description="Small test dataset for economy simulation", + filepath=filepath, + year=year, + data=USYearData( + person=person_df, + household=household_df, + marital_unit=marital_unit_df, + family=family_df, + spm_unit=spm_unit_df, + tax_unit=tax_unit_df, + ), + ) + + +def create_policy_like_modal_app(model_version) -> PEPolicy: + """Create a policy exactly like _get_pe_policy_us does in modal_app.py. + + This mimics the exact flow: + 1. Look up parameter by name from model_version.parameters + 2. Create PEParameterValue with the parameter, value, start_date, end_date + 3. Create PEPolicy with the parameter values + """ + param_lookup = {p.name: p for p in model_version.parameters} + + # This is exactly what _get_pe_policy_us does + pe_param = param_lookup.get("gov.irs.credits.ctc.refundable.fully_refundable") + if not pe_param: + raise ValueError("Parameter not found!") + + pe_pv = PEParameterValue( + parameter=pe_param, + value=True, # Make CTC fully refundable + start_date=datetime(2024, 1, 1), + end_date=None, + ) + + return PEPolicy( + name="CTC Fully Refundable", + description="Makes CTC fully refundable", + parameter_values=[pe_pv], + ) + + +def run_economy_simulation(dataset: PolicyEngineUSDataset, policy: PEPolicy | None, label: str) -> dict: + """Run an economy simulation exactly like modal_app.py does. + + This follows the exact flow from simulate_economy_us: + 1. Create PESimulation with dataset, model version, policy, dynamic + 2. Call pe_sim.ensure() (which calls run() internally) + 3. Access output via pe_sim.output_dataset + """ + print(f"\n=== {label} ===") + print(f" Policy: {policy.name if policy else 'None (baseline)'}") + if policy: + print(f" Policy parameter_values: {len(policy.parameter_values)}") + for pv in policy.parameter_values: + print(f" - {pv.parameter.name}: {pv.value} (start: {pv.start_date})") + + pe_model_version = us_latest + + # Create and run simulation - EXACTLY like modal_app.py lines 1006-1012 + pe_sim = PESimulation( + dataset=dataset, + tax_benefit_model_version=pe_model_version, + policy=policy, + dynamic=None, + ) + pe_sim.ensure() + + # Extract results from output dataset + output_data = pe_sim.output_dataset.data + + # Sum up key metrics across all tax units (weighted) + tax_unit_df = pd.DataFrame(output_data.tax_unit) + + # Get the variables we care about + total_ctc = 0 + total_income_tax = 0 + total_eitc = 0 + + for var in ["ctc", "income_tax", "eitc"]: + if var in tax_unit_df.columns: + # Weighted sum + weights = tax_unit_df.get("tax_unit_weight", pd.Series([1.0] * len(tax_unit_df))) + if var == "ctc": + total_ctc = (tax_unit_df[var] * weights).sum() + elif var == "income_tax": + total_income_tax = (tax_unit_df[var] * weights).sum() + elif var == "eitc": + total_eitc = (tax_unit_df[var] * weights).sum() + + print(f" Results (weighted totals across {len(tax_unit_df)} tax units):") + print(f" Total CTC: ${total_ctc:,.0f}") + print(f" Total Income Tax: ${total_income_tax:,.0f}") + print(f" Total EITC: ${total_eitc:,.0f}") + + # Also show per-household breakdown + print(f" Per tax unit breakdown:") + for i in range(len(tax_unit_df)): + ctc = tax_unit_df["ctc"].iloc[i] if "ctc" in tax_unit_df.columns else 0 + income_tax = tax_unit_df["income_tax"].iloc[i] if "income_tax" in tax_unit_df.columns else 0 + print(f" Tax Unit {i}: CTC=${ctc:,.0f}, Income Tax=${income_tax:,.0f}") + + return { + "total_ctc": total_ctc, + "total_income_tax": total_income_tax, + "total_eitc": total_eitc, + "tax_unit_df": tax_unit_df, + } + + +def main(): + print("=" * 60) + print("ECONOMY-WIDE SIMULATION TEST") + print("Following the exact code path from modal_app.py") + print("=" * 60) + + year = 2024 + + # Create test dataset (same for both simulations) + print("\nCreating test dataset...") + + # Run baseline simulation + baseline_dataset = create_test_dataset(year) + baseline_results = run_economy_simulation(baseline_dataset, None, "BASELINE (no policy)") + + # Create policy exactly like modal_app.py does + policy = create_policy_like_modal_app(us_latest) + + # Run reform simulation + reform_dataset = create_test_dataset(year) + reform_results = run_economy_simulation(reform_dataset, policy, "REFORM (CTC fully refundable)") + + # Compare results + print("\n" + "=" * 60) + print("COMPARISON") + print("=" * 60) + + ctc_diff = reform_results["total_ctc"] - baseline_results["total_ctc"] + tax_diff = reform_results["total_income_tax"] - baseline_results["total_income_tax"] + + print(f"\nTotal CTC:") + print(f" Baseline: ${baseline_results['total_ctc']:,.0f}") + print(f" Reform: ${reform_results['total_ctc']:,.0f}") + print(f" Change: ${ctc_diff:,.0f}") + + print(f"\nTotal Income Tax:") + print(f" Baseline: ${baseline_results['total_income_tax']:,.0f}") + print(f" Reform: ${reform_results['total_income_tax']:,.0f}") + print(f" Change: ${tax_diff:,.0f}") + + # Verdict + print("\n" + "=" * 60) + print("VERDICT") + print("=" * 60) + + if baseline_results["total_income_tax"] == reform_results["total_income_tax"]: + print("\n❌ BUG CONFIRMED: Results are IDENTICAL!") + print(" The policy reform is NOT being applied to economy simulations.") + else: + print("\n✓ NO BUG: Results differ as expected!") + print(f" The fully refundable CTC reform changed income tax by ${tax_diff:,.0f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_household_scenarios.py b/scripts/test_household_scenarios.py new file mode 100644 index 0000000..fb418a4 --- /dev/null +++ b/scripts/test_household_scenarios.py @@ -0,0 +1,344 @@ +"""Test household calculation scenarios. + +Tests: +1. US California household under current law +2. Scotland household under current law +3. US household: current law vs CTC fully refundable reform +""" + +import sys +import time +import requests + +BASE_URL = "http://127.0.0.1:8000" + + +def poll_for_completion(report_id: str, max_attempts: int = 60) -> dict: + """Poll until report is completed or failed.""" + for attempt in range(max_attempts): + resp = requests.get(f"{BASE_URL}/analysis/household-impact/{report_id}") + if resp.status_code != 200: + raise Exception(f"Failed to get report: {resp.status_code} - {resp.text}") + + result = resp.json() + status = result["status"].upper() + + if status == "COMPLETED": + return result + elif status == "FAILED": + raise Exception(f"Report failed: {result.get('error_message', 'Unknown error')}") + + time.sleep(0.5) + + raise Exception(f"Timed out after {max_attempts} attempts") + + +def print_household_summary(result: dict, label: str): + """Print summary of household calculation result.""" + print(f"\n {label}:") + + baseline = result.get("baseline_result", {}) + reform = result.get("reform_result", {}) + + # Get key metrics from person/household + if "person" in baseline and baseline["person"]: + person = baseline["person"][0] + if "household_net_income" in person: + baseline_income = person["household_net_income"] + print(f" Baseline net income: ${baseline_income:,.2f}") + + if reform and "person" in reform and reform["person"]: + reform_income = reform["person"][0].get("household_net_income", 0) + print(f" Reform net income: ${reform_income:,.2f}") + print(f" Difference: ${reform_income - baseline_income:,.2f}") + + # Show some tax/benefit info if available + for key in ["income_tax", "federal_income_tax", "state_income_tax", "ctc", "refundable_ctc"]: + if key in person: + print(f" {key}: ${person[key]:,.2f}") + + +def test_us_california(): + """Test 1: US California household under current law.""" + print("\n" + "=" * 60) + print("TEST 1: US California Household - Current Law") + print("=" * 60) + + # Create California household + household_data = { + "tax_benefit_model_name": "policyengine_us", + "year": 2024, + "label": "California test household", + "people": [ + {"age": 35, "employment_income": 75000}, + {"age": 33, "employment_income": 45000}, + {"age": 8}, # Child + ], + "tax_unit": {}, + "family": {}, + "spm_unit": {}, + "marital_unit": {}, + "household": {"state_code": "CA"}, + } + + print("\n Creating household...") + resp = requests.post(f"{BASE_URL}/households/", json=household_data) + if resp.status_code != 201: + print(f" FAILED: {resp.status_code} - {resp.text}") + return None + + household = resp.json() + household_id = household["id"] + print(f" Household ID: {household_id}") + + # Run analysis under current law (no policy_id) + print(" Running analysis...") + resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ + "household_id": household_id, + "policy_id": None, + }) + + if resp.status_code != 200: + print(f" FAILED: {resp.status_code} - {resp.text}") + return household_id + + report_id = resp.json()["report_id"] + print(f" Report ID: {report_id}") + + # Poll for results + try: + result = poll_for_completion(report_id) + print(" Status: COMPLETED") + print_household_summary(result, "Results") + except Exception as e: + print(f" FAILED: {e}") + + return household_id + + +def test_scotland(): + """Test 2: Scotland household under current law.""" + print("\n" + "=" * 60) + print("TEST 2: Scotland Household - Current Law") + print("=" * 60) + + # Create Scotland household + household_data = { + "tax_benefit_model_name": "policyengine_uk", + "year": 2024, + "label": "Scotland test household", + "people": [ + {"age": 40, "employment_income": 45000}, + ], + "benunit": {}, + "household": {"region": "SCOTLAND"}, + } + + print("\n Creating household...") + resp = requests.post(f"{BASE_URL}/households/", json=household_data) + if resp.status_code != 201: + print(f" FAILED: {resp.status_code} - {resp.text}") + return None + + household = resp.json() + household_id = household["id"] + print(f" Household ID: {household_id}") + + # Run analysis under current law + print(" Running analysis...") + resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ + "household_id": household_id, + "policy_id": None, + }) + + if resp.status_code != 200: + print(f" FAILED: {resp.status_code} - {resp.text}") + return household_id + + report_id = resp.json()["report_id"] + print(f" Report ID: {report_id}") + + # Poll for results + try: + result = poll_for_completion(report_id) + print(" Status: COMPLETED") + print_household_summary(result, "Results") + except Exception as e: + print(f" FAILED: {e}") + + return household_id + + +def test_us_ctc_reform(): + """Test 3: US household - current law vs CTC fully refundable.""" + print("\n" + "=" * 60) + print("TEST 3: US Household - Current Law vs CTC Fully Refundable") + print("=" * 60) + + # First, find the CTC refundability parameter + print("\n Finding CTC refundability parameter...") + resp = requests.get(f"{BASE_URL}/parameters", params={"search": "ctc", "limit": 50}) + if resp.status_code != 200: + print(f" FAILED to search parameters: {resp.status_code}") + return None, None + + params = resp.json() + ctc_param = None + for p in params: + # Look for the refundable portion parameter + if "refundable" in p["name"].lower() and "ctc" in p["name"].lower(): + print(f" Found: {p['name']} (label: {p.get('label')})") + ctc_param = p + break + + if not ctc_param: + # Try searching for child tax credit parameters + print(" Searching for child_tax_credit parameters...") + resp = requests.get(f"{BASE_URL}/parameters", params={"search": "child_tax_credit", "limit": 50}) + params = resp.json() + for p in params: + print(f" - {p['name']}") + if "refundable" in p["name"].lower(): + ctc_param = p + break + + if not ctc_param: + print(" Could not find CTC refundability parameter") + print(" Continuing with household creation anyway...") + + # Create household with children (needed for CTC) + household_data = { + "tax_benefit_model_name": "policyengine_us", + "year": 2024, + "label": "CTC test household", + "people": [ + {"age": 35, "employment_income": 30000}, # Lower income to see CTC effect + {"age": 33, "employment_income": 0}, + {"age": 5}, # Child 1 + {"age": 3}, # Child 2 + ], + "tax_unit": {}, + "family": {}, + "spm_unit": {}, + "marital_unit": {}, + "household": {"state_code": "TX"}, # Texas - no state income tax + } + + print("\n Creating household...") + resp = requests.post(f"{BASE_URL}/households/", json=household_data) + if resp.status_code != 201: + print(f" FAILED: {resp.status_code} - {resp.text}") + return None, None + + household = resp.json() + household_id = household["id"] + print(f" Household ID: {household_id}") + + # Create a policy that makes CTC fully refundable + policy_id = None + if ctc_param: + print("\n Creating CTC fully refundable policy...") + policy_data = { + "name": "CTC Fully Refundable", + "description": "Makes the Child Tax Credit fully refundable", + } + resp = requests.post(f"{BASE_URL}/policies/", json=policy_data) + if resp.status_code == 201: + policy = resp.json() + policy_id = policy["id"] + print(f" Policy ID: {policy_id}") + + # Add parameter value to make CTC fully refundable + # The parameter should set refundable portion to 100% or max amount + pv_data = { + "parameter_id": ctc_param["id"], + "value_json": 1.0, # 100% refundable + "start_date": "2024-01-01T00:00:00Z", + "end_date": None, + "policy_id": policy_id, + } + resp = requests.post(f"{BASE_URL}/parameter-values/", json=pv_data) + if resp.status_code == 201: + print(" Added parameter value for full refundability") + else: + print(f" Warning: Failed to add parameter value: {resp.status_code} - {resp.text}") + else: + print(f" Warning: Failed to create policy: {resp.status_code}") + + # Run analysis with reform policy + print("\n Running analysis (baseline vs reform)...") + resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ + "household_id": household_id, + "policy_id": policy_id, + }) + + if resp.status_code != 200: + print(f" FAILED: {resp.status_code} - {resp.text}") + return household_id, policy_id + + report_id = resp.json()["report_id"] + print(f" Report ID: {report_id}") + + # Poll for results + try: + result = poll_for_completion(report_id) + print(" Status: COMPLETED") + print_household_summary(result, "Results") + except Exception as e: + print(f" FAILED: {e}") + + return household_id, policy_id + + +def main(): + print("=" * 60) + print("HOUSEHOLD CALCULATION SCENARIO TESTS") + print("=" * 60) + + # Track created resources for cleanup + households = [] + policies = [] + + # Test 1: US California + hh_id = test_us_california() + if hh_id: + households.append(hh_id) + + # Test 2: Scotland + hh_id = test_scotland() + if hh_id: + households.append(hh_id) + + # Test 3: CTC Reform + hh_id, policy_id = test_us_ctc_reform() + if hh_id: + households.append(hh_id) + if policy_id: + policies.append(policy_id) + + # Cleanup + print("\n" + "=" * 60) + print("CLEANUP") + print("=" * 60) + + for hh_id in households: + resp = requests.delete(f"{BASE_URL}/households/{hh_id}") + if resp.status_code == 204: + print(f" Deleted household: {hh_id}") + else: + print(f" Warning: Failed to delete household {hh_id}: {resp.status_code}") + + for policy_id in policies: + resp = requests.delete(f"{BASE_URL}/policies/{policy_id}") + if resp.status_code == 204: + print(f" Deleted policy: {policy_id}") + else: + print(f" Warning: Failed to delete policy {policy_id}: {resp.status_code}") + + print("\n" + "=" * 60) + print("TESTS COMPLETE") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/src/policyengine_api/api/household.py b/src/policyengine_api/api/household.py index 0e89b5e..adb6ac9 100644 --- a/src/policyengine_api/api/household.py +++ b/src/policyengine_api/api/household.py @@ -294,17 +294,16 @@ def _calculate_household_uk( Supports multiple households via entity relational dataframes. If entity IDs are not provided, defaults to single household with all people in it. - """ - import tempfile - from datetime import datetime - from pathlib import Path + Uses policyengine-uk Microsimulation directly with reform dict to ensure + policy changes are applied correctly. + """ + import numpy as np import pandas as pd - from policyengine.core import Simulation - from microdf import MicroDataFrame from policyengine.tax_benefit_models.uk import uk_latest - from policyengine.tax_benefit_models.uk.datasets import PolicyEngineUKDataset - from policyengine.tax_benefit_models.uk.datasets import UKYearData + from policyengine_core.simulations.simulation_builder import SimulationBuilder + from policyengine_uk import Microsimulation + from policyengine_uk.system import system n_people = len(people) n_benunits = max(1, len(benunit)) @@ -350,68 +349,88 @@ def _calculate_household_uk( household_data[key] = [0.0] * n_households household_data[key][i] = value - # Create MicroDataFrames - person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") - benunit_df = MicroDataFrame(pd.DataFrame(benunit_data), weights="benunit_weight") - household_df = MicroDataFrame( - pd.DataFrame(household_data), weights="household_weight" + # Convert policy_data to policyengine-uk reform dict format + # Format: {"param.name": {"YYYY-MM-DD": value}} + reform = None + if policy_data and policy_data.get("parameter_values"): + reform = {} + for pv in policy_data["parameter_values"]: + param_name = pv.get("parameter_name") + value = pv.get("value") + start_date = pv.get("start_date") + + if param_name and start_date: + # Parse ISO date string to get just the date part + if "T" in start_date: + date_str = start_date.split("T")[0] + else: + date_str = start_date + + if param_name not in reform: + reform[param_name] = {} + reform[param_name][date_str] = value + + # Create Microsimulation with reform applied at construction time + microsim = Microsimulation(reform=reform) + + # Build simulation from entity data using SimulationBuilder + person_df = pd.DataFrame(person_data) + + # Determine column naming convention + benunit_id_col = ( + "person_benunit_id" + if "person_benunit_id" in person_df.columns + else "benunit_id" ) - - # Create temporary dataset - tmpdir = tempfile.mkdtemp() - filepath = str(Path(tmpdir) / "household_calc.h5") - - dataset = PolicyEngineUKDataset( - name="Household calculation", - description="Household(s) for calculation", - filepath=filepath, - year=year, - data=UKYearData( - person=person_df, - benunit=benunit_df, - household=household_df, - ), + household_id_col = ( + "person_household_id" + if "person_household_id" in person_df.columns + else "household_id" ) - # Build policy if provided - policy = None - if policy_data: - from policyengine.core.policy import ParameterValue as PEParameterValue - from policyengine.core.policy import Policy as PEPolicy - - pe_param_values = [] - param_lookup = {p.name: p for p in uk_latest.parameters} - for pv in policy_data.get("parameter_values", []): - pe_param = param_lookup.get(pv["parameter_name"]) - if pe_param: - pe_pv = PEParameterValue( - parameter=pe_param, - value=pv["value"], - start_date=datetime.fromisoformat(pv["start_date"]) - if pv.get("start_date") - else None, - end_date=datetime.fromisoformat(pv["end_date"]) - if pv.get("end_date") - else None, - ) - pe_param_values.append(pe_pv) - policy = PEPolicy( - name=policy_data.get("name", ""), - description=policy_data.get("description", ""), - parameter_values=pe_param_values, - ) + # Declare entities using SimulationBuilder + builder = SimulationBuilder() + builder.populations = system.instantiate_entities() + + builder.declare_person_entity("person", person_df["person_id"].values) + builder.declare_entity("benunit", np.unique(person_df[benunit_id_col].values)) + builder.declare_entity("household", np.unique(person_df[household_id_col].values)) - # Run simulation - simulation = Simulation( - dataset=dataset, - tax_benefit_model_version=uk_latest, - policy=policy, + # Join persons to group entities + builder.join_with_persons( + builder.populations["benunit"], + person_df[benunit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["household"], + person_df[household_id_col].values, + np.array(["member"] * len(person_df)), ) - simulation.run() - # Extract outputs - output_data = simulation.output_dataset.data + # Build simulation from populations + microsim.build_from_populations(builder.populations) + # Set input variables for each entity + id_columns = { + "person_id", + "benunit_id", + "person_benunit_id", + "household_id", + "person_household_id", + } + + for entity_name, entity_df in [ + ("person", person_data), + ("benunit", benunit_data), + ("household", household_data), + ]: + df = pd.DataFrame(entity_df) + for column in df.columns: + if column not in id_columns and column in system.variables: + microsim.set_input(column, year, df[column].values) + + # Calculate output variables def safe_convert(value): try: return float(value) @@ -422,21 +441,24 @@ def safe_convert(value): for i in range(n_people): person_dict = {} for var in uk_latest.entity_variables["person"]: - person_dict[var] = safe_convert(output_data.person[var].iloc[i]) + val = microsim.calculate(var, period=year, map_to="person") + person_dict[var] = safe_convert(val.values[i]) person_outputs.append(person_dict) benunit_outputs = [] - for i in range(len(output_data.benunit)): + for i in range(n_benunits): benunit_dict = {} for var in uk_latest.entity_variables["benunit"]: - benunit_dict[var] = safe_convert(output_data.benunit[var].iloc[i]) + val = microsim.calculate(var, period=year, map_to="benunit") + benunit_dict[var] = safe_convert(val.values[i]) benunit_outputs.append(benunit_dict) household_outputs = [] - for i in range(len(output_data.household)): + for i in range(n_households): household_dict = {} for var in uk_latest.entity_variables["household"]: - household_dict[var] = safe_convert(output_data.household[var].iloc[i]) + val = microsim.calculate(var, period=year, map_to="household") + household_dict[var] = safe_convert(val.values[i]) household_outputs.append(household_dict) return { @@ -466,7 +488,14 @@ def _run_local_household_us( try: result = _calculate_household_us( - people, marital_unit, family, spm_unit, tax_unit, household, year, policy_data + people, + marital_unit, + family, + spm_unit, + tax_unit, + household, + year, + policy_data, ) # Update job with result @@ -506,17 +535,16 @@ def _calculate_household_us( Supports multiple households via entity relational dataframes. If entity IDs are not provided, defaults to single household with all people in it. - """ - import tempfile - from datetime import datetime - from pathlib import Path + Uses policyengine-us Microsimulation directly with reform dict to ensure + policy changes are applied correctly. + """ + import numpy as np import pandas as pd - from policyengine.core import Simulation - from microdf import MicroDataFrame from policyengine.tax_benefit_models.us import us_latest - from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset - from policyengine.tax_benefit_models.us.datasets import USYearData + from policyengine_core.simulations.simulation_builder import SimulationBuilder + from policyengine_us import Microsimulation + from policyengine_us.system import system n_people = len(people) n_households = max(1, len(household)) @@ -596,108 +624,158 @@ def _calculate_household_us( tax_unit_data[key] = [0.0] * n_tax_units tax_unit_data[key][i] = value - # Create MicroDataFrames - person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") - household_df = MicroDataFrame( - pd.DataFrame(household_data), weights="household_weight" + # Convert policy_data to policyengine-us reform dict format + # Format: {"param.name": {"YYYY-MM-DD": value}} + reform = None + if policy_data and policy_data.get("parameter_values"): + reform = {} + for pv in policy_data["parameter_values"]: + param_name = pv.get("parameter_name") + value = pv.get("value") + start_date = pv.get("start_date") + + if param_name and start_date: + # Parse ISO date string to get just the date part + if "T" in start_date: + date_str = start_date.split("T")[0] + else: + date_str = start_date + + if param_name not in reform: + reform[param_name] = {} + reform[param_name][date_str] = value + + # Create Microsimulation with reform applied at construction time + # This ensures the reform is properly integrated into the tax benefit system + microsim = Microsimulation(reform=reform) + + # Build simulation from entity data using SimulationBuilder + person_df = pd.DataFrame(person_data) + + # Determine column naming convention + household_id_col = ( + "person_household_id" + if "person_household_id" in person_df.columns + else "household_id" ) - marital_unit_df = MicroDataFrame( - pd.DataFrame(marital_unit_data), weights="marital_unit_weight" + marital_unit_id_col = ( + "person_marital_unit_id" + if "person_marital_unit_id" in person_df.columns + else "marital_unit_id" ) - family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") - spm_unit_df = MicroDataFrame(pd.DataFrame(spm_unit_data), weights="spm_unit_weight") - tax_unit_df = MicroDataFrame(pd.DataFrame(tax_unit_data), weights="tax_unit_weight") - - # Create temporary dataset - tmpdir = tempfile.mkdtemp() - filepath = str(Path(tmpdir) / "household_calc.h5") - - dataset = PolicyEngineUSDataset( - name="Household calculation", - description="Household(s) for calculation", - filepath=filepath, - year=year, - data=USYearData( - person=person_df, - household=household_df, - marital_unit=marital_unit_df, - family=family_df, - spm_unit=spm_unit_df, - tax_unit=tax_unit_df, - ), + family_id_col = ( + "person_family_id" if "person_family_id" in person_df.columns else "family_id" + ) + spm_unit_id_col = ( + "person_spm_unit_id" + if "person_spm_unit_id" in person_df.columns + else "spm_unit_id" + ) + tax_unit_id_col = ( + "person_tax_unit_id" + if "person_tax_unit_id" in person_df.columns + else "tax_unit_id" ) - # Build policy if provided - policy = None - if policy_data: - from policyengine.core.policy import ParameterValue as PEParameterValue - from policyengine.core.policy import Policy as PEPolicy - - pe_param_values = [] - param_lookup = {p.name: p for p in us_latest.parameters} - for pv in policy_data.get("parameter_values", []): - pe_param = param_lookup.get(pv["parameter_name"]) - if pe_param: - pe_pv = PEParameterValue( - parameter=pe_param, - value=pv["value"], - start_date=datetime.fromisoformat(pv["start_date"]) - if pv.get("start_date") - else None, - end_date=datetime.fromisoformat(pv["end_date"]) - if pv.get("end_date") - else None, - ) - pe_param_values.append(pe_pv) - policy = PEPolicy( - name=policy_data.get("name", ""), - description=policy_data.get("description", ""), - parameter_values=pe_param_values, - ) + # Declare entities using SimulationBuilder + builder = SimulationBuilder() + builder.populations = system.instantiate_entities() + + builder.declare_person_entity("person", person_df["person_id"].values) + builder.declare_entity("household", np.unique(person_df[household_id_col].values)) + builder.declare_entity("spm_unit", np.unique(person_df[spm_unit_id_col].values)) + builder.declare_entity("family", np.unique(person_df[family_id_col].values)) + builder.declare_entity("tax_unit", np.unique(person_df[tax_unit_id_col].values)) + builder.declare_entity( + "marital_unit", np.unique(person_df[marital_unit_id_col].values) + ) - # Run simulation - simulation = Simulation( - dataset=dataset, - tax_benefit_model_version=us_latest, - policy=policy, + # Join persons to group entities + builder.join_with_persons( + builder.populations["household"], + person_df[household_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["spm_unit"], + person_df[spm_unit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["family"], + person_df[family_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["tax_unit"], + person_df[tax_unit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["marital_unit"], + person_df[marital_unit_id_col].values, + np.array(["member"] * len(person_df)), ) - simulation.run() - # Extract outputs - output_data = simulation.output_dataset.data + # Build simulation from populations + microsim.build_from_populations(builder.populations) + + # Set input variables for each entity + id_columns = { + "person_id", + "household_id", + "person_household_id", + "spm_unit_id", + "person_spm_unit_id", + "family_id", + "person_family_id", + "tax_unit_id", + "person_tax_unit_id", + "marital_unit_id", + "person_marital_unit_id", + } + for entity_name, entity_df in [ + ("person", person_data), + ("household", household_data), + ("spm_unit", spm_unit_data), + ("family", family_data), + ("tax_unit", tax_unit_data), + ("marital_unit", marital_unit_data), + ]: + df = pd.DataFrame(entity_df) + for column in df.columns: + if column not in id_columns and column in system.variables: + microsim.set_input(column, year, df[column].values) + + # Calculate output variables def safe_convert(value): try: return float(value) except (ValueError, TypeError): return str(value) - def extract_entity_outputs(entity_name: str, entity_data, n_rows: int) -> list[dict]: + def extract_entity_outputs( + entity_name: str, n_rows: int, map_to: str + ) -> list[dict]: outputs = [] for i in range(n_rows): row_dict = {} for var in us_latest.entity_variables[entity_name]: - row_dict[var] = safe_convert(entity_data[var].iloc[i]) + val = microsim.calculate(var, period=year, map_to=map_to) + row_dict[var] = safe_convert(val.values[i]) outputs.append(row_dict) return outputs return { - "person": extract_entity_outputs("person", output_data.person, n_people), + "person": extract_entity_outputs("person", n_people, "person"), "marital_unit": extract_entity_outputs( - "marital_unit", output_data.marital_unit, len(output_data.marital_unit) - ), - "family": extract_entity_outputs( - "family", output_data.family, len(output_data.family) - ), - "spm_unit": extract_entity_outputs( - "spm_unit", output_data.spm_unit, len(output_data.spm_unit) - ), - "tax_unit": extract_entity_outputs( - "tax_unit", output_data.tax_unit, len(output_data.tax_unit) - ), - "household": extract_entity_outputs( - "household", output_data.household, len(output_data.household) + "marital_unit", n_marital_units, "marital_unit" ), + "family": extract_entity_outputs("family", n_families, "family"), + "spm_unit": extract_entity_outputs("spm_unit", n_spm_units, "spm_unit"), + "tax_unit": extract_entity_outputs("tax_unit", n_tax_units, "tax_unit"), + "household": extract_entity_outputs("household", n_households, "household"), } diff --git a/src/policyengine_api/api/household_analysis.py b/src/policyengine_api/api/household_analysis.py index 981d968..5f36fda 100644 --- a/src/policyengine_api/api/household_analysis.py +++ b/src/policyengine_api/api/household_analysis.py @@ -158,22 +158,45 @@ def _ensure_list(value: Any) -> list: def _extract_policy_data(policy: Policy | None) -> dict | None: - """Extract policy data from a Policy model into calculation format.""" + """Extract policy data from a Policy model into calculation format. + + Returns format expected by _calculate_household_us/_calculate_household_uk: + { + "name": "policy name", + "description": "policy description", + "parameter_values": [ + { + "parameter_name": "gov.irs.credits.ctc...", + "value": 0.16, + "start_date": "2024-01-01T00:00:00+00:00", + "end_date": null + } + ] + } + """ if not policy or not policy.parameter_values: return None - policy_data = {} + parameter_values = [] for pv in policy.parameter_values: if not pv.parameter: continue - policy_data[pv.parameter.name] = { + parameter_values.append({ + "parameter_name": pv.parameter.name, "value": _extract_value(pv.value_json), "start_date": _format_date(pv.start_date), "end_date": _format_date(pv.end_date), - } + }) + + if not parameter_values: + return None - return policy_data if policy_data else None + return { + "name": policy.name, + "description": policy.description or "", + "parameter_values": parameter_values, + } def _extract_value(value_json: Any) -> Any: diff --git a/src/policyengine_api/modal_app.py b/src/policyengine_api/modal_app.py index 14083cf..2b486f3 100644 --- a/src/policyengine_api/modal_app.py +++ b/src/policyengine_api/modal_app.py @@ -807,7 +807,6 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N raise ValueError(f"Dataset {simulation.dataset_id} not found") # Import policyengine - from policyengine.core import Simulation as PESimulation from policyengine.tax_benefit_models.uk import uk_latest from policyengine.tax_benefit_models.uk.datasets import ( PolicyEngineUKDataset, @@ -815,7 +814,7 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N pe_model_version = uk_latest - # Get policy and dynamic + # Get policy and dynamic as PEPolicy/PEDynamic objects policy = _get_pe_policy_uk( simulation.policy_id, pe_model_version, session ) @@ -823,6 +822,13 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N simulation.dynamic_id, pe_model_version, session ) + # Convert to reform dict format for Microsimulation + # This is necessary because policyengine.core.Simulation applies + # reforms AFTER creating Microsimulation, which doesn't work + policy_reform = _pe_policy_to_reform_dict(policy) + dynamic_reform = _pe_policy_to_reform_dict(dynamic) + reform = _merge_reform_dicts(policy_reform, dynamic_reform) + # Download dataset local_path = download_dataset( dataset.filepath, supabase_url, supabase_key, storage_bucket @@ -835,15 +841,12 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N year=dataset.year, ) - # Create and run simulation + # Run simulation using Microsimulation directly with reform + # This ensures reforms are applied at construction time with logfire.span("run_simulation"): - pe_sim = PESimulation( - dataset=pe_dataset, - tax_benefit_model_version=pe_model_version, - policy=policy, - dynamic=dynamic, + pe_output_dataset = _run_uk_economy_simulation( + pe_dataset, reform, pe_model_version, simulation_id ) - pe_sim.ensure() # Save output dataset with logfire.span("save_output_dataset"): @@ -853,8 +856,8 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N output_path = f"/tmp/{output_filename}" # Set filepath and save - pe_sim.output_dataset.filepath = output_path - pe_sim.output_dataset.save() + pe_output_dataset.filepath = output_path + pe_output_dataset.save() # Upload to Supabase storage supabase = create_client(supabase_url, supabase_key) @@ -869,7 +872,7 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N ) # Create output dataset record - output_dataset = Dataset( + output_dataset_record = Dataset( name=f"Output: {dataset.name}", description=f"Output from simulation {simulation_id}", filepath=output_filename, @@ -877,12 +880,12 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N is_output_dataset=True, tax_benefit_model_id=dataset.tax_benefit_model_id, ) - session.add(output_dataset) + session.add(output_dataset_record) session.commit() - session.refresh(output_dataset) + session.refresh(output_dataset_record) # Link to simulation - simulation.output_dataset_id = output_dataset.id + simulation.output_dataset_id = output_dataset_record.id # Mark as completed simulation.status = SimulationStatus.COMPLETED @@ -973,15 +976,15 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N raise ValueError(f"Dataset {simulation.dataset_id} not found") # Import policyengine - from policyengine.core import Simulation as PESimulation from policyengine.tax_benefit_models.us import us_latest from policyengine.tax_benefit_models.us.datasets import ( PolicyEngineUSDataset, + USYearData, ) pe_model_version = us_latest - # Get policy and dynamic + # Get policy and dynamic as PEPolicy/PEDynamic objects policy = _get_pe_policy_us( simulation.policy_id, pe_model_version, session ) @@ -989,6 +992,13 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N simulation.dynamic_id, pe_model_version, session ) + # Convert to reform dict format for Microsimulation + # This is necessary because policyengine.core.Simulation applies + # reforms AFTER creating Microsimulation, which doesn't work + policy_reform = _pe_policy_to_reform_dict(policy) + dynamic_reform = _pe_policy_to_reform_dict(dynamic) + reform = _merge_reform_dicts(policy_reform, dynamic_reform) + # Download dataset local_path = download_dataset( dataset.filepath, supabase_url, supabase_key, storage_bucket @@ -1001,15 +1011,12 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N year=dataset.year, ) - # Create and run simulation + # Run simulation using Microsimulation directly with reform + # This ensures reforms are applied at construction time with logfire.span("run_simulation"): - pe_sim = PESimulation( - dataset=pe_dataset, - tax_benefit_model_version=pe_model_version, - policy=policy, - dynamic=dynamic, + pe_output_dataset = _run_us_economy_simulation( + pe_dataset, reform, pe_model_version, simulation_id ) - pe_sim.ensure() # Save output dataset with logfire.span("save_output_dataset"): @@ -1019,8 +1026,8 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N output_path = f"/tmp/{output_filename}" # Set filepath and save - pe_sim.output_dataset.filepath = output_path - pe_sim.output_dataset.save() + pe_output_dataset.filepath = output_path + pe_output_dataset.save() # Upload to Supabase storage supabase = create_client(supabase_url, supabase_key) @@ -1035,7 +1042,7 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N ) # Create output dataset record - output_dataset = Dataset( + output_dataset_record = Dataset( name=f"Output: {dataset.name}", description=f"Output from simulation {simulation_id}", filepath=output_filename, @@ -1043,12 +1050,12 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N is_output_dataset=True, tax_benefit_model_id=dataset.tax_benefit_model_id, ) - session.add(output_dataset) + session.add(output_dataset_record) session.commit() - session.refresh(output_dataset) + session.refresh(output_dataset_record) # Link to simulation - simulation.output_dataset_id = output_dataset.id + simulation.output_dataset_id = output_dataset_record.id # Mark as completed simulation.status = SimulationStatus.COMPLETED @@ -1816,6 +1823,403 @@ def _get_pe_dynamic_us(dynamic_id, model_version, session): return _get_pe_dynamic_uk(dynamic_id, model_version, session) +def _pe_policy_to_reform_dict(policy) -> dict | None: + """Convert a policyengine.core.policy.Policy to reform dict format. + + The policyengine-us/uk Microsimulation expects reforms in the format: + {"parameter.name": {"YYYY-MM-DD": value}} + + This is necessary because the policyengine.core.Simulation applies reforms + AFTER creating the Microsimulation, which doesn't work due to caching. + We need to pass the reform at Microsimulation construction time. + """ + if policy is None: + return None + + if not policy.parameter_values: + return None + + reform = {} + for pv in policy.parameter_values: + if not pv.parameter: + continue + param_name = pv.parameter.name + value = pv.value + start_date = pv.start_date + + if param_name and start_date: + # Format date as YYYY-MM-DD string + if hasattr(start_date, "strftime"): + date_str = start_date.strftime("%Y-%m-%d") + else: + date_str = str(start_date).split("T")[0] + + if param_name not in reform: + reform[param_name] = {} + reform[param_name][date_str] = value + + return reform if reform else None + + +def _merge_reform_dicts(reform1: dict | None, reform2: dict | None) -> dict | None: + """Merge two reform dicts, with reform2 taking precedence.""" + if reform1 is None and reform2 is None: + return None + if reform1 is None: + return reform2 + if reform2 is None: + return reform1 + + merged = dict(reform1) + for param_name, dates in reform2.items(): + if param_name not in merged: + merged[param_name] = {} + merged[param_name].update(dates) + return merged + + +def _run_us_economy_simulation(pe_dataset, reform, pe_model_version, simulation_id): + """Run US economy simulation using Microsimulation directly. + + This bypasses policyengine.core.Simulation which has a bug where reforms + are applied AFTER creating Microsimulation (when it's too late). + Instead, we pass the reform dict at Microsimulation construction time. + """ + from pathlib import Path + + import numpy as np + import pandas as pd + from microdf import MicroDataFrame + from policyengine.tax_benefit_models.us.datasets import ( + PolicyEngineUSDataset, + USYearData, + ) + from policyengine_core.simulations.simulation_builder import SimulationBuilder + from policyengine_us import Microsimulation + from policyengine_us.system import system + + # Load dataset + pe_dataset.load() + year = pe_dataset.year + + # Create Microsimulation with reform applied at construction time + microsim = Microsimulation(reform=reform) + + # Build simulation from dataset using SimulationBuilder + person_df = pd.DataFrame(pe_dataset.data.person) + + # Determine column naming convention + household_id_col = ( + "person_household_id" + if "person_household_id" in person_df.columns + else "household_id" + ) + marital_unit_id_col = ( + "person_marital_unit_id" + if "person_marital_unit_id" in person_df.columns + else "marital_unit_id" + ) + family_id_col = ( + "person_family_id" if "person_family_id" in person_df.columns else "family_id" + ) + spm_unit_id_col = ( + "person_spm_unit_id" + if "person_spm_unit_id" in person_df.columns + else "spm_unit_id" + ) + tax_unit_id_col = ( + "person_tax_unit_id" + if "person_tax_unit_id" in person_df.columns + else "tax_unit_id" + ) + + # Declare entities + builder = SimulationBuilder() + builder.populations = system.instantiate_entities() + + builder.declare_person_entity("person", person_df["person_id"].values) + builder.declare_entity("household", np.unique(person_df[household_id_col].values)) + builder.declare_entity("spm_unit", np.unique(person_df[spm_unit_id_col].values)) + builder.declare_entity("family", np.unique(person_df[family_id_col].values)) + builder.declare_entity("tax_unit", np.unique(person_df[tax_unit_id_col].values)) + builder.declare_entity( + "marital_unit", np.unique(person_df[marital_unit_id_col].values) + ) + + # Join persons to entities + builder.join_with_persons( + builder.populations["household"], + person_df[household_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["spm_unit"], + person_df[spm_unit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["family"], + person_df[family_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["tax_unit"], + person_df[tax_unit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["marital_unit"], + person_df[marital_unit_id_col].values, + np.array(["member"] * len(person_df)), + ) + + microsim.build_from_populations(builder.populations) + + # Set input variables + id_columns = { + "person_id", + "household_id", + "person_household_id", + "spm_unit_id", + "person_spm_unit_id", + "family_id", + "person_family_id", + "tax_unit_id", + "person_tax_unit_id", + "marital_unit_id", + "person_marital_unit_id", + } + + for entity_name, entity_data in [ + ("person", pe_dataset.data.person), + ("household", pe_dataset.data.household), + ("spm_unit", pe_dataset.data.spm_unit), + ("family", pe_dataset.data.family), + ("tax_unit", pe_dataset.data.tax_unit), + ("marital_unit", pe_dataset.data.marital_unit), + ]: + df = pd.DataFrame(entity_data) + for column in df.columns: + if column not in id_columns and column in system.variables: + microsim.set_input(column, year, df[column].values) + + # Calculate output variables and build output dataset + data = { + "person": pd.DataFrame(), + "marital_unit": pd.DataFrame(), + "family": pd.DataFrame(), + "spm_unit": pd.DataFrame(), + "tax_unit": pd.DataFrame(), + "household": pd.DataFrame(), + } + + weight_columns = { + "person_weight", + "household_weight", + "marital_unit_weight", + "family_weight", + "spm_unit_weight", + "tax_unit_weight", + } + + # Copy ID and weight columns from input dataset + for entity in data.keys(): + input_df = pd.DataFrame(getattr(pe_dataset.data, entity)) + entity_id_col = f"{entity}_id" + entity_weight_col = f"{entity}_weight" + + if entity_id_col in input_df.columns: + data[entity][entity_id_col] = input_df[entity_id_col].values + if entity_weight_col in input_df.columns: + data[entity][entity_weight_col] = input_df[entity_weight_col].values + + # Copy person-level group ID columns + for col in person_df.columns: + if col.startswith("person_") and col.endswith("_id"): + target_col = col.replace("person_", "") + if target_col in id_columns: + data["person"][target_col] = person_df[col].values + + # Calculate non-ID, non-weight variables + for entity, variables in pe_model_version.entity_variables.items(): + for var in variables: + if var not in id_columns and var not in weight_columns: + data[entity][var] = microsim.calculate( + var, period=year, map_to=entity + ).values + + # Convert to MicroDataFrames + data["person"] = MicroDataFrame(data["person"], weights="person_weight") + data["marital_unit"] = MicroDataFrame( + data["marital_unit"], weights="marital_unit_weight" + ) + data["family"] = MicroDataFrame(data["family"], weights="family_weight") + data["spm_unit"] = MicroDataFrame(data["spm_unit"], weights="spm_unit_weight") + data["tax_unit"] = MicroDataFrame(data["tax_unit"], weights="tax_unit_weight") + data["household"] = MicroDataFrame(data["household"], weights="household_weight") + + # Create output dataset + return PolicyEngineUSDataset( + id=simulation_id, + name=pe_dataset.name, + description=pe_dataset.description, + filepath=str(Path(pe_dataset.filepath).parent / (simulation_id + ".h5")), + year=year, + is_output_dataset=True, + data=USYearData( + person=data["person"], + marital_unit=data["marital_unit"], + family=data["family"], + spm_unit=data["spm_unit"], + tax_unit=data["tax_unit"], + household=data["household"], + ), + ) + + +def _run_uk_economy_simulation(pe_dataset, reform, pe_model_version, simulation_id): + """Run UK economy simulation using Microsimulation directly. + + This bypasses policyengine.core.Simulation which has a bug where reforms + are applied AFTER creating Microsimulation (when it's too late). + Instead, we pass the reform dict at Microsimulation construction time. + """ + from pathlib import Path + + import numpy as np + import pandas as pd + from microdf import MicroDataFrame + from policyengine.tax_benefit_models.uk.datasets import ( + PolicyEngineUKDataset, + UKYearData, + ) + from policyengine_core.simulations.simulation_builder import SimulationBuilder + from policyengine_uk import Microsimulation + from policyengine_uk.system import system + + # Load dataset + pe_dataset.load() + year = pe_dataset.year + + # Create Microsimulation with reform applied at construction time + microsim = Microsimulation(reform=reform) + + # Build simulation from dataset using SimulationBuilder + person_df = pd.DataFrame(pe_dataset.data.person) + + # Determine column naming convention + benunit_id_col = ( + "person_benunit_id" + if "person_benunit_id" in person_df.columns + else "benunit_id" + ) + household_id_col = ( + "person_household_id" + if "person_household_id" in person_df.columns + else "household_id" + ) + + # Declare entities + builder = SimulationBuilder() + builder.populations = system.instantiate_entities() + + builder.declare_person_entity("person", person_df["person_id"].values) + builder.declare_entity("benunit", np.unique(person_df[benunit_id_col].values)) + builder.declare_entity("household", np.unique(person_df[household_id_col].values)) + + # Join persons to entities + builder.join_with_persons( + builder.populations["benunit"], + person_df[benunit_id_col].values, + np.array(["member"] * len(person_df)), + ) + builder.join_with_persons( + builder.populations["household"], + person_df[household_id_col].values, + np.array(["member"] * len(person_df)), + ) + + microsim.build_from_populations(builder.populations) + + # Set input variables + id_columns = { + "person_id", + "benunit_id", + "person_benunit_id", + "household_id", + "person_household_id", + } + + for entity_name, entity_data in [ + ("person", pe_dataset.data.person), + ("benunit", pe_dataset.data.benunit), + ("household", pe_dataset.data.household), + ]: + df = pd.DataFrame(entity_data) + for column in df.columns: + if column not in id_columns and column in system.variables: + microsim.set_input(column, year, df[column].values) + + # Calculate output variables and build output dataset + data = { + "person": pd.DataFrame(), + "benunit": pd.DataFrame(), + "household": pd.DataFrame(), + } + + weight_columns = { + "person_weight", + "benunit_weight", + "household_weight", + } + + # Copy ID and weight columns from input dataset + for entity in data.keys(): + input_df = pd.DataFrame(getattr(pe_dataset.data, entity)) + entity_id_col = f"{entity}_id" + entity_weight_col = f"{entity}_weight" + + if entity_id_col in input_df.columns: + data[entity][entity_id_col] = input_df[entity_id_col].values + if entity_weight_col in input_df.columns: + data[entity][entity_weight_col] = input_df[entity_weight_col].values + + # Copy person-level group ID columns + for col in person_df.columns: + if col.startswith("person_") and col.endswith("_id"): + target_col = col.replace("person_", "") + if target_col in id_columns: + data["person"][target_col] = person_df[col].values + + # Calculate non-ID, non-weight variables + for entity, variables in pe_model_version.entity_variables.items(): + for var in variables: + if var not in id_columns and var not in weight_columns: + data[entity][var] = microsim.calculate( + var, period=year, map_to=entity + ).values + + # Convert to MicroDataFrames + data["person"] = MicroDataFrame(data["person"], weights="person_weight") + data["benunit"] = MicroDataFrame(data["benunit"], weights="benunit_weight") + data["household"] = MicroDataFrame(data["household"], weights="household_weight") + + # Create output dataset + return PolicyEngineUKDataset( + id=simulation_id, + name=pe_dataset.name, + description=pe_dataset.description, + filepath=str(Path(pe_dataset.filepath).parent / (simulation_id + ".h5")), + year=year, + is_output_dataset=True, + data=UKYearData( + person=data["person"], + benunit=data["benunit"], + household=data["household"], + ), + ) + + @app.function( image=uk_image, secrets=[db_secrets, logfire_secrets], diff --git a/test_fixtures/fixtures_policy_reform.py b/test_fixtures/fixtures_policy_reform.py new file mode 100644 index 0000000..f7534a5 --- /dev/null +++ b/test_fixtures/fixtures_policy_reform.py @@ -0,0 +1,282 @@ +"""Fixtures for policy reform conversion tests.""" + +from dataclasses import dataclass +from datetime import date, datetime +from typing import Any + + +# ============================================================================= +# Mock objects for testing _pe_policy_to_reform_dict +# ============================================================================= + + +@dataclass +class MockParameter: + """Mock policyengine.core.models.parameter.Parameter.""" + + name: str + + +@dataclass +class MockParameterValue: + """Mock policyengine.core.models.parameter_value.ParameterValue.""" + + parameter: MockParameter | None + value: Any + start_date: date | datetime | str | None + + +@dataclass +class MockPolicy: + """Mock policyengine.core.policy.Policy.""" + + parameter_values: list[MockParameterValue] | None + + +# ============================================================================= +# Test data constants +# ============================================================================= + +# Simple policy with single parameter change +SIMPLE_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date=date(2024, 1, 1), + ) + ] +) + +SIMPLE_POLICY_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} +} + +# Policy with multiple parameter changes +MULTI_PARAM_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date=date(2024, 1, 1), + ), + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.refundable.fully_refundable"), + value=True, + start_date=date(2024, 1, 1), + ), + MockParameterValue( + parameter=MockParameter(name="gov.irs.income.bracket.rates.1"), + value=0.12, + start_date=date(2024, 1, 1), + ), + ] +) + +MULTI_PARAM_POLICY_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, + "gov.irs.credits.ctc.refundable.fully_refundable": {"2024-01-01": True}, + "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.12}, +} + +# Policy with same parameter at different dates +MULTI_DATE_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=2500, + start_date=date(2024, 1, 1), + ), + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date=date(2025, 1, 1), + ), + ] +) + +MULTI_DATE_POLICY_EXPECTED = { + "gov.irs.credits.ctc.amount.base": { + "2024-01-01": 2500, + "2025-01-01": 3000, + } +} + +# Policy with datetime start_date (has time component) +DATETIME_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date=datetime(2024, 1, 1, 12, 30, 45), + ) + ] +) + +DATETIME_POLICY_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} +} + +# Policy with ISO string start_date +ISO_STRING_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date="2024-01-01T00:00:00", + ) + ] +) + +ISO_STRING_POLICY_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} +} + +# Empty policy (no parameter values) +EMPTY_POLICY = MockPolicy(parameter_values=[]) + +# None policy +NONE_POLICY = None + +# Policy with None parameter_values +NONE_PARAM_VALUES_POLICY = MockPolicy(parameter_values=None) + +# Policy with invalid entries (missing parameter or start_date) +INVALID_ENTRIES_POLICY = MockPolicy( + parameter_values=[ + MockParameterValue( + parameter=None, # Missing parameter + value=3000, + start_date=date(2024, 1, 1), + ), + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), + value=3000, + start_date=None, # Missing start_date + ), + MockParameterValue( + parameter=MockParameter(name="gov.irs.credits.eitc.max.0"), + value=600, + start_date=date(2024, 1, 1), # This one is valid + ), + ] +) + +INVALID_ENTRIES_POLICY_EXPECTED = { + "gov.irs.credits.eitc.max.0": {"2024-01-01": 600} +} + + +# ============================================================================= +# Test data for _merge_reform_dicts +# ============================================================================= + +REFORM_DICT_1 = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 2000}, + "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.10}, +} + +REFORM_DICT_2 = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, # Overwrites + "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, # New param +} + +MERGED_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, # From reform2 + "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.10}, # From reform1 + "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, # From reform2 +} + +REFORM_DICT_3 = { + "gov.irs.credits.ctc.amount.base": { + "2024-01-01": 2500, + "2025-01-01": 2700, + }, +} + +REFORM_DICT_4 = { + "gov.irs.credits.ctc.amount.base": { + "2025-01-01": 3000, # Overwrites 2025 date + "2026-01-01": 3500, # New date + }, +} + +MERGED_MULTI_DATE_EXPECTED = { + "gov.irs.credits.ctc.amount.base": { + "2024-01-01": 2500, # From reform3 + "2025-01-01": 3000, # From reform4 (overwrites) + "2026-01-01": 3500, # From reform4 (new) + }, +} + + +# ============================================================================= +# Test data for household calculation policy conversion +# ============================================================================= + +# Policy data as it comes from the API (stored in database) +HOUSEHOLD_POLICY_DATA = { + "parameter_values": [ + { + "parameter_name": "gov.irs.credits.ctc.amount.base", + "value": 3000, + "start_date": "2024-01-01", + }, + { + "parameter_name": "gov.irs.credits.ctc.refundable.fully_refundable", + "value": True, + "start_date": "2024-01-01", + }, + ] +} + +HOUSEHOLD_POLICY_DATA_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, + "gov.irs.credits.ctc.refundable.fully_refundable": {"2024-01-01": True}, +} + +# Policy data with ISO datetime strings +HOUSEHOLD_POLICY_DATA_DATETIME = { + "parameter_values": [ + { + "parameter_name": "gov.irs.credits.ctc.amount.base", + "value": 3000, + "start_date": "2024-01-01T00:00:00.000Z", + }, + ] +} + +HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED = { + "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, +} + +# Empty policy data +HOUSEHOLD_EMPTY_POLICY_DATA = {"parameter_values": []} + +# None policy data +HOUSEHOLD_NONE_POLICY_DATA = None + +# Policy data with missing fields +HOUSEHOLD_INCOMPLETE_POLICY_DATA = { + "parameter_values": [ + { + "parameter_name": None, # Missing + "value": 3000, + "start_date": "2024-01-01", + }, + { + "parameter_name": "gov.irs.credits.ctc.amount.base", + "value": 3000, + "start_date": None, # Missing + }, + { + "parameter_name": "gov.irs.credits.eitc.max.0", + "value": 600, + "start_date": "2024-01-01", # Valid + }, + ] +} + +HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED = { + "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, +} diff --git a/tests/test_policy_reform.py b/tests/test_policy_reform.py new file mode 100644 index 0000000..cfee3b8 --- /dev/null +++ b/tests/test_policy_reform.py @@ -0,0 +1,327 @@ +"""Tests for policy reform conversion logic. + +Tests the helper functions that convert policy objects to reform dict format +for use with Microsimulation. These are critical for fixing the bug where +reforms weren't being applied to economy-wide and household simulations. +""" + +import sys +from unittest.mock import MagicMock + +import pytest + +# Mock modal before importing modal_app +sys.modules["modal"] = MagicMock() + +from test_fixtures.fixtures_policy_reform import ( + DATETIME_POLICY, + DATETIME_POLICY_EXPECTED, + EMPTY_POLICY, + HOUSEHOLD_EMPTY_POLICY_DATA, + HOUSEHOLD_INCOMPLETE_POLICY_DATA, + HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED, + HOUSEHOLD_NONE_POLICY_DATA, + HOUSEHOLD_POLICY_DATA, + HOUSEHOLD_POLICY_DATA_DATETIME, + HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED, + HOUSEHOLD_POLICY_DATA_EXPECTED, + INVALID_ENTRIES_POLICY, + INVALID_ENTRIES_POLICY_EXPECTED, + ISO_STRING_POLICY, + ISO_STRING_POLICY_EXPECTED, + MERGED_EXPECTED, + MERGED_MULTI_DATE_EXPECTED, + MULTI_DATE_POLICY, + MULTI_DATE_POLICY_EXPECTED, + MULTI_PARAM_POLICY, + MULTI_PARAM_POLICY_EXPECTED, + NONE_PARAM_VALUES_POLICY, + NONE_POLICY, + REFORM_DICT_1, + REFORM_DICT_2, + REFORM_DICT_3, + REFORM_DICT_4, + SIMPLE_POLICY, + SIMPLE_POLICY_EXPECTED, +) + +# Import after mocking modal +from policyengine_api.modal_app import _merge_reform_dicts, _pe_policy_to_reform_dict + + +class TestPePolicyToReformDict: + """Tests for _pe_policy_to_reform_dict function.""" + + # ========================================================================= + # Given: Valid policy with single parameter + # ========================================================================= + + def test__given_simple_policy_with_date_object__then_returns_correct_reform_dict( + self, + ): + """Given a policy with a single parameter using date object, + then returns correctly formatted reform dict.""" + # When + result = _pe_policy_to_reform_dict(SIMPLE_POLICY) + + # Then + assert result == SIMPLE_POLICY_EXPECTED + + def test__given_policy_with_datetime_object__then_extracts_date_correctly(self): + """Given a policy with datetime start_date (has time component), + then extracts just the date part for the reform dict.""" + # When + result = _pe_policy_to_reform_dict(DATETIME_POLICY) + + # Then + assert result == DATETIME_POLICY_EXPECTED + + def test__given_policy_with_iso_string_date__then_parses_date_correctly(self): + """Given a policy with ISO string start_date, + then parses and extracts the date correctly.""" + # When + result = _pe_policy_to_reform_dict(ISO_STRING_POLICY) + + # Then + assert result == ISO_STRING_POLICY_EXPECTED + + # ========================================================================= + # Given: Policy with multiple parameters + # ========================================================================= + + def test__given_policy_with_multiple_parameters__then_includes_all_in_dict(self): + """Given a policy with multiple parameter changes, + then includes all parameters in the reform dict.""" + # When + result = _pe_policy_to_reform_dict(MULTI_PARAM_POLICY) + + # Then + assert result == MULTI_PARAM_POLICY_EXPECTED + + def test__given_policy_with_same_param_multiple_dates__then_includes_all_dates( + self, + ): + """Given a policy with the same parameter changed at different dates, + then includes all date entries for that parameter.""" + # When + result = _pe_policy_to_reform_dict(MULTI_DATE_POLICY) + + # Then + assert result == MULTI_DATE_POLICY_EXPECTED + + # ========================================================================= + # Given: Empty or None policy + # ========================================================================= + + def test__given_none_policy__then_returns_none(self): + """Given None as policy, + then returns None.""" + # When + result = _pe_policy_to_reform_dict(NONE_POLICY) + + # Then + assert result is None + + def test__given_policy_with_empty_parameter_values__then_returns_none(self): + """Given a policy with empty parameter_values list, + then returns None.""" + # When + result = _pe_policy_to_reform_dict(EMPTY_POLICY) + + # Then + assert result is None + + def test__given_policy_with_none_parameter_values__then_returns_none(self): + """Given a policy with parameter_values=None, + then returns None.""" + # When + result = _pe_policy_to_reform_dict(NONE_PARAM_VALUES_POLICY) + + # Then + assert result is None + + # ========================================================================= + # Given: Policy with invalid entries + # ========================================================================= + + def test__given_policy_with_invalid_entries__then_skips_invalid_keeps_valid(self): + """Given a policy with some invalid entries (missing parameter or date), + then skips invalid entries and keeps valid ones.""" + # When + result = _pe_policy_to_reform_dict(INVALID_ENTRIES_POLICY) + + # Then + assert result == INVALID_ENTRIES_POLICY_EXPECTED + + +class TestMergeReformDicts: + """Tests for _merge_reform_dicts function.""" + + # ========================================================================= + # Given: Two valid reform dicts + # ========================================================================= + + def test__given_two_reform_dicts__then_merges_with_second_taking_precedence(self): + """Given two reform dicts with overlapping parameters, + then merges them with the second dict taking precedence.""" + # When + result = _merge_reform_dicts(REFORM_DICT_1, REFORM_DICT_2) + + # Then + assert result == MERGED_EXPECTED + + def test__given_dicts_with_multiple_dates__then_merges_date_entries_correctly(self): + """Given reform dicts with same parameter at multiple dates, + then merges date entries correctly with second taking precedence.""" + # When + result = _merge_reform_dicts(REFORM_DICT_3, REFORM_DICT_4) + + # Then + assert result == MERGED_MULTI_DATE_EXPECTED + + # ========================================================================= + # Given: None values + # ========================================================================= + + def test__given_both_none__then_returns_none(self): + """Given both reform dicts are None, + then returns None.""" + # When + result = _merge_reform_dicts(None, None) + + # Then + assert result is None + + def test__given_first_none__then_returns_second(self): + """Given first reform dict is None, + then returns the second dict.""" + # When + result = _merge_reform_dicts(None, REFORM_DICT_1) + + # Then + assert result == REFORM_DICT_1 + + def test__given_second_none__then_returns_first(self): + """Given second reform dict is None, + then returns the first dict.""" + # When + result = _merge_reform_dicts(REFORM_DICT_1, None) + + # Then + assert result == REFORM_DICT_1 + + # ========================================================================= + # Given: Original dict should not be mutated + # ========================================================================= + + def test__given_two_dicts__then_does_not_mutate_original_dicts(self): + """Given two reform dicts, + then merging does not mutate the original dicts.""" + # Given + original_dict1 = {"param.a": {"2024-01-01": 100}} + original_dict2 = {"param.b": {"2024-01-01": 200}} + dict1_copy = dict(original_dict1) + dict2_copy = dict(original_dict2) + + # When + _merge_reform_dicts(original_dict1, original_dict2) + + # Then + assert original_dict1 == dict1_copy + assert original_dict2 == dict2_copy + + +class TestHouseholdPolicyDataConversion: + """Tests for the policy data conversion logic used in household calculations. + + This tests the conversion logic as it appears in _calculate_household_us + and _calculate_household_uk functions. + """ + + def _convert_policy_data_to_reform(self, policy_data: dict | None) -> dict | None: + """Convert policy_data (from API) to reform dict format. + + This mirrors the conversion logic in _calculate_household_us. + """ + if not policy_data or not policy_data.get("parameter_values"): + return None + + reform = {} + for pv in policy_data["parameter_values"]: + param_name = pv.get("parameter_name") + value = pv.get("value") + start_date = pv.get("start_date") + + if param_name and start_date: + # Parse ISO date string to get just the date part + if "T" in start_date: + date_str = start_date.split("T")[0] + else: + date_str = start_date + + if param_name not in reform: + reform[param_name] = {} + reform[param_name][date_str] = value + + return reform if reform else None + + # ========================================================================= + # Given: Valid policy data from API + # ========================================================================= + + def test__given_valid_policy_data__then_converts_to_reform_dict(self): + """Given valid policy data from the API, + then converts it to the correct reform dict format.""" + # When + result = self._convert_policy_data_to_reform(HOUSEHOLD_POLICY_DATA) + + # Then + assert result == HOUSEHOLD_POLICY_DATA_EXPECTED + + def test__given_policy_data_with_datetime_strings__then_extracts_date_part(self): + """Given policy data with ISO datetime strings (with T and timezone), + then extracts just the date part.""" + # When + result = self._convert_policy_data_to_reform(HOUSEHOLD_POLICY_DATA_DATETIME) + + # Then + assert result == HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED + + # ========================================================================= + # Given: Empty or None policy data + # ========================================================================= + + def test__given_none_policy_data__then_returns_none(self): + """Given None policy data, + then returns None.""" + # When + result = self._convert_policy_data_to_reform(HOUSEHOLD_NONE_POLICY_DATA) + + # Then + assert result is None + + def test__given_empty_parameter_values__then_returns_none(self): + """Given policy data with empty parameter_values list, + then returns None.""" + # When + result = self._convert_policy_data_to_reform(HOUSEHOLD_EMPTY_POLICY_DATA) + + # Then + assert result is None + + # ========================================================================= + # Given: Incomplete policy data + # ========================================================================= + + def test__given_incomplete_entries__then_skips_invalid_keeps_valid(self): + """Given policy data with some entries missing required fields, + then skips invalid entries and keeps valid ones.""" + # When + result = self._convert_policy_data_to_reform(HOUSEHOLD_INCOMPLETE_POLICY_DATA) + + # Then + assert result == HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 34c34c47a57f87d733c4f847ff565d69d2589749 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 6 Feb 2026 20:10:08 +0300 Subject: [PATCH 10/87] fix: Fix household user models; add variable default values --- .../20260204_0002_add_household_support.py | 10 +++++- ...0260206_0004_add_variable_default_value.py | 34 +++++++++++++++++++ docker-compose.yml | 14 ++++---- scripts/seed_common.py | 13 +++++++ .../models/user_household_association.py | 3 +- src/policyengine_api/models/variable.py | 3 ++ 6 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/20260206_0004_add_variable_default_value.py diff --git a/alembic/versions/20260204_0002_add_household_support.py b/alembic/versions/20260204_0002_add_household_support.py index beb00a0..186db37 100644 --- a/alembic/versions/20260204_0002_add_household_support.py +++ b/alembic/versions/20260204_0002_add_household_support.py @@ -57,19 +57,27 @@ def upgrade() -> None: op.create_index("idx_households_year", "households", ["year"]) # User-household associations (many-to-many for saved households) + # Note: user_id is a client-generated UUID stored in localStorage, not a foreign key op.create_table( "user_household_associations", sa.Column("id", sa.Uuid(), nullable=False), sa.Column("user_id", sa.Uuid(), nullable=False), sa.Column("household_id", sa.Uuid(), nullable=False), + sa.Column("country_id", sa.String(), nullable=False), + sa.Column("label", sa.String(), nullable=True), sa.Column( "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False, ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["household_id"], ["households.id"], ondelete="CASCADE"), sa.UniqueConstraint("user_id", "household_id"), ) diff --git a/alembic/versions/20260206_0004_add_variable_default_value.py b/alembic/versions/20260206_0004_add_variable_default_value.py new file mode 100644 index 0000000..2471491 --- /dev/null +++ b/alembic/versions/20260206_0004_add_variable_default_value.py @@ -0,0 +1,34 @@ +"""Add default_value to variables + +Revision ID: 0004_var_default +Revises: 0003_param_idx +Create Date: 2026-02-06 03:30:00.000000 + +This migration adds a default_value column to the variables table. +The default_value is stored as JSON to handle different types (int, float, bool, str, etc.). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSON + +# revision identifiers, used by Alembic. +revision: str = "0004_var_default" +down_revision: Union[str, Sequence[str], None] = "0003_param_idx" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add default_value column to variables table.""" + op.add_column( + "variables", + sa.Column("default_value", JSON, nullable=True), + ) + + +def downgrade() -> None: + """Remove default_value column from variables table.""" + op.drop_column("variables", "default_value") diff --git a/docker-compose.yml b/docker-compose.yml index 60e8645..60aa598 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,10 @@ services: ports: - "${API_PORT:-8000}:${API_PORT:-8000}" environment: - SUPABASE_URL: http://supabase_kong_policyengine-api-v2:8000 + SUPABASE_URL: http://supabase_kong_policyengine-api-v2-alpha:8000 SUPABASE_KEY: ${SUPABASE_KEY} SUPABASE_SERVICE_KEY: ${SUPABASE_SERVICE_KEY} - SUPABASE_DB_URL: postgresql://postgres:postgres@supabase_db_policyengine-api-v2:5432/postgres + SUPABASE_DB_URL: postgresql://postgres:postgres@supabase_db_policyengine-api-v2-alpha:5432/postgres LOGFIRE_TOKEN: ${LOGFIRE_TOKEN} DEBUG: "false" API_PORT: ${API_PORT:-8000} @@ -19,7 +19,7 @@ services: - ./src:/app/src - ./docs/out:/app/docs/out networks: - - supabase_network_policyengine-api-v2 + - supabase_network_policyengine-api-v2-alpha healthcheck: test: ["CMD", "python", "-c", "import httpx; exit(0 if httpx.get('http://localhost:${API_PORT:-8000}/health', timeout=2).status_code == 200 else 1)"] interval: 5s @@ -31,16 +31,16 @@ services: build: . command: pytest tests/ -v environment: - SUPABASE_URL: http://supabase_kong_policyengine-api-v2:8000 + SUPABASE_URL: http://supabase_kong_policyengine-api-v2-alpha:8000 SUPABASE_KEY: ${SUPABASE_KEY} SUPABASE_SERVICE_KEY: ${SUPABASE_SERVICE_KEY} - SUPABASE_DB_URL: postgresql://postgres:postgres@supabase_db_policyengine-api-v2:5432/postgres + SUPABASE_DB_URL: postgresql://postgres:postgres@supabase_db_policyengine-api-v2-alpha:5432/postgres LOGFIRE_TOKEN: ${LOGFIRE_TOKEN} volumes: - ./src:/app/src - ./tests:/app/tests networks: - - supabase_network_policyengine-api-v2 + - supabase_network_policyengine-api-v2-alpha depends_on: api: condition: service_healthy @@ -48,5 +48,5 @@ services: - test networks: - supabase_network_policyengine-api-v2: + supabase_network_policyengine-api-v2-alpha: external: true diff --git a/scripts/seed_common.py b/scripts/seed_common.py index f6d7ab6..db248a6 100644 --- a/scripts/seed_common.py +++ b/scripts/seed_common.py @@ -7,6 +7,7 @@ import sys import warnings from datetime import datetime, timezone +from enum import Enum from pathlib import Path from uuid import uuid4 @@ -152,6 +153,16 @@ def seed_model(model_version, session, lite: bool = False): total=len(model_version.variables), ) for var in model_version.variables: + # Serialize default_value for JSON storage + default_val = var.default_value + if var.value_type is Enum: + # Enum variables: extract the member name (e.g., "SINGLE") + # NOTE: This may need to change when we determine how to properly + # add possible_values (the list of enum members) into the database. + default_val = default_val.name + elif hasattr(default_val, "isoformat"): # datetime.date + default_val = default_val.isoformat() + var_rows.append( { "id": uuid4(), @@ -162,6 +173,7 @@ def seed_model(model_version, session, lite: bool = False): if hasattr(var.data_type, "__name__") else str(var.data_type), "possible_values": None, + "default_value": json.dumps(default_val), "tax_benefit_model_version_id": db_version.id, "created_at": datetime.now(timezone.utc), } @@ -179,6 +191,7 @@ def seed_model(model_version, session, lite: bool = False): "description", "data_type", "possible_values", + "default_value", "tax_benefit_model_version_id", "created_at", ], diff --git a/src/policyengine_api/models/user_household_association.py b/src/policyengine_api/models/user_household_association.py index 208279a..9a961cc 100644 --- a/src/policyengine_api/models/user_household_association.py +++ b/src/policyengine_api/models/user_household_association.py @@ -9,7 +9,8 @@ class UserHouseholdAssociationBase(SQLModel): """Base association fields.""" - user_id: UUID = Field(foreign_key="users.id", index=True) + # user_id is a client-generated UUID stored in localStorage, not a foreign key + user_id: UUID = Field(index=True) household_id: UUID = Field(foreign_key="households.id", index=True) country_id: str label: str | None = None diff --git a/src/policyengine_api/models/variable.py b/src/policyengine_api/models/variable.py index f163577..16c83b0 100644 --- a/src/policyengine_api/models/variable.py +++ b/src/policyengine_api/models/variable.py @@ -18,6 +18,9 @@ class VariableBase(SQLModel): possible_values: str | None = Field( default=None, sa_column=Column(JSON) ) # Store as JSON list + default_value: str | None = Field( + default=None, sa_column=Column(JSON) + ) # Store as JSON (handles int, float, bool, str, etc.) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id" ) From 013304f2047d50c94f60d1f328d34c94ee1a9e39 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 6 Feb 2026 21:47:51 +0300 Subject: [PATCH 11/87] refactor: Simplify seed scripts to use policyengine.py for default_value serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove Enum/date serialization from seed_common.py since policyengine.py now pre-serializes default_value for JSON compatibility - Change default_value type from `str | None` to `Any` in Variable model since it stores JSON values (bool, int, float, str) Depends on policyengine.py feat/add-variable-default-value branch 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/seed_common.py | 22 ++++++++++------------ src/policyengine_api/models/variable.py | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/seed_common.py b/scripts/seed_common.py index db248a6..49797cb 100644 --- a/scripts/seed_common.py +++ b/scripts/seed_common.py @@ -7,7 +7,6 @@ import sys import warnings from datetime import datetime, timezone -from enum import Enum from pathlib import Path from uuid import uuid4 @@ -81,6 +80,11 @@ def bulk_insert(session, table: str, columns: list[str], rows: list[dict]): def seed_model(model_version, session, lite: bool = False): """Seed a tax-benefit model with its variables and parameters. + Args: + model_version: The policyengine package model version + session: Database session + lite: If True, skip state-level parameters + Returns the TaxBenefitModelVersion that was created or found. """ from policyengine_api.models import ( @@ -153,16 +157,10 @@ def seed_model(model_version, session, lite: bool = False): total=len(model_version.variables), ) for var in model_version.variables: - # Serialize default_value for JSON storage - default_val = var.default_value - if var.value_type is Enum: - # Enum variables: extract the member name (e.g., "SINGLE") - # NOTE: This may need to change when we determine how to properly - # add possible_values (the list of enum members) into the database. - default_val = default_val.name - elif hasattr(default_val, "isoformat"): # datetime.date - default_val = default_val.isoformat() - + # default_value is pre-serialized by policyengine.py: + # - Enum values are converted to their name (e.g., "SINGLE") + # - datetime.date values are converted to ISO format + # - Primitives (bool, int, float, str) are kept as-is var_rows.append( { "id": uuid4(), @@ -173,7 +171,7 @@ def seed_model(model_version, session, lite: bool = False): if hasattr(var.data_type, "__name__") else str(var.data_type), "possible_values": None, - "default_value": json.dumps(default_val), + "default_value": json.dumps(var.default_value), "tax_benefit_model_version_id": db_version.id, "created_at": datetime.now(timezone.utc), } diff --git a/src/policyengine_api/models/variable.py b/src/policyengine_api/models/variable.py index 16c83b0..eeebddc 100644 --- a/src/policyengine_api/models/variable.py +++ b/src/policyengine_api/models/variable.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from uuid import UUID, uuid4 from sqlmodel import JSON, Column, Field, Relationship, SQLModel @@ -18,7 +18,7 @@ class VariableBase(SQLModel): possible_values: str | None = Field( default=None, sa_column=Column(JSON) ) # Store as JSON list - default_value: str | None = Field( + default_value: Any = Field( default=None, sa_column=Column(JSON) ) # Store as JSON (handles int, float, bool, str, etc.) tax_benefit_model_version_id: UUID = Field( From a97f4736947955099c63615b66a2c9e1cf7940ca Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Sat, 7 Feb 2026 02:07:23 +0300 Subject: [PATCH 12/87] fix: FINALLY use the ACTUAL Alembic script to generate migrations --- .../versions/20260204_0001_initial_schema.py | 537 ------------------ .../20260204_0002_add_household_support.py | 178 ------ ...60204_0003_add_parameter_values_indexes.py | 52 -- ...0260206_0004_add_variable_default_value.py | 34 -- .../20260207_36f9d434e95b_initial_schema.py | 321 +++++++++++ ...0207_f419b5f4acba_add_household_support.py | 81 +++ 6 files changed, 402 insertions(+), 801 deletions(-) delete mode 100644 alembic/versions/20260204_0001_initial_schema.py delete mode 100644 alembic/versions/20260204_0002_add_household_support.py delete mode 100644 alembic/versions/20260204_0003_add_parameter_values_indexes.py delete mode 100644 alembic/versions/20260206_0004_add_variable_default_value.py create mode 100644 alembic/versions/20260207_36f9d434e95b_initial_schema.py create mode 100644 alembic/versions/20260207_f419b5f4acba_add_household_support.py diff --git a/alembic/versions/20260204_0001_initial_schema.py b/alembic/versions/20260204_0001_initial_schema.py deleted file mode 100644 index 273124a..0000000 --- a/alembic/versions/20260204_0001_initial_schema.py +++ /dev/null @@ -1,537 +0,0 @@ -"""Initial schema (main branch state) - -Revision ID: 0001_initial -Revises: -Create Date: 2026-02-04 - -This migration creates all base tables for the PolicyEngine API as they -exist on the main branch, BEFORE the household CRUD changes. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0001_initial" -down_revision: Union[str, Sequence[str], None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Create all tables as they exist on main branch.""" - # ======================================================================== - # TIER 1: Tables with no foreign key dependencies - # ======================================================================== - - # Tax benefit models (e.g., "uk", "us") - op.create_table( - "tax_benefit_models", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # Users - op.create_table( - "users", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("first_name", sa.String(), nullable=False), - sa.Column("last_name", sa.String(), nullable=False), - sa.Column("email", sa.String(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("email"), - ) - op.create_index("ix_users_email", "users", ["email"]) - - # Policies (reform definitions) - op.create_table( - "policies", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # Dynamics (behavioral response definitions) - op.create_table( - "dynamics", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # ======================================================================== - # TIER 2: Tables depending on tier 1 - # ======================================================================== - - # Tax benefit model versions - op.create_table( - "tax_benefit_model_versions", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("model_id", sa.Uuid(), nullable=False), - sa.Column("version", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["model_id"], ["tax_benefit_models.id"]), - ) - - # Datasets (h5 files in storage) - op.create_table( - "datasets", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("filepath", sa.String(), nullable=False), - sa.Column("year", sa.Integer(), nullable=False), - sa.Column("is_output_dataset", sa.Boolean(), nullable=False, default=False), - sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), - ) - - # ======================================================================== - # TIER 3: Tables depending on tier 2 - # ======================================================================== - - # Parameters (tax-benefit system parameters) - op.create_table( - "parameters", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("label", sa.String(), nullable=True), - sa.Column("description", sa.String(), nullable=True), - sa.Column("data_type", sa.String(), nullable=True), - sa.Column("unit", sa.String(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - ) - - # Variables (tax-benefit system variables) - op.create_table( - "variables", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("data_type", sa.String(), nullable=True), - sa.Column("possible_values", sa.JSON(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - ) - - # Dataset versions - op.create_table( - "dataset_versions", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=False), - sa.Column("dataset_id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), - sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), - ) - - # ======================================================================== - # TIER 4: Tables depending on tier 3 - # ======================================================================== - - # Parameter values (policy/dynamic parameter modifications) - op.create_table( - "parameter_values", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("parameter_id", sa.Uuid(), nullable=False), - sa.Column("value_json", sa.JSON(), nullable=True), - sa.Column("start_date", sa.DateTime(timezone=True), nullable=False), - sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["parameter_id"], ["parameters.id"]), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - ) - - # Simulations (economy calculations) - NOTE: No household support yet - op.create_table( - "simulations", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("dataset_id", sa.Uuid(), nullable=False), # Required in main - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column("output_dataset_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - sa.ForeignKeyConstraint(["output_dataset_id"], ["datasets.id"]), - ) - - # Household jobs (async household calculations) - legacy approach - op.create_table( - "household_jobs", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_name", sa.String(), nullable=False), - sa.Column("request_data", sa.JSON(), nullable=False), - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.JSON(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - ) - - # ======================================================================== - # TIER 5: Tables depending on simulations - # ======================================================================== - - # Reports (analysis reports) - NOTE: No report_type yet - op.create_table( - "reports", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("label", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("markdown", sa.Text(), nullable=True), - sa.Column("parent_report_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=True), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["parent_report_id"], ["reports.id"]), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - ) - - # Aggregates (single-simulation aggregate outputs) - op.create_table( - "aggregates", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("variable", sa.String(), nullable=False), - sa.Column("aggregate_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("filter_config", sa.JSON(), nullable=False, default={}), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Change aggregates (baseline vs reform comparison) - op.create_table( - "change_aggregates", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("variable", sa.String(), nullable=False), - sa.Column("aggregate_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("filter_config", sa.JSON(), nullable=False, default={}), - sa.Column("change_geq", sa.Float(), nullable=True), - sa.Column("change_leq", sa.Float(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Decile impacts - op.create_table( - "decile_impacts", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("income_variable", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("decile", sa.Integer(), nullable=False), - sa.Column("quantiles", sa.Integer(), nullable=False, default=10), - sa.Column("baseline_mean", sa.Float(), nullable=True), - sa.Column("reform_mean", sa.Float(), nullable=True), - sa.Column("absolute_change", sa.Float(), nullable=True), - sa.Column("relative_change", sa.Float(), nullable=True), - sa.Column("count_better_off", sa.Float(), nullable=True), - sa.Column("count_worse_off", sa.Float(), nullable=True), - sa.Column("count_no_change", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Program statistics - op.create_table( - "program_statistics", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("program_name", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False), - sa.Column("is_tax", sa.Boolean(), nullable=False, default=False), - sa.Column("baseline_total", sa.Float(), nullable=True), - sa.Column("reform_total", sa.Float(), nullable=True), - sa.Column("change", sa.Float(), nullable=True), - sa.Column("baseline_count", sa.Float(), nullable=True), - sa.Column("reform_count", sa.Float(), nullable=True), - sa.Column("winners", sa.Float(), nullable=True), - sa.Column("losers", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Poverty - op.create_table( - "poverty", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("poverty_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False, default="person"), - sa.Column("filter_variable", sa.String(), nullable=True), - sa.Column("headcount", sa.Float(), nullable=True), - sa.Column("total_population", sa.Float(), nullable=True), - sa.Column("rate", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["simulation_id"], ["simulations.id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), - ) - op.create_index("idx_poverty_simulation_id", "poverty", ["simulation_id"]) - op.create_index("idx_poverty_report_id", "poverty", ["report_id"]) - - # Inequality - op.create_table( - "inequality", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("income_variable", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False, default="household"), - sa.Column("gini", sa.Float(), nullable=True), - sa.Column("top_10_share", sa.Float(), nullable=True), - sa.Column("top_1_share", sa.Float(), nullable=True), - sa.Column("bottom_50_share", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["simulation_id"], ["simulations.id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), - ) - op.create_index("idx_inequality_simulation_id", "inequality", ["simulation_id"]) - op.create_index("idx_inequality_report_id", "inequality", ["report_id"]) - - -def downgrade() -> None: - """Drop all tables in reverse order.""" - # Tier 5 - op.drop_index("idx_inequality_report_id", "inequality") - op.drop_index("idx_inequality_simulation_id", "inequality") - op.drop_table("inequality") - op.drop_index("idx_poverty_report_id", "poverty") - op.drop_index("idx_poverty_simulation_id", "poverty") - op.drop_table("poverty") - op.drop_table("program_statistics") - op.drop_table("decile_impacts") - op.drop_table("change_aggregates") - op.drop_table("aggregates") - op.drop_table("reports") - - # Tier 4 - op.drop_table("household_jobs") - op.drop_table("simulations") - op.drop_table("parameter_values") - - # Tier 3 - op.drop_table("dataset_versions") - op.drop_table("variables") - op.drop_table("parameters") - - # Tier 2 - op.drop_table("datasets") - op.drop_table("tax_benefit_model_versions") - - # Tier 1 - op.drop_table("dynamics") - op.drop_table("policies") - op.drop_index("ix_users_email", "users") - op.drop_table("users") - op.drop_table("tax_benefit_models") diff --git a/alembic/versions/20260204_0002_add_household_support.py b/alembic/versions/20260204_0002_add_household_support.py deleted file mode 100644 index 186db37..0000000 --- a/alembic/versions/20260204_0002_add_household_support.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Add household CRUD and impact analysis support - -Revision ID: 0002_household -Revises: 0001_initial -Create Date: 2026-02-04 - -This migration adds support for: -- Storing household definitions (households table) -- User-household associations for saved households -- Household-based simulations (adds household_id to simulations) -- Household impact reports (adds report_type to reports) -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0002_household" -down_revision: Union[str, Sequence[str], None] = "0001_initial" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Add household support.""" - # ======================================================================== - # NEW TABLES - # ======================================================================== - - # Households (stored household definitions) - op.create_table( - "households", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_name", sa.String(), nullable=False), - sa.Column("year", sa.Integer(), nullable=False), - sa.Column("label", sa.String(), nullable=True), - sa.Column("household_data", sa.JSON(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - "idx_households_model_name", "households", ["tax_benefit_model_name"] - ) - op.create_index("idx_households_year", "households", ["year"]) - - # User-household associations (many-to-many for saved households) - # Note: user_id is a client-generated UUID stored in localStorage, not a foreign key - op.create_table( - "user_household_associations", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=False), - sa.Column("household_id", sa.Uuid(), nullable=False), - sa.Column("country_id", sa.String(), nullable=False), - sa.Column("label", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["household_id"], ["households.id"], ondelete="CASCADE"), - sa.UniqueConstraint("user_id", "household_id"), - ) - op.create_index( - "idx_user_household_user", "user_household_associations", ["user_id"] - ) - op.create_index( - "idx_user_household_household", "user_household_associations", ["household_id"] - ) - - # ======================================================================== - # MODIFY SIMULATIONS TABLE - # ======================================================================== - - # Add simulation_type column (economy vs household) - op.add_column( - "simulations", - sa.Column( - "simulation_type", - sa.String(), - nullable=False, - server_default="economy", - ), - ) - - # Add household_id column (for household simulations) - op.add_column( - "simulations", - sa.Column("household_id", sa.Uuid(), nullable=True), - ) - op.create_foreign_key( - "fk_simulations_household_id", - "simulations", - "households", - ["household_id"], - ["id"], - ) - - # Add household_result column (stores household calculation results) - op.add_column( - "simulations", - sa.Column("household_result", sa.JSON(), nullable=True), - ) - - # Make dataset_id nullable (household simulations don't need a dataset) - op.alter_column( - "simulations", - "dataset_id", - existing_type=sa.Uuid(), - nullable=True, - ) - - # ======================================================================== - # MODIFY REPORTS TABLE - # ======================================================================== - - # Add report_type column (economy_comparison, household_impact, etc.) - op.add_column( - "reports", - sa.Column("report_type", sa.String(), nullable=True), - ) - - -def downgrade() -> None: - """Remove household support.""" - # ======================================================================== - # REVERT REPORTS TABLE - # ======================================================================== - op.drop_column("reports", "report_type") - - # ======================================================================== - # REVERT SIMULATIONS TABLE - # ======================================================================== - - # Make dataset_id required again - op.alter_column( - "simulations", - "dataset_id", - existing_type=sa.Uuid(), - nullable=False, - ) - - # Remove household columns - op.drop_column("simulations", "household_result") - op.drop_constraint("fk_simulations_household_id", "simulations", type_="foreignkey") - op.drop_column("simulations", "household_id") - op.drop_column("simulations", "simulation_type") - - # ======================================================================== - # DROP NEW TABLES - # ======================================================================== - op.drop_index("idx_user_household_household", "user_household_associations") - op.drop_index("idx_user_household_user", "user_household_associations") - op.drop_table("user_household_associations") - - op.drop_index("idx_households_year", "households") - op.drop_index("idx_households_model_name", "households") - op.drop_table("households") diff --git a/alembic/versions/20260204_0003_add_parameter_values_indexes.py b/alembic/versions/20260204_0003_add_parameter_values_indexes.py deleted file mode 100644 index 53518cf..0000000 --- a/alembic/versions/20260204_0003_add_parameter_values_indexes.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Add parameter_values indexes - -Revision ID: 0003_param_idx -Revises: 0002_household -Create Date: 2026-02-04 02:20:00.000000 - -This migration adds performance indexes to the parameter_values table -for optimizing common query patterns. -""" - -from typing import Sequence, Union - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0003_param_idx" -down_revision: Union[str, Sequence[str], None] = "0002_household" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Add performance indexes to parameter_values.""" - # Composite index for the most common query pattern (filtering by both) - op.create_index( - "idx_parameter_values_parameter_policy", - "parameter_values", - ["parameter_id", "policy_id"], - ) - - # Single index on policy_id for filtering by policy alone - op.create_index( - "idx_parameter_values_policy", - "parameter_values", - ["policy_id"], - ) - - # Partial index for baseline values (policy_id IS NULL) - # This optimizes the common "get current law values" query - op.create_index( - "idx_parameter_values_baseline", - "parameter_values", - ["parameter_id"], - postgresql_where="policy_id IS NULL", - ) - - -def downgrade() -> None: - """Remove parameter_values indexes.""" - op.drop_index("idx_parameter_values_baseline", "parameter_values") - op.drop_index("idx_parameter_values_policy", "parameter_values") - op.drop_index("idx_parameter_values_parameter_policy", "parameter_values") diff --git a/alembic/versions/20260206_0004_add_variable_default_value.py b/alembic/versions/20260206_0004_add_variable_default_value.py deleted file mode 100644 index 2471491..0000000 --- a/alembic/versions/20260206_0004_add_variable_default_value.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Add default_value to variables - -Revision ID: 0004_var_default -Revises: 0003_param_idx -Create Date: 2026-02-06 03:30:00.000000 - -This migration adds a default_value column to the variables table. -The default_value is stored as JSON to handle different types (int, float, bool, str, etc.). -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects.postgresql import JSON - -# revision identifiers, used by Alembic. -revision: str = "0004_var_default" -down_revision: Union[str, Sequence[str], None] = "0003_param_idx" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Add default_value column to variables table.""" - op.add_column( - "variables", - sa.Column("default_value", JSON, nullable=True), - ) - - -def downgrade() -> None: - """Remove default_value column from variables table.""" - op.drop_column("variables", "default_value") diff --git a/alembic/versions/20260207_36f9d434e95b_initial_schema.py b/alembic/versions/20260207_36f9d434e95b_initial_schema.py new file mode 100644 index 0000000..0dce2e7 --- /dev/null +++ b/alembic/versions/20260207_36f9d434e95b_initial_schema.py @@ -0,0 +1,321 @@ +"""initial schema + +Revision ID: 36f9d434e95b +Revises: +Create Date: 2026-02-07 01:52:16.497121 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes + + +# revision identifiers, used by Alembic. +revision: str = '36f9d434e95b' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('dynamics', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('policies', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('tax_benefit_models', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('first_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('last_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('email', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True) + op.create_table('datasets', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('filepath', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('year', sa.Integer(), nullable=False), + sa.Column('is_output_dataset', sa.Boolean(), nullable=False), + sa.Column('tax_benefit_model_id', sa.Uuid(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['tax_benefit_model_id'], ['tax_benefit_models.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('household_jobs', + sa.Column('tax_benefit_model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('request_data', sa.JSON(), nullable=True), + sa.Column('policy_id', sa.Uuid(), nullable=True), + sa.Column('dynamic_id', sa.Uuid(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='householdjobstatus'), nullable=False), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('result', sa.JSON(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['dynamic_id'], ['dynamics.id'], ), + sa.ForeignKeyConstraint(['policy_id'], ['policies.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('tax_benefit_model_versions', + sa.Column('model_id', sa.Uuid(), nullable=False), + sa.Column('version', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['model_id'], ['tax_benefit_models.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('dataset_versions', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('dataset_id', sa.Uuid(), nullable=False), + sa.Column('tax_benefit_model_id', sa.Uuid(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['dataset_id'], ['datasets.id'], ), + sa.ForeignKeyConstraint(['tax_benefit_model_id'], ['tax_benefit_models.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameters', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('label', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('data_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('unit', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('tax_benefit_model_version_id', sa.Uuid(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['tax_benefit_model_version_id'], ['tax_benefit_model_versions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('simulations', + sa.Column('dataset_id', sa.Uuid(), nullable=False), + sa.Column('policy_id', sa.Uuid(), nullable=True), + sa.Column('dynamic_id', sa.Uuid(), nullable=True), + sa.Column('tax_benefit_model_version_id', sa.Uuid(), nullable=False), + sa.Column('output_dataset_id', sa.Uuid(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='simulationstatus'), nullable=False), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['datasets.id'], ), + sa.ForeignKeyConstraint(['dynamic_id'], ['dynamics.id'], ), + sa.ForeignKeyConstraint(['output_dataset_id'], ['datasets.id'], ), + sa.ForeignKeyConstraint(['policy_id'], ['policies.id'], ), + sa.ForeignKeyConstraint(['tax_benefit_model_version_id'], ['tax_benefit_model_versions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('variables', + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('data_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('possible_values', sa.JSON(), nullable=True), + sa.Column('tax_benefit_model_version_id', sa.Uuid(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['tax_benefit_model_version_id'], ['tax_benefit_model_versions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameter_values', + sa.Column('parameter_id', sa.Uuid(), nullable=False), + sa.Column('value_json', sa.JSON(), nullable=True), + sa.Column('start_date', sa.DateTime(), nullable=False), + sa.Column('end_date', sa.DateTime(), nullable=True), + sa.Column('policy_id', sa.Uuid(), nullable=True), + sa.Column('dynamic_id', sa.Uuid(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['dynamic_id'], ['dynamics.id'], ), + sa.ForeignKeyConstraint(['parameter_id'], ['parameters.id'], ), + sa.ForeignKeyConstraint(['policy_id'], ['policies.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('reports', + sa.Column('label', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('user_id', sa.Uuid(), nullable=True), + sa.Column('markdown', sa.Text(), nullable=True), + sa.Column('parent_report_id', sa.Uuid(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='reportstatus'), nullable=False), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('baseline_simulation_id', sa.Uuid(), nullable=True), + sa.Column('reform_simulation_id', sa.Uuid(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['baseline_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['parent_report_id'], ['reports.id'], ), + sa.ForeignKeyConstraint(['reform_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('aggregates', + sa.Column('simulation_id', sa.Uuid(), nullable=False), + sa.Column('user_id', sa.Uuid(), nullable=True), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('variable', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('aggregate_type', sa.Enum('SUM', 'MEAN', 'COUNT', name='aggregatetype'), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('filter_config', sa.JSON(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='aggregatestatus'), nullable=False), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('result', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.ForeignKeyConstraint(['simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('change_aggregates', + sa.Column('baseline_simulation_id', sa.Uuid(), nullable=False), + sa.Column('reform_simulation_id', sa.Uuid(), nullable=False), + sa.Column('user_id', sa.Uuid(), nullable=True), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('variable', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('aggregate_type', sa.Enum('SUM', 'MEAN', 'COUNT', name='changeaggregatetype'), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('filter_config', sa.JSON(), nullable=True), + sa.Column('change_geq', sa.Float(), nullable=True), + sa.Column('change_leq', sa.Float(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='changeaggregatestatus'), nullable=False), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('result', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['baseline_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['reform_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('decile_impacts', + sa.Column('baseline_simulation_id', sa.Uuid(), nullable=False), + sa.Column('reform_simulation_id', sa.Uuid(), nullable=False), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('income_variable', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('decile', sa.Integer(), nullable=False), + sa.Column('quantiles', sa.Integer(), nullable=False), + sa.Column('baseline_mean', sa.Float(), nullable=True), + sa.Column('reform_mean', sa.Float(), nullable=True), + sa.Column('absolute_change', sa.Float(), nullable=True), + sa.Column('relative_change', sa.Float(), nullable=True), + sa.Column('count_better_off', sa.Float(), nullable=True), + sa.Column('count_worse_off', sa.Float(), nullable=True), + sa.Column('count_no_change', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['baseline_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['reform_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('inequality', + sa.Column('simulation_id', sa.Uuid(), nullable=False), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('income_variable', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('gini', sa.Float(), nullable=True), + sa.Column('top_10_share', sa.Float(), nullable=True), + sa.Column('top_1_share', sa.Float(), nullable=True), + sa.Column('bottom_50_share', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.ForeignKeyConstraint(['simulation_id'], ['simulations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('poverty', + sa.Column('simulation_id', sa.Uuid(), nullable=False), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('poverty_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('filter_variable', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('headcount', sa.Float(), nullable=True), + sa.Column('total_population', sa.Float(), nullable=True), + sa.Column('rate', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.ForeignKeyConstraint(['simulation_id'], ['simulations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('program_statistics', + sa.Column('baseline_simulation_id', sa.Uuid(), nullable=False), + sa.Column('reform_simulation_id', sa.Uuid(), nullable=False), + sa.Column('report_id', sa.Uuid(), nullable=True), + sa.Column('program_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('entity', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('is_tax', sa.Boolean(), nullable=False), + sa.Column('baseline_total', sa.Float(), nullable=True), + sa.Column('reform_total', sa.Float(), nullable=True), + sa.Column('change', sa.Float(), nullable=True), + sa.Column('baseline_count', sa.Float(), nullable=True), + sa.Column('reform_count', sa.Float(), nullable=True), + sa.Column('winners', sa.Float(), nullable=True), + sa.Column('losers', sa.Float(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['baseline_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['reform_simulation_id'], ['simulations.id'], ), + sa.ForeignKeyConstraint(['report_id'], ['reports.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('program_statistics') + op.drop_table('poverty') + op.drop_table('inequality') + op.drop_table('decile_impacts') + op.drop_table('change_aggregates') + op.drop_table('aggregates') + op.drop_table('reports') + op.drop_table('parameter_values') + op.drop_table('variables') + op.drop_table('simulations') + op.drop_table('parameters') + op.drop_table('dataset_versions') + op.drop_table('tax_benefit_model_versions') + op.drop_table('household_jobs') + op.drop_table('datasets') + op.drop_index(op.f('ix_users_email'), table_name='users') + op.drop_table('users') + op.drop_table('tax_benefit_models') + op.drop_table('policies') + op.drop_table('dynamics') + # ### end Alembic commands ### diff --git a/alembic/versions/20260207_f419b5f4acba_add_household_support.py b/alembic/versions/20260207_f419b5f4acba_add_household_support.py new file mode 100644 index 0000000..cef65f3 --- /dev/null +++ b/alembic/versions/20260207_f419b5f4acba_add_household_support.py @@ -0,0 +1,81 @@ +"""add household support + +Revision ID: f419b5f4acba +Revises: 36f9d434e95b +Create Date: 2026-02-07 01:56:31.064511 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'f419b5f4acba' +down_revision: Union[str, Sequence[str], None] = '36f9d434e95b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('households', + sa.Column('tax_benefit_model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('year', sa.Integer(), nullable=False), + sa.Column('label', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('household_data', sa.JSON(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_household_associations', + sa.Column('user_id', sa.Uuid(), nullable=False), + sa.Column('household_id', sa.Uuid(), nullable=False), + sa.Column('country_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('label', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['household_id'], ['households.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_user_household_associations_household_id'), 'user_household_associations', ['household_id'], unique=False) + op.create_index(op.f('ix_user_household_associations_user_id'), 'user_household_associations', ['user_id'], unique=False) + op.add_column('reports', sa.Column('report_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + # Create enum type first + simulationtype = postgresql.ENUM('HOUSEHOLD', 'ECONOMY', name='simulationtype', create_type=False) + simulationtype.create(op.get_bind(), checkfirst=True) + op.add_column('simulations', sa.Column('simulation_type', sa.Enum('HOUSEHOLD', 'ECONOMY', name='simulationtype', create_type=False), nullable=False)) + op.add_column('simulations', sa.Column('household_id', sa.Uuid(), nullable=True)) + op.add_column('simulations', sa.Column('household_result', postgresql.JSON(astext_type=sa.Text()), nullable=True)) + op.alter_column('simulations', 'dataset_id', + existing_type=sa.UUID(), + nullable=True) + op.create_foreign_key(None, 'simulations', 'households', ['household_id'], ['id']) + op.add_column('variables', sa.Column('default_value', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('variables', 'default_value') + op.drop_constraint(None, 'simulations', type_='foreignkey') + op.alter_column('simulations', 'dataset_id', + existing_type=sa.UUID(), + nullable=False) + op.drop_column('simulations', 'household_result') + op.drop_column('simulations', 'household_id') + op.drop_column('simulations', 'simulation_type') + # Drop enum type + postgresql.ENUM('HOUSEHOLD', 'ECONOMY', name='simulationtype').drop(op.get_bind(), checkfirst=True) + op.drop_column('reports', 'report_type') + op.drop_index(op.f('ix_user_household_associations_user_id'), table_name='user_household_associations') + op.drop_index(op.f('ix_user_household_associations_household_id'), table_name='user_household_associations') + op.drop_table('user_household_associations') + op.drop_table('households') + # ### end Alembic commands ### From bdebc9e0f169e123fdfc6bb04d9898ff528f52d6 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Sat, 7 Feb 2026 03:42:40 +0300 Subject: [PATCH 13/87] test: Add Variable model tests for default_value field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add tests for Variable with int, float, bool, and string default values - Add test for null default_value handling - Add test for Household model creation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- tests/test_models.py | 89 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 0f84140..e3a83d9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -6,9 +6,11 @@ AggregateOutput, AggregateType, Dataset, + Household, Policy, Simulation, SimulationStatus, + Variable, ) @@ -66,3 +68,90 @@ def test_aggregate_output_creation(): assert output.simulation_id == simulation_id assert output.aggregate_type == AggregateType.SUM assert output.result is None + + +def test_variable_creation_with_default_value(): + """Test variable model creation with default_value field.""" + model_version_id = uuid4() + variable = Variable( + name="age", + entity="person", + description="Age of the person", + data_type="int", + default_value=40, + tax_benefit_model_version_id=model_version_id, + ) + assert variable.name == "age" + assert variable.entity == "person" + assert variable.data_type == "int" + assert variable.default_value == 40 + assert variable.id is not None + + +def test_variable_with_float_default_value(): + """Test variable model with float default value.""" + model_version_id = uuid4() + variable = Variable( + name="employment_income", + entity="person", + data_type="float", + default_value=0.0, + tax_benefit_model_version_id=model_version_id, + ) + assert variable.default_value == 0.0 + + +def test_variable_with_bool_default_value(): + """Test variable model with boolean default value.""" + model_version_id = uuid4() + variable = Variable( + name="is_disabled", + entity="person", + data_type="bool", + default_value=False, + tax_benefit_model_version_id=model_version_id, + ) + assert variable.default_value is False + + +def test_variable_with_string_default_value(): + """Test variable model with string default value (enum).""" + model_version_id = uuid4() + variable = Variable( + name="state_name", + entity="household", + data_type="Enum", + default_value="CA", + possible_values=["CA", "NY", "TX"], + tax_benefit_model_version_id=model_version_id, + ) + assert variable.default_value == "CA" + assert variable.possible_values == ["CA", "NY", "TX"] + + +def test_variable_with_null_default_value(): + """Test variable model with null default value.""" + model_version_id = uuid4() + variable = Variable( + name="optional_field", + entity="person", + data_type="str", + default_value=None, + tax_benefit_model_version_id=model_version_id, + ) + assert variable.default_value is None + + +def test_household_creation(): + """Test household model creation.""" + household = Household( + tax_benefit_model_name="policyengine_us", + year=2024, + label="Test household", + household_data={"people": [{"age": 30}], "household": {}}, + ) + assert household.household_data == {"people": [{"age": 30}], "household": {}} + assert household.label == "Test household" + assert household.tax_benefit_model_name == "policyengine_us" + assert household.year == 2024 + assert household.id is not None From b15c26842cab6b984e6511a43dd8a7644ee19940 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Sat, 7 Feb 2026 03:53:10 +0300 Subject: [PATCH 14/87] fix: Convert string report_id to UUID in _run_local_household_impact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit session.get(Report, report_id) expects a UUID, but report_id was passed as a string. This caused 'str' object has no attribute 'hex' errors. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/policyengine_api/api/household_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/policyengine_api/api/household_analysis.py b/src/policyengine_api/api/household_analysis.py index 5f36fda..d321be4 100644 --- a/src/policyengine_api/api/household_analysis.py +++ b/src/policyengine_api/api/household_analysis.py @@ -376,7 +376,7 @@ def _run_local_household_impact(report_id: str, session: Session) -> None: locally (agent_use_modal=False). This mirrors the economic impact behavior. True async execution requires Modal. """ - report = session.get(Report, report_id) + report = session.get(Report, UUID(report_id)) if not report: return From 298540b4372127bcba043ae13bfc70232c640053 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 11 Feb 2026 21:47:33 +0100 Subject: [PATCH 15/87] refactor: Use policyengine.py's Simulation class directly now that US reform bug is fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit removes the workaround code that was added to bypass policyengine.py's Simulation class. The workaround was needed because policyengine.py's US simulation applied reforms via p.update() after Microsimulation construction, which didn't work due to the US package's shared singleton TaxBenefitSystem. That bug has now been fixed in policyengine.py (issue #232), so we can use policyengine.py's Simulation class directly again. Changes: - Revert household.py to use policyengine.core.Simulation instead of manually building Microsimulation with reform dicts - Revert modal_app.py to use PESimulation instead of custom helper functions (_pe_policy_to_reform_dict, _merge_reform_dicts, _run_us_economy_simulation, _run_uk_economy_simulation) - Remove now-obsolete test files for the workaround functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/policyengine_api/api/household.py | 388 +++----- src/policyengine_api/modal_app.py | 1153 +---------------------- test_fixtures/fixtures_policy_reform.py | 282 ------ tests/test_policy_reform.py | 327 ------- 4 files changed, 186 insertions(+), 1964 deletions(-) delete mode 100644 test_fixtures/fixtures_policy_reform.py delete mode 100644 tests/test_policy_reform.py diff --git a/src/policyengine_api/api/household.py b/src/policyengine_api/api/household.py index adb6ac9..0e89b5e 100644 --- a/src/policyengine_api/api/household.py +++ b/src/policyengine_api/api/household.py @@ -294,16 +294,17 @@ def _calculate_household_uk( Supports multiple households via entity relational dataframes. If entity IDs are not provided, defaults to single household with all people in it. - - Uses policyengine-uk Microsimulation directly with reform dict to ensure - policy changes are applied correctly. """ - import numpy as np + import tempfile + from datetime import datetime + from pathlib import Path + import pandas as pd + from policyengine.core import Simulation + from microdf import MicroDataFrame from policyengine.tax_benefit_models.uk import uk_latest - from policyengine_core.simulations.simulation_builder import SimulationBuilder - from policyengine_uk import Microsimulation - from policyengine_uk.system import system + from policyengine.tax_benefit_models.uk.datasets import PolicyEngineUKDataset + from policyengine.tax_benefit_models.uk.datasets import UKYearData n_people = len(people) n_benunits = max(1, len(benunit)) @@ -349,88 +350,68 @@ def _calculate_household_uk( household_data[key] = [0.0] * n_households household_data[key][i] = value - # Convert policy_data to policyengine-uk reform dict format - # Format: {"param.name": {"YYYY-MM-DD": value}} - reform = None - if policy_data and policy_data.get("parameter_values"): - reform = {} - for pv in policy_data["parameter_values"]: - param_name = pv.get("parameter_name") - value = pv.get("value") - start_date = pv.get("start_date") - - if param_name and start_date: - # Parse ISO date string to get just the date part - if "T" in start_date: - date_str = start_date.split("T")[0] - else: - date_str = start_date - - if param_name not in reform: - reform[param_name] = {} - reform[param_name][date_str] = value - - # Create Microsimulation with reform applied at construction time - microsim = Microsimulation(reform=reform) - - # Build simulation from entity data using SimulationBuilder - person_df = pd.DataFrame(person_data) - - # Determine column naming convention - benunit_id_col = ( - "person_benunit_id" - if "person_benunit_id" in person_df.columns - else "benunit_id" - ) - household_id_col = ( - "person_household_id" - if "person_household_id" in person_df.columns - else "household_id" + # Create MicroDataFrames + person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") + benunit_df = MicroDataFrame(pd.DataFrame(benunit_data), weights="benunit_weight") + household_df = MicroDataFrame( + pd.DataFrame(household_data), weights="household_weight" ) - # Declare entities using SimulationBuilder - builder = SimulationBuilder() - builder.populations = system.instantiate_entities() + # Create temporary dataset + tmpdir = tempfile.mkdtemp() + filepath = str(Path(tmpdir) / "household_calc.h5") + + dataset = PolicyEngineUKDataset( + name="Household calculation", + description="Household(s) for calculation", + filepath=filepath, + year=year, + data=UKYearData( + person=person_df, + benunit=benunit_df, + household=household_df, + ), + ) - builder.declare_person_entity("person", person_df["person_id"].values) - builder.declare_entity("benunit", np.unique(person_df[benunit_id_col].values)) - builder.declare_entity("household", np.unique(person_df[household_id_col].values)) + # Build policy if provided + policy = None + if policy_data: + from policyengine.core.policy import ParameterValue as PEParameterValue + from policyengine.core.policy import Policy as PEPolicy + + pe_param_values = [] + param_lookup = {p.name: p for p in uk_latest.parameters} + for pv in policy_data.get("parameter_values", []): + pe_param = param_lookup.get(pv["parameter_name"]) + if pe_param: + pe_pv = PEParameterValue( + parameter=pe_param, + value=pv["value"], + start_date=datetime.fromisoformat(pv["start_date"]) + if pv.get("start_date") + else None, + end_date=datetime.fromisoformat(pv["end_date"]) + if pv.get("end_date") + else None, + ) + pe_param_values.append(pe_pv) + policy = PEPolicy( + name=policy_data.get("name", ""), + description=policy_data.get("description", ""), + parameter_values=pe_param_values, + ) - # Join persons to group entities - builder.join_with_persons( - builder.populations["benunit"], - person_df[benunit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["household"], - person_df[household_id_col].values, - np.array(["member"] * len(person_df)), + # Run simulation + simulation = Simulation( + dataset=dataset, + tax_benefit_model_version=uk_latest, + policy=policy, ) + simulation.run() - # Build simulation from populations - microsim.build_from_populations(builder.populations) + # Extract outputs + output_data = simulation.output_dataset.data - # Set input variables for each entity - id_columns = { - "person_id", - "benunit_id", - "person_benunit_id", - "household_id", - "person_household_id", - } - - for entity_name, entity_df in [ - ("person", person_data), - ("benunit", benunit_data), - ("household", household_data), - ]: - df = pd.DataFrame(entity_df) - for column in df.columns: - if column not in id_columns and column in system.variables: - microsim.set_input(column, year, df[column].values) - - # Calculate output variables def safe_convert(value): try: return float(value) @@ -441,24 +422,21 @@ def safe_convert(value): for i in range(n_people): person_dict = {} for var in uk_latest.entity_variables["person"]: - val = microsim.calculate(var, period=year, map_to="person") - person_dict[var] = safe_convert(val.values[i]) + person_dict[var] = safe_convert(output_data.person[var].iloc[i]) person_outputs.append(person_dict) benunit_outputs = [] - for i in range(n_benunits): + for i in range(len(output_data.benunit)): benunit_dict = {} for var in uk_latest.entity_variables["benunit"]: - val = microsim.calculate(var, period=year, map_to="benunit") - benunit_dict[var] = safe_convert(val.values[i]) + benunit_dict[var] = safe_convert(output_data.benunit[var].iloc[i]) benunit_outputs.append(benunit_dict) household_outputs = [] - for i in range(n_households): + for i in range(len(output_data.household)): household_dict = {} for var in uk_latest.entity_variables["household"]: - val = microsim.calculate(var, period=year, map_to="household") - household_dict[var] = safe_convert(val.values[i]) + household_dict[var] = safe_convert(output_data.household[var].iloc[i]) household_outputs.append(household_dict) return { @@ -488,14 +466,7 @@ def _run_local_household_us( try: result = _calculate_household_us( - people, - marital_unit, - family, - spm_unit, - tax_unit, - household, - year, - policy_data, + people, marital_unit, family, spm_unit, tax_unit, household, year, policy_data ) # Update job with result @@ -535,16 +506,17 @@ def _calculate_household_us( Supports multiple households via entity relational dataframes. If entity IDs are not provided, defaults to single household with all people in it. - - Uses policyengine-us Microsimulation directly with reform dict to ensure - policy changes are applied correctly. """ - import numpy as np + import tempfile + from datetime import datetime + from pathlib import Path + import pandas as pd + from policyengine.core import Simulation + from microdf import MicroDataFrame from policyengine.tax_benefit_models.us import us_latest - from policyengine_core.simulations.simulation_builder import SimulationBuilder - from policyengine_us import Microsimulation - from policyengine_us.system import system + from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset + from policyengine.tax_benefit_models.us.datasets import USYearData n_people = len(people) n_households = max(1, len(household)) @@ -624,158 +596,108 @@ def _calculate_household_us( tax_unit_data[key] = [0.0] * n_tax_units tax_unit_data[key][i] = value - # Convert policy_data to policyengine-us reform dict format - # Format: {"param.name": {"YYYY-MM-DD": value}} - reform = None - if policy_data and policy_data.get("parameter_values"): - reform = {} - for pv in policy_data["parameter_values"]: - param_name = pv.get("parameter_name") - value = pv.get("value") - start_date = pv.get("start_date") - - if param_name and start_date: - # Parse ISO date string to get just the date part - if "T" in start_date: - date_str = start_date.split("T")[0] - else: - date_str = start_date - - if param_name not in reform: - reform[param_name] = {} - reform[param_name][date_str] = value - - # Create Microsimulation with reform applied at construction time - # This ensures the reform is properly integrated into the tax benefit system - microsim = Microsimulation(reform=reform) - - # Build simulation from entity data using SimulationBuilder - person_df = pd.DataFrame(person_data) - - # Determine column naming convention - household_id_col = ( - "person_household_id" - if "person_household_id" in person_df.columns - else "household_id" - ) - marital_unit_id_col = ( - "person_marital_unit_id" - if "person_marital_unit_id" in person_df.columns - else "marital_unit_id" - ) - family_id_col = ( - "person_family_id" if "person_family_id" in person_df.columns else "family_id" + # Create MicroDataFrames + person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") + household_df = MicroDataFrame( + pd.DataFrame(household_data), weights="household_weight" ) - spm_unit_id_col = ( - "person_spm_unit_id" - if "person_spm_unit_id" in person_df.columns - else "spm_unit_id" + marital_unit_df = MicroDataFrame( + pd.DataFrame(marital_unit_data), weights="marital_unit_weight" ) - tax_unit_id_col = ( - "person_tax_unit_id" - if "person_tax_unit_id" in person_df.columns - else "tax_unit_id" + family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") + spm_unit_df = MicroDataFrame(pd.DataFrame(spm_unit_data), weights="spm_unit_weight") + tax_unit_df = MicroDataFrame(pd.DataFrame(tax_unit_data), weights="tax_unit_weight") + + # Create temporary dataset + tmpdir = tempfile.mkdtemp() + filepath = str(Path(tmpdir) / "household_calc.h5") + + dataset = PolicyEngineUSDataset( + name="Household calculation", + description="Household(s) for calculation", + filepath=filepath, + year=year, + data=USYearData( + person=person_df, + household=household_df, + marital_unit=marital_unit_df, + family=family_df, + spm_unit=spm_unit_df, + tax_unit=tax_unit_df, + ), ) - # Declare entities using SimulationBuilder - builder = SimulationBuilder() - builder.populations = system.instantiate_entities() - - builder.declare_person_entity("person", person_df["person_id"].values) - builder.declare_entity("household", np.unique(person_df[household_id_col].values)) - builder.declare_entity("spm_unit", np.unique(person_df[spm_unit_id_col].values)) - builder.declare_entity("family", np.unique(person_df[family_id_col].values)) - builder.declare_entity("tax_unit", np.unique(person_df[tax_unit_id_col].values)) - builder.declare_entity( - "marital_unit", np.unique(person_df[marital_unit_id_col].values) - ) + # Build policy if provided + policy = None + if policy_data: + from policyengine.core.policy import ParameterValue as PEParameterValue + from policyengine.core.policy import Policy as PEPolicy + + pe_param_values = [] + param_lookup = {p.name: p for p in us_latest.parameters} + for pv in policy_data.get("parameter_values", []): + pe_param = param_lookup.get(pv["parameter_name"]) + if pe_param: + pe_pv = PEParameterValue( + parameter=pe_param, + value=pv["value"], + start_date=datetime.fromisoformat(pv["start_date"]) + if pv.get("start_date") + else None, + end_date=datetime.fromisoformat(pv["end_date"]) + if pv.get("end_date") + else None, + ) + pe_param_values.append(pe_pv) + policy = PEPolicy( + name=policy_data.get("name", ""), + description=policy_data.get("description", ""), + parameter_values=pe_param_values, + ) - # Join persons to group entities - builder.join_with_persons( - builder.populations["household"], - person_df[household_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["spm_unit"], - person_df[spm_unit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["family"], - person_df[family_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["tax_unit"], - person_df[tax_unit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["marital_unit"], - person_df[marital_unit_id_col].values, - np.array(["member"] * len(person_df)), + # Run simulation + simulation = Simulation( + dataset=dataset, + tax_benefit_model_version=us_latest, + policy=policy, ) + simulation.run() - # Build simulation from populations - microsim.build_from_populations(builder.populations) - - # Set input variables for each entity - id_columns = { - "person_id", - "household_id", - "person_household_id", - "spm_unit_id", - "person_spm_unit_id", - "family_id", - "person_family_id", - "tax_unit_id", - "person_tax_unit_id", - "marital_unit_id", - "person_marital_unit_id", - } + # Extract outputs + output_data = simulation.output_dataset.data - for entity_name, entity_df in [ - ("person", person_data), - ("household", household_data), - ("spm_unit", spm_unit_data), - ("family", family_data), - ("tax_unit", tax_unit_data), - ("marital_unit", marital_unit_data), - ]: - df = pd.DataFrame(entity_df) - for column in df.columns: - if column not in id_columns and column in system.variables: - microsim.set_input(column, year, df[column].values) - - # Calculate output variables def safe_convert(value): try: return float(value) except (ValueError, TypeError): return str(value) - def extract_entity_outputs( - entity_name: str, n_rows: int, map_to: str - ) -> list[dict]: + def extract_entity_outputs(entity_name: str, entity_data, n_rows: int) -> list[dict]: outputs = [] for i in range(n_rows): row_dict = {} for var in us_latest.entity_variables[entity_name]: - val = microsim.calculate(var, period=year, map_to=map_to) - row_dict[var] = safe_convert(val.values[i]) + row_dict[var] = safe_convert(entity_data[var].iloc[i]) outputs.append(row_dict) return outputs return { - "person": extract_entity_outputs("person", n_people, "person"), + "person": extract_entity_outputs("person", output_data.person, n_people), "marital_unit": extract_entity_outputs( - "marital_unit", n_marital_units, "marital_unit" + "marital_unit", output_data.marital_unit, len(output_data.marital_unit) + ), + "family": extract_entity_outputs( + "family", output_data.family, len(output_data.family) + ), + "spm_unit": extract_entity_outputs( + "spm_unit", output_data.spm_unit, len(output_data.spm_unit) + ), + "tax_unit": extract_entity_outputs( + "tax_unit", output_data.tax_unit, len(output_data.tax_unit) + ), + "household": extract_entity_outputs( + "household", output_data.household, len(output_data.household) ), - "family": extract_entity_outputs("family", n_families, "family"), - "spm_unit": extract_entity_outputs("spm_unit", n_spm_units, "spm_unit"), - "tax_unit": extract_entity_outputs("tax_unit", n_tax_units, "tax_unit"), - "household": extract_entity_outputs("household", n_households, "household"), } diff --git a/src/policyengine_api/modal_app.py b/src/policyengine_api/modal_app.py index 2b486f3..1aa8119 100644 --- a/src/policyengine_api/modal_app.py +++ b/src/policyengine_api/modal_app.py @@ -7,8 +7,7 @@ Function naming follows the API hierarchy: - simulate_household_*: Single household calculation (/simulate/household) - simulate_economy_*: Single economy simulation (/simulate/economy) -- economy_comparison_*: Full economy comparison analysis (/analysis/economic-impact) -- household_impact_*: Household impact analysis (/analysis/household-impact) +- economy_comparison_*: Full economy comparison analysis (/analysis/compare/economy) Deploy with: modal deploy src/policyengine_api/modal_app.py """ @@ -807,6 +806,7 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N raise ValueError(f"Dataset {simulation.dataset_id} not found") # Import policyengine + from policyengine.core import Simulation as PESimulation from policyengine.tax_benefit_models.uk import uk_latest from policyengine.tax_benefit_models.uk.datasets import ( PolicyEngineUKDataset, @@ -814,7 +814,7 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N pe_model_version = uk_latest - # Get policy and dynamic as PEPolicy/PEDynamic objects + # Get policy and dynamic policy = _get_pe_policy_uk( simulation.policy_id, pe_model_version, session ) @@ -822,13 +822,6 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N simulation.dynamic_id, pe_model_version, session ) - # Convert to reform dict format for Microsimulation - # This is necessary because policyengine.core.Simulation applies - # reforms AFTER creating Microsimulation, which doesn't work - policy_reform = _pe_policy_to_reform_dict(policy) - dynamic_reform = _pe_policy_to_reform_dict(dynamic) - reform = _merge_reform_dicts(policy_reform, dynamic_reform) - # Download dataset local_path = download_dataset( dataset.filepath, supabase_url, supabase_key, storage_bucket @@ -841,12 +834,15 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N year=dataset.year, ) - # Run simulation using Microsimulation directly with reform - # This ensures reforms are applied at construction time + # Create and run simulation with logfire.span("run_simulation"): - pe_output_dataset = _run_uk_economy_simulation( - pe_dataset, reform, pe_model_version, simulation_id + pe_sim = PESimulation( + dataset=pe_dataset, + tax_benefit_model_version=pe_model_version, + policy=policy, + dynamic=dynamic, ) + pe_sim.ensure() # Save output dataset with logfire.span("save_output_dataset"): @@ -856,8 +852,8 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N output_path = f"/tmp/{output_filename}" # Set filepath and save - pe_output_dataset.filepath = output_path - pe_output_dataset.save() + pe_sim.output_dataset.filepath = output_path + pe_sim.output_dataset.save() # Upload to Supabase storage supabase = create_client(supabase_url, supabase_key) @@ -872,7 +868,7 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N ) # Create output dataset record - output_dataset_record = Dataset( + output_dataset = Dataset( name=f"Output: {dataset.name}", description=f"Output from simulation {simulation_id}", filepath=output_filename, @@ -880,12 +876,12 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N is_output_dataset=True, tax_benefit_model_id=dataset.tax_benefit_model_id, ) - session.add(output_dataset_record) + session.add(output_dataset) session.commit() - session.refresh(output_dataset_record) + session.refresh(output_dataset) # Link to simulation - simulation.output_dataset_id = output_dataset_record.id + simulation.output_dataset_id = output_dataset.id # Mark as completed simulation.status = SimulationStatus.COMPLETED @@ -976,15 +972,15 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N raise ValueError(f"Dataset {simulation.dataset_id} not found") # Import policyengine + from policyengine.core import Simulation as PESimulation from policyengine.tax_benefit_models.us import us_latest from policyengine.tax_benefit_models.us.datasets import ( PolicyEngineUSDataset, - USYearData, ) pe_model_version = us_latest - # Get policy and dynamic as PEPolicy/PEDynamic objects + # Get policy and dynamic policy = _get_pe_policy_us( simulation.policy_id, pe_model_version, session ) @@ -992,13 +988,6 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N simulation.dynamic_id, pe_model_version, session ) - # Convert to reform dict format for Microsimulation - # This is necessary because policyengine.core.Simulation applies - # reforms AFTER creating Microsimulation, which doesn't work - policy_reform = _pe_policy_to_reform_dict(policy) - dynamic_reform = _pe_policy_to_reform_dict(dynamic) - reform = _merge_reform_dicts(policy_reform, dynamic_reform) - # Download dataset local_path = download_dataset( dataset.filepath, supabase_url, supabase_key, storage_bucket @@ -1011,12 +1000,15 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N year=dataset.year, ) - # Run simulation using Microsimulation directly with reform - # This ensures reforms are applied at construction time + # Create and run simulation with logfire.span("run_simulation"): - pe_output_dataset = _run_us_economy_simulation( - pe_dataset, reform, pe_model_version, simulation_id + pe_sim = PESimulation( + dataset=pe_dataset, + tax_benefit_model_version=pe_model_version, + policy=policy, + dynamic=dynamic, ) + pe_sim.ensure() # Save output dataset with logfire.span("save_output_dataset"): @@ -1026,8 +1018,8 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N output_path = f"/tmp/{output_filename}" # Set filepath and save - pe_output_dataset.filepath = output_path - pe_output_dataset.save() + pe_sim.output_dataset.filepath = output_path + pe_sim.output_dataset.save() # Upload to Supabase storage supabase = create_client(supabase_url, supabase_key) @@ -1042,7 +1034,7 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N ) # Create output dataset record - output_dataset_record = Dataset( + output_dataset = Dataset( name=f"Output: {dataset.name}", description=f"Output from simulation {simulation_id}", filepath=output_filename, @@ -1050,12 +1042,12 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N is_output_dataset=True, tax_benefit_model_id=dataset.tax_benefit_model_id, ) - session.add(output_dataset_record) + session.add(output_dataset) session.commit() - session.refresh(output_dataset_record) + session.refresh(output_dataset) # Link to simulation - simulation.output_dataset_id = output_dataset_record.id + simulation.output_dataset_id = output_dataset.id # Mark as completed simulation.status = SimulationStatus.COMPLETED @@ -1823,403 +1815,6 @@ def _get_pe_dynamic_us(dynamic_id, model_version, session): return _get_pe_dynamic_uk(dynamic_id, model_version, session) -def _pe_policy_to_reform_dict(policy) -> dict | None: - """Convert a policyengine.core.policy.Policy to reform dict format. - - The policyengine-us/uk Microsimulation expects reforms in the format: - {"parameter.name": {"YYYY-MM-DD": value}} - - This is necessary because the policyengine.core.Simulation applies reforms - AFTER creating the Microsimulation, which doesn't work due to caching. - We need to pass the reform at Microsimulation construction time. - """ - if policy is None: - return None - - if not policy.parameter_values: - return None - - reform = {} - for pv in policy.parameter_values: - if not pv.parameter: - continue - param_name = pv.parameter.name - value = pv.value - start_date = pv.start_date - - if param_name and start_date: - # Format date as YYYY-MM-DD string - if hasattr(start_date, "strftime"): - date_str = start_date.strftime("%Y-%m-%d") - else: - date_str = str(start_date).split("T")[0] - - if param_name not in reform: - reform[param_name] = {} - reform[param_name][date_str] = value - - return reform if reform else None - - -def _merge_reform_dicts(reform1: dict | None, reform2: dict | None) -> dict | None: - """Merge two reform dicts, with reform2 taking precedence.""" - if reform1 is None and reform2 is None: - return None - if reform1 is None: - return reform2 - if reform2 is None: - return reform1 - - merged = dict(reform1) - for param_name, dates in reform2.items(): - if param_name not in merged: - merged[param_name] = {} - merged[param_name].update(dates) - return merged - - -def _run_us_economy_simulation(pe_dataset, reform, pe_model_version, simulation_id): - """Run US economy simulation using Microsimulation directly. - - This bypasses policyengine.core.Simulation which has a bug where reforms - are applied AFTER creating Microsimulation (when it's too late). - Instead, we pass the reform dict at Microsimulation construction time. - """ - from pathlib import Path - - import numpy as np - import pandas as pd - from microdf import MicroDataFrame - from policyengine.tax_benefit_models.us.datasets import ( - PolicyEngineUSDataset, - USYearData, - ) - from policyengine_core.simulations.simulation_builder import SimulationBuilder - from policyengine_us import Microsimulation - from policyengine_us.system import system - - # Load dataset - pe_dataset.load() - year = pe_dataset.year - - # Create Microsimulation with reform applied at construction time - microsim = Microsimulation(reform=reform) - - # Build simulation from dataset using SimulationBuilder - person_df = pd.DataFrame(pe_dataset.data.person) - - # Determine column naming convention - household_id_col = ( - "person_household_id" - if "person_household_id" in person_df.columns - else "household_id" - ) - marital_unit_id_col = ( - "person_marital_unit_id" - if "person_marital_unit_id" in person_df.columns - else "marital_unit_id" - ) - family_id_col = ( - "person_family_id" if "person_family_id" in person_df.columns else "family_id" - ) - spm_unit_id_col = ( - "person_spm_unit_id" - if "person_spm_unit_id" in person_df.columns - else "spm_unit_id" - ) - tax_unit_id_col = ( - "person_tax_unit_id" - if "person_tax_unit_id" in person_df.columns - else "tax_unit_id" - ) - - # Declare entities - builder = SimulationBuilder() - builder.populations = system.instantiate_entities() - - builder.declare_person_entity("person", person_df["person_id"].values) - builder.declare_entity("household", np.unique(person_df[household_id_col].values)) - builder.declare_entity("spm_unit", np.unique(person_df[spm_unit_id_col].values)) - builder.declare_entity("family", np.unique(person_df[family_id_col].values)) - builder.declare_entity("tax_unit", np.unique(person_df[tax_unit_id_col].values)) - builder.declare_entity( - "marital_unit", np.unique(person_df[marital_unit_id_col].values) - ) - - # Join persons to entities - builder.join_with_persons( - builder.populations["household"], - person_df[household_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["spm_unit"], - person_df[spm_unit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["family"], - person_df[family_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["tax_unit"], - person_df[tax_unit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["marital_unit"], - person_df[marital_unit_id_col].values, - np.array(["member"] * len(person_df)), - ) - - microsim.build_from_populations(builder.populations) - - # Set input variables - id_columns = { - "person_id", - "household_id", - "person_household_id", - "spm_unit_id", - "person_spm_unit_id", - "family_id", - "person_family_id", - "tax_unit_id", - "person_tax_unit_id", - "marital_unit_id", - "person_marital_unit_id", - } - - for entity_name, entity_data in [ - ("person", pe_dataset.data.person), - ("household", pe_dataset.data.household), - ("spm_unit", pe_dataset.data.spm_unit), - ("family", pe_dataset.data.family), - ("tax_unit", pe_dataset.data.tax_unit), - ("marital_unit", pe_dataset.data.marital_unit), - ]: - df = pd.DataFrame(entity_data) - for column in df.columns: - if column not in id_columns and column in system.variables: - microsim.set_input(column, year, df[column].values) - - # Calculate output variables and build output dataset - data = { - "person": pd.DataFrame(), - "marital_unit": pd.DataFrame(), - "family": pd.DataFrame(), - "spm_unit": pd.DataFrame(), - "tax_unit": pd.DataFrame(), - "household": pd.DataFrame(), - } - - weight_columns = { - "person_weight", - "household_weight", - "marital_unit_weight", - "family_weight", - "spm_unit_weight", - "tax_unit_weight", - } - - # Copy ID and weight columns from input dataset - for entity in data.keys(): - input_df = pd.DataFrame(getattr(pe_dataset.data, entity)) - entity_id_col = f"{entity}_id" - entity_weight_col = f"{entity}_weight" - - if entity_id_col in input_df.columns: - data[entity][entity_id_col] = input_df[entity_id_col].values - if entity_weight_col in input_df.columns: - data[entity][entity_weight_col] = input_df[entity_weight_col].values - - # Copy person-level group ID columns - for col in person_df.columns: - if col.startswith("person_") and col.endswith("_id"): - target_col = col.replace("person_", "") - if target_col in id_columns: - data["person"][target_col] = person_df[col].values - - # Calculate non-ID, non-weight variables - for entity, variables in pe_model_version.entity_variables.items(): - for var in variables: - if var not in id_columns and var not in weight_columns: - data[entity][var] = microsim.calculate( - var, period=year, map_to=entity - ).values - - # Convert to MicroDataFrames - data["person"] = MicroDataFrame(data["person"], weights="person_weight") - data["marital_unit"] = MicroDataFrame( - data["marital_unit"], weights="marital_unit_weight" - ) - data["family"] = MicroDataFrame(data["family"], weights="family_weight") - data["spm_unit"] = MicroDataFrame(data["spm_unit"], weights="spm_unit_weight") - data["tax_unit"] = MicroDataFrame(data["tax_unit"], weights="tax_unit_weight") - data["household"] = MicroDataFrame(data["household"], weights="household_weight") - - # Create output dataset - return PolicyEngineUSDataset( - id=simulation_id, - name=pe_dataset.name, - description=pe_dataset.description, - filepath=str(Path(pe_dataset.filepath).parent / (simulation_id + ".h5")), - year=year, - is_output_dataset=True, - data=USYearData( - person=data["person"], - marital_unit=data["marital_unit"], - family=data["family"], - spm_unit=data["spm_unit"], - tax_unit=data["tax_unit"], - household=data["household"], - ), - ) - - -def _run_uk_economy_simulation(pe_dataset, reform, pe_model_version, simulation_id): - """Run UK economy simulation using Microsimulation directly. - - This bypasses policyengine.core.Simulation which has a bug where reforms - are applied AFTER creating Microsimulation (when it's too late). - Instead, we pass the reform dict at Microsimulation construction time. - """ - from pathlib import Path - - import numpy as np - import pandas as pd - from microdf import MicroDataFrame - from policyengine.tax_benefit_models.uk.datasets import ( - PolicyEngineUKDataset, - UKYearData, - ) - from policyengine_core.simulations.simulation_builder import SimulationBuilder - from policyengine_uk import Microsimulation - from policyengine_uk.system import system - - # Load dataset - pe_dataset.load() - year = pe_dataset.year - - # Create Microsimulation with reform applied at construction time - microsim = Microsimulation(reform=reform) - - # Build simulation from dataset using SimulationBuilder - person_df = pd.DataFrame(pe_dataset.data.person) - - # Determine column naming convention - benunit_id_col = ( - "person_benunit_id" - if "person_benunit_id" in person_df.columns - else "benunit_id" - ) - household_id_col = ( - "person_household_id" - if "person_household_id" in person_df.columns - else "household_id" - ) - - # Declare entities - builder = SimulationBuilder() - builder.populations = system.instantiate_entities() - - builder.declare_person_entity("person", person_df["person_id"].values) - builder.declare_entity("benunit", np.unique(person_df[benunit_id_col].values)) - builder.declare_entity("household", np.unique(person_df[household_id_col].values)) - - # Join persons to entities - builder.join_with_persons( - builder.populations["benunit"], - person_df[benunit_id_col].values, - np.array(["member"] * len(person_df)), - ) - builder.join_with_persons( - builder.populations["household"], - person_df[household_id_col].values, - np.array(["member"] * len(person_df)), - ) - - microsim.build_from_populations(builder.populations) - - # Set input variables - id_columns = { - "person_id", - "benunit_id", - "person_benunit_id", - "household_id", - "person_household_id", - } - - for entity_name, entity_data in [ - ("person", pe_dataset.data.person), - ("benunit", pe_dataset.data.benunit), - ("household", pe_dataset.data.household), - ]: - df = pd.DataFrame(entity_data) - for column in df.columns: - if column not in id_columns and column in system.variables: - microsim.set_input(column, year, df[column].values) - - # Calculate output variables and build output dataset - data = { - "person": pd.DataFrame(), - "benunit": pd.DataFrame(), - "household": pd.DataFrame(), - } - - weight_columns = { - "person_weight", - "benunit_weight", - "household_weight", - } - - # Copy ID and weight columns from input dataset - for entity in data.keys(): - input_df = pd.DataFrame(getattr(pe_dataset.data, entity)) - entity_id_col = f"{entity}_id" - entity_weight_col = f"{entity}_weight" - - if entity_id_col in input_df.columns: - data[entity][entity_id_col] = input_df[entity_id_col].values - if entity_weight_col in input_df.columns: - data[entity][entity_weight_col] = input_df[entity_weight_col].values - - # Copy person-level group ID columns - for col in person_df.columns: - if col.startswith("person_") and col.endswith("_id"): - target_col = col.replace("person_", "") - if target_col in id_columns: - data["person"][target_col] = person_df[col].values - - # Calculate non-ID, non-weight variables - for entity, variables in pe_model_version.entity_variables.items(): - for var in variables: - if var not in id_columns and var not in weight_columns: - data[entity][var] = microsim.calculate( - var, period=year, map_to=entity - ).values - - # Convert to MicroDataFrames - data["person"] = MicroDataFrame(data["person"], weights="person_weight") - data["benunit"] = MicroDataFrame(data["benunit"], weights="benunit_weight") - data["household"] = MicroDataFrame(data["household"], weights="household_weight") - - # Create output dataset - return PolicyEngineUKDataset( - id=simulation_id, - name=pe_dataset.name, - description=pe_dataset.description, - filepath=str(Path(pe_dataset.filepath).parent / (simulation_id + ".h5")), - year=year, - is_output_dataset=True, - data=UKYearData( - person=data["person"], - benunit=data["benunit"], - household=data["household"], - ), - ) - - @app.function( image=uk_image, secrets=[db_secrets, logfire_secrets], @@ -2921,689 +2516,3 @@ def compute_change_aggregate_us( raise finally: logfire.force_flush() - - -# ============================================================================= -# Household Impact Functions -# ============================================================================= - - -@app.function( - image=uk_image, - secrets=[db_secrets, logfire_secrets], - memory=2048, - cpu=2, - timeout=300, -) -def household_impact_uk(report_id: str, traceparent: str | None = None) -> None: - """Run UK household impact analysis and write results to database.""" - import logfire - - configure_logfire("policyengine-modal-uk", traceparent) - - try: - with logfire.span("household_impact_uk", report_id=report_id): - from datetime import datetime, timezone - from uuid import UUID - - from sqlmodel import Session, create_engine - - database_url = get_database_url() - engine = create_engine(database_url) - - try: - from policyengine_api.models import ( - Household, - Report, - ReportStatus, - Simulation, - SimulationStatus, - ) - - with Session(engine) as session: - # Load report - report = session.get(Report, UUID(report_id)) - if not report: - raise ValueError(f"Report {report_id} not found") - - # Mark as running - report.status = ReportStatus.RUNNING - session.add(report) - session.commit() - - # Run baseline simulation - if report.baseline_simulation_id: - _run_household_simulation_uk( - report.baseline_simulation_id, session - ) - - # Run reform simulation if present - if report.reform_simulation_id: - _run_household_simulation_uk( - report.reform_simulation_id, session - ) - - # Mark report as completed - report.status = ReportStatus.COMPLETED - session.add(report) - session.commit() - - except Exception as e: - logfire.error( - "UK household impact failed", report_id=report_id, error=str(e) - ) - try: - from sqlmodel import text - - with Session(engine) as session: - session.execute( - text( - "UPDATE reports SET status = 'FAILED', error_message = :error " - "WHERE id = :report_id" - ), - {"report_id": report_id, "error": str(e)[:1000]}, - ) - session.commit() - except Exception as db_error: - logfire.error("Failed to update DB", error=str(db_error)) - raise - finally: - logfire.force_flush() - - -def _run_household_simulation_uk(simulation_id, session) -> None: - """Run a single UK household simulation.""" - from datetime import datetime, timezone - - from policyengine_api.models import ( - Household, - Simulation, - SimulationStatus, - ) - - simulation = session.get(Simulation, simulation_id) - if not simulation or simulation.status != SimulationStatus.PENDING: - return - - household = session.get(Household, simulation.household_id) - if not household: - raise ValueError(f"Household {simulation.household_id} not found") - - # Mark as running - simulation.status = SimulationStatus.RUNNING - simulation.started_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - - try: - # Get policy data if present - policy_data = _get_household_policy_data(simulation.policy_id, session) - - # Run calculation - result = _calculate_uk_household( - household.household_data, - household.year, - policy_data, - ) - - # Store result - simulation.household_result = result - simulation.status = SimulationStatus.COMPLETED - simulation.completed_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - except Exception as e: - simulation.status = SimulationStatus.FAILED - simulation.error_message = str(e) - simulation.completed_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - raise - - -def _calculate_uk_household( - household_data: dict, year: int, policy_data: dict | None -) -> dict: - """Calculate UK household and return result dict.""" - import tempfile - from pathlib import Path - - import pandas as pd - from microdf import MicroDataFrame - from policyengine.core import Simulation - from policyengine.tax_benefit_models.uk import uk_latest - from policyengine.tax_benefit_models.uk.datasets import ( - PolicyEngineUKDataset, - UKYearData, - ) - - people = household_data.get("people", []) - benunit = household_data.get("benunit", []) - hh = household_data.get("household", []) - - # Ensure lists - if isinstance(benunit, dict): - benunit = [benunit] - if isinstance(hh, dict): - hh = [hh] - - n_people = len(people) - n_benunits = max(1, len(benunit) if benunit else 1) - n_households = max(1, len(hh) if hh else 1) - - # Build person data - person_data = { - "person_id": list(range(n_people)), - "person_benunit_id": [0] * n_people, - "person_household_id": [0] * n_people, - "person_weight": [1.0] * n_people, - } - for i, person in enumerate(people): - for key, value in person.items(): - if key not in person_data: - person_data[key] = [0.0] * n_people - person_data[key][i] = value - - # Build benunit data - benunit_data = { - "benunit_id": list(range(n_benunits)), - "benunit_weight": [1.0] * n_benunits, - } - for i, bu in enumerate(benunit if benunit else [{}]): - for key, value in bu.items(): - if key not in benunit_data: - benunit_data[key] = [0.0] * n_benunits - benunit_data[key][i] = value - - # Build household data - household_df_data = { - "household_id": list(range(n_households)), - "household_weight": [1.0] * n_households, - "region": ["LONDON"] * n_households, - "tenure_type": ["RENT_PRIVATELY"] * n_households, - "council_tax": [0.0] * n_households, - "rent": [0.0] * n_households, - } - for i, h in enumerate(hh if hh else [{}]): - for key, value in h.items(): - if key not in household_df_data: - household_df_data[key] = [0.0] * n_households - household_df_data[key][i] = value - - # Create MicroDataFrames - person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") - benunit_df = MicroDataFrame(pd.DataFrame(benunit_data), weights="benunit_weight") - household_df = MicroDataFrame( - pd.DataFrame(household_df_data), weights="household_weight" - ) - - # Create temporary dataset - tmpdir = tempfile.mkdtemp() - filepath = str(Path(tmpdir) / "household_calc.h5") - - dataset = PolicyEngineUKDataset( - name="Household calculation", - description="Household(s) for calculation", - person=person_df, - benunit=benunit_df, - household=household_df, - filepath=filepath, - year_data_class=UKYearData, - ) - dataset.save() - - # Build policy if provided - policy = None - if policy_data: - from policyengine.core.policy import ParameterValue, Policy - - pe_param_values = [] - param_lookup = {p.name: p for p in uk_latest.parameters} - for pv in policy_data.get("parameter_values", []): - param_name = pv.get("parameter_name") - if param_name and param_name in param_lookup: - pe_pv = ParameterValue( - parameter=param_lookup[param_name], - value=pv.get("value"), - start_date=pv.get("start_date"), - end_date=pv.get("end_date"), - ) - pe_param_values.append(pe_pv) - - if pe_param_values: - policy = Policy( - name=policy_data.get("name", "Reform"), - description=policy_data.get("description", ""), - parameter_values=pe_param_values, - ) - - # Run simulation - sim = Simulation( - dataset=dataset, - tax_benefit_model_version=uk_latest, - policy=policy, - ) - sim.ensure() - - # Extract results - result = {"person": [], "benunit": [], "household": []} - - for i in range(n_people): - person_result = {} - for var in sim.output_dataset.person.columns: - val = sim.output_dataset.person[var].iloc[i] - person_result[var] = float(val) if hasattr(val, "item") else val - result["person"].append(person_result) - - for i in range(n_benunits): - benunit_result = {} - for var in sim.output_dataset.benunit.columns: - val = sim.output_dataset.benunit[var].iloc[i] - benunit_result[var] = float(val) if hasattr(val, "item") else val - result["benunit"].append(benunit_result) - - for i in range(n_households): - household_result = {} - for var in sim.output_dataset.household.columns: - val = sim.output_dataset.household[var].iloc[i] - household_result[var] = float(val) if hasattr(val, "item") else val - result["household"].append(household_result) - - return result - - -@app.function( - image=us_image, - secrets=[db_secrets, logfire_secrets], - memory=2048, - cpu=2, - timeout=300, -) -def household_impact_us(report_id: str, traceparent: str | None = None) -> None: - """Run US household impact analysis and write results to database.""" - import logfire - - configure_logfire("policyengine-modal-us", traceparent) - - try: - with logfire.span("household_impact_us", report_id=report_id): - from datetime import datetime, timezone - from uuid import UUID - - from sqlmodel import Session, create_engine - - database_url = get_database_url() - engine = create_engine(database_url) - - try: - from policyengine_api.models import ( - Household, - Report, - ReportStatus, - Simulation, - SimulationStatus, - ) - - with Session(engine) as session: - # Load report - report = session.get(Report, UUID(report_id)) - if not report: - raise ValueError(f"Report {report_id} not found") - - # Mark as running - report.status = ReportStatus.RUNNING - session.add(report) - session.commit() - - # Run baseline simulation - if report.baseline_simulation_id: - _run_household_simulation_us( - report.baseline_simulation_id, session - ) - - # Run reform simulation if present - if report.reform_simulation_id: - _run_household_simulation_us( - report.reform_simulation_id, session - ) - - # Mark report as completed - report.status = ReportStatus.COMPLETED - session.add(report) - session.commit() - - except Exception as e: - logfire.error( - "US household impact failed", report_id=report_id, error=str(e) - ) - try: - from sqlmodel import text - - with Session(engine) as session: - session.execute( - text( - "UPDATE reports SET status = 'FAILED', error_message = :error " - "WHERE id = :report_id" - ), - {"report_id": report_id, "error": str(e)[:1000]}, - ) - session.commit() - except Exception as db_error: - logfire.error("Failed to update DB", error=str(db_error)) - raise - finally: - logfire.force_flush() - - -def _run_household_simulation_us(simulation_id, session) -> None: - """Run a single US household simulation.""" - from datetime import datetime, timezone - - from policyengine_api.models import ( - Household, - Simulation, - SimulationStatus, - ) - - simulation = session.get(Simulation, simulation_id) - if not simulation or simulation.status != SimulationStatus.PENDING: - return - - household = session.get(Household, simulation.household_id) - if not household: - raise ValueError(f"Household {simulation.household_id} not found") - - # Mark as running - simulation.status = SimulationStatus.RUNNING - simulation.started_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - - try: - # Get policy data if present - policy_data = _get_household_policy_data(simulation.policy_id, session) - - # Run calculation - result = _calculate_us_household( - household.household_data, - household.year, - policy_data, - ) - - # Store result - simulation.household_result = result - simulation.status = SimulationStatus.COMPLETED - simulation.completed_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - except Exception as e: - simulation.status = SimulationStatus.FAILED - simulation.error_message = str(e) - simulation.completed_at = datetime.now(timezone.utc) - session.add(simulation) - session.commit() - raise - - -def _calculate_us_household( - household_data: dict, year: int, policy_data: dict | None -) -> dict: - """Calculate US household and return result dict.""" - import tempfile - from pathlib import Path - - import pandas as pd - from microdf import MicroDataFrame - from policyengine.core import Simulation - from policyengine.tax_benefit_models.us import us_latest - from policyengine.tax_benefit_models.us.datasets import ( - PolicyEngineUSDataset, - USYearData, - ) - - people = household_data.get("people", []) - tax_unit = household_data.get("tax_unit", []) - family = household_data.get("family", []) - spm_unit = household_data.get("spm_unit", []) - marital_unit = household_data.get("marital_unit", []) - hh = household_data.get("household", []) - - # Ensure lists - if isinstance(tax_unit, dict): - tax_unit = [tax_unit] - if isinstance(family, dict): - family = [family] - if isinstance(spm_unit, dict): - spm_unit = [spm_unit] - if isinstance(marital_unit, dict): - marital_unit = [marital_unit] - if isinstance(hh, dict): - hh = [hh] - - n_people = len(people) - n_tax_units = max(1, len(tax_unit) if tax_unit else 1) - n_families = max(1, len(family) if family else 1) - n_spm_units = max(1, len(spm_unit) if spm_unit else 1) - n_marital_units = max(1, len(marital_unit) if marital_unit else 1) - n_households = max(1, len(hh) if hh else 1) - - # Build person data - person_data = { - "person_id": list(range(n_people)), - "person_tax_unit_id": [0] * n_people, - "person_family_id": [0] * n_people, - "person_spm_unit_id": [0] * n_people, - "person_marital_unit_id": [0] * n_people, - "person_household_id": [0] * n_people, - "person_weight": [1.0] * n_people, - } - for i, person in enumerate(people): - for key, value in person.items(): - if key not in person_data: - person_data[key] = [0.0] * n_people - person_data[key][i] = value - - # Build tax_unit data - tax_unit_data = { - "tax_unit_id": list(range(n_tax_units)), - "tax_unit_weight": [1.0] * n_tax_units, - } - for i, tu in enumerate(tax_unit if tax_unit else [{}]): - for key, value in tu.items(): - if key not in tax_unit_data: - tax_unit_data[key] = [0.0] * n_tax_units - tax_unit_data[key][i] = value - - # Build family data - family_data = { - "family_id": list(range(n_families)), - "family_weight": [1.0] * n_families, - } - for i, fam in enumerate(family if family else [{}]): - for key, value in fam.items(): - if key not in family_data: - family_data[key] = [0.0] * n_families - family_data[key][i] = value - - # Build spm_unit data - spm_unit_data = { - "spm_unit_id": list(range(n_spm_units)), - "spm_unit_weight": [1.0] * n_spm_units, - } - for i, spm in enumerate(spm_unit if spm_unit else [{}]): - for key, value in spm.items(): - if key not in spm_unit_data: - spm_unit_data[key] = [0.0] * n_spm_units - spm_unit_data[key][i] = value - - # Build marital_unit data - marital_unit_data = { - "marital_unit_id": list(range(n_marital_units)), - "marital_unit_weight": [1.0] * n_marital_units, - } - for i, mu in enumerate(marital_unit if marital_unit else [{}]): - for key, value in mu.items(): - if key not in marital_unit_data: - marital_unit_data[key] = [0.0] * n_marital_units - marital_unit_data[key][i] = value - - # Build household data - household_df_data = { - "household_id": list(range(n_households)), - "household_weight": [1.0] * n_households, - } - for i, h in enumerate(hh if hh else [{}]): - for key, value in h.items(): - if key not in household_df_data: - household_df_data[key] = [0.0] * n_households - household_df_data[key][i] = value - - # Create MicroDataFrames - person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") - tax_unit_df = MicroDataFrame( - pd.DataFrame(tax_unit_data), weights="tax_unit_weight" - ) - family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") - spm_unit_df = MicroDataFrame( - pd.DataFrame(spm_unit_data), weights="spm_unit_weight" - ) - marital_unit_df = MicroDataFrame( - pd.DataFrame(marital_unit_data), weights="marital_unit_weight" - ) - household_df = MicroDataFrame( - pd.DataFrame(household_df_data), weights="household_weight" - ) - - # Create temporary dataset - tmpdir = tempfile.mkdtemp() - filepath = str(Path(tmpdir) / "household_calc.h5") - - dataset = PolicyEngineUSDataset( - name="Household calculation", - description="Household(s) for calculation", - person=person_df, - tax_unit=tax_unit_df, - family=family_df, - spm_unit=spm_unit_df, - marital_unit=marital_unit_df, - household=household_df, - filepath=filepath, - year_data_class=USYearData, - ) - dataset.save() - - # Build policy if provided - policy = None - if policy_data: - from policyengine.core.policy import ParameterValue, Policy - - pe_param_values = [] - param_lookup = {p.name: p for p in us_latest.parameters} - for pv in policy_data.get("parameter_values", []): - param_name = pv.get("parameter_name") - if param_name and param_name in param_lookup: - pe_pv = ParameterValue( - parameter=param_lookup[param_name], - value=pv.get("value"), - start_date=pv.get("start_date"), - end_date=pv.get("end_date"), - ) - pe_param_values.append(pe_pv) - - if pe_param_values: - policy = Policy( - name=policy_data.get("name", "Reform"), - description=policy_data.get("description", ""), - parameter_values=pe_param_values, - ) - - # Run simulation - sim = Simulation( - dataset=dataset, - tax_benefit_model_version=us_latest, - policy=policy, - ) - sim.ensure() - - # Extract results - result = { - "person": [], - "tax_unit": [], - "family": [], - "spm_unit": [], - "marital_unit": [], - "household": [], - } - - for i in range(n_people): - person_result = {} - for var in sim.output_dataset.person.columns: - val = sim.output_dataset.person[var].iloc[i] - person_result[var] = float(val) if hasattr(val, "item") else val - result["person"].append(person_result) - - for i in range(n_tax_units): - tu_result = {} - for var in sim.output_dataset.tax_unit.columns: - val = sim.output_dataset.tax_unit[var].iloc[i] - tu_result[var] = float(val) if hasattr(val, "item") else val - result["tax_unit"].append(tu_result) - - for i in range(n_families): - fam_result = {} - for var in sim.output_dataset.family.columns: - val = sim.output_dataset.family[var].iloc[i] - fam_result[var] = float(val) if hasattr(val, "item") else val - result["family"].append(fam_result) - - for i in range(n_spm_units): - spm_result = {} - for var in sim.output_dataset.spm_unit.columns: - val = sim.output_dataset.spm_unit[var].iloc[i] - spm_result[var] = float(val) if hasattr(val, "item") else val - result["spm_unit"].append(spm_result) - - for i in range(n_marital_units): - mu_result = {} - for var in sim.output_dataset.marital_unit.columns: - val = sim.output_dataset.marital_unit[var].iloc[i] - mu_result[var] = float(val) if hasattr(val, "item") else val - result["marital_unit"].append(mu_result) - - for i in range(n_households): - hh_result = {} - for var in sim.output_dataset.household.columns: - val = sim.output_dataset.household[var].iloc[i] - hh_result[var] = float(val) if hasattr(val, "item") else val - result["household"].append(hh_result) - - return result - - -def _get_household_policy_data(policy_id, session) -> dict | None: - """Get policy data for household calculation.""" - if policy_id is None: - return None - - from policyengine_api.models import Policy - - db_policy = session.get(Policy, policy_id) - if not db_policy: - return None - - return { - "name": db_policy.name, - "description": db_policy.description, - "parameter_values": [ - { - "parameter_name": pv.parameter.name if pv.parameter else None, - "value": pv.value_json.get("value") - if isinstance(pv.value_json, dict) - else pv.value_json, - "start_date": pv.start_date.isoformat() if pv.start_date else None, - "end_date": pv.end_date.isoformat() if pv.end_date else None, - } - for pv in db_policy.parameter_values - if pv.parameter - ], - } diff --git a/test_fixtures/fixtures_policy_reform.py b/test_fixtures/fixtures_policy_reform.py deleted file mode 100644 index f7534a5..0000000 --- a/test_fixtures/fixtures_policy_reform.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Fixtures for policy reform conversion tests.""" - -from dataclasses import dataclass -from datetime import date, datetime -from typing import Any - - -# ============================================================================= -# Mock objects for testing _pe_policy_to_reform_dict -# ============================================================================= - - -@dataclass -class MockParameter: - """Mock policyengine.core.models.parameter.Parameter.""" - - name: str - - -@dataclass -class MockParameterValue: - """Mock policyengine.core.models.parameter_value.ParameterValue.""" - - parameter: MockParameter | None - value: Any - start_date: date | datetime | str | None - - -@dataclass -class MockPolicy: - """Mock policyengine.core.policy.Policy.""" - - parameter_values: list[MockParameterValue] | None - - -# ============================================================================= -# Test data constants -# ============================================================================= - -# Simple policy with single parameter change -SIMPLE_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date=date(2024, 1, 1), - ) - ] -) - -SIMPLE_POLICY_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} -} - -# Policy with multiple parameter changes -MULTI_PARAM_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date=date(2024, 1, 1), - ), - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.refundable.fully_refundable"), - value=True, - start_date=date(2024, 1, 1), - ), - MockParameterValue( - parameter=MockParameter(name="gov.irs.income.bracket.rates.1"), - value=0.12, - start_date=date(2024, 1, 1), - ), - ] -) - -MULTI_PARAM_POLICY_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, - "gov.irs.credits.ctc.refundable.fully_refundable": {"2024-01-01": True}, - "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.12}, -} - -# Policy with same parameter at different dates -MULTI_DATE_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=2500, - start_date=date(2024, 1, 1), - ), - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date=date(2025, 1, 1), - ), - ] -) - -MULTI_DATE_POLICY_EXPECTED = { - "gov.irs.credits.ctc.amount.base": { - "2024-01-01": 2500, - "2025-01-01": 3000, - } -} - -# Policy with datetime start_date (has time component) -DATETIME_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date=datetime(2024, 1, 1, 12, 30, 45), - ) - ] -) - -DATETIME_POLICY_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} -} - -# Policy with ISO string start_date -ISO_STRING_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date="2024-01-01T00:00:00", - ) - ] -) - -ISO_STRING_POLICY_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000} -} - -# Empty policy (no parameter values) -EMPTY_POLICY = MockPolicy(parameter_values=[]) - -# None policy -NONE_POLICY = None - -# Policy with None parameter_values -NONE_PARAM_VALUES_POLICY = MockPolicy(parameter_values=None) - -# Policy with invalid entries (missing parameter or start_date) -INVALID_ENTRIES_POLICY = MockPolicy( - parameter_values=[ - MockParameterValue( - parameter=None, # Missing parameter - value=3000, - start_date=date(2024, 1, 1), - ), - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.ctc.amount.base"), - value=3000, - start_date=None, # Missing start_date - ), - MockParameterValue( - parameter=MockParameter(name="gov.irs.credits.eitc.max.0"), - value=600, - start_date=date(2024, 1, 1), # This one is valid - ), - ] -) - -INVALID_ENTRIES_POLICY_EXPECTED = { - "gov.irs.credits.eitc.max.0": {"2024-01-01": 600} -} - - -# ============================================================================= -# Test data for _merge_reform_dicts -# ============================================================================= - -REFORM_DICT_1 = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 2000}, - "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.10}, -} - -REFORM_DICT_2 = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, # Overwrites - "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, # New param -} - -MERGED_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, # From reform2 - "gov.irs.income.bracket.rates.1": {"2024-01-01": 0.10}, # From reform1 - "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, # From reform2 -} - -REFORM_DICT_3 = { - "gov.irs.credits.ctc.amount.base": { - "2024-01-01": 2500, - "2025-01-01": 2700, - }, -} - -REFORM_DICT_4 = { - "gov.irs.credits.ctc.amount.base": { - "2025-01-01": 3000, # Overwrites 2025 date - "2026-01-01": 3500, # New date - }, -} - -MERGED_MULTI_DATE_EXPECTED = { - "gov.irs.credits.ctc.amount.base": { - "2024-01-01": 2500, # From reform3 - "2025-01-01": 3000, # From reform4 (overwrites) - "2026-01-01": 3500, # From reform4 (new) - }, -} - - -# ============================================================================= -# Test data for household calculation policy conversion -# ============================================================================= - -# Policy data as it comes from the API (stored in database) -HOUSEHOLD_POLICY_DATA = { - "parameter_values": [ - { - "parameter_name": "gov.irs.credits.ctc.amount.base", - "value": 3000, - "start_date": "2024-01-01", - }, - { - "parameter_name": "gov.irs.credits.ctc.refundable.fully_refundable", - "value": True, - "start_date": "2024-01-01", - }, - ] -} - -HOUSEHOLD_POLICY_DATA_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, - "gov.irs.credits.ctc.refundable.fully_refundable": {"2024-01-01": True}, -} - -# Policy data with ISO datetime strings -HOUSEHOLD_POLICY_DATA_DATETIME = { - "parameter_values": [ - { - "parameter_name": "gov.irs.credits.ctc.amount.base", - "value": 3000, - "start_date": "2024-01-01T00:00:00.000Z", - }, - ] -} - -HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED = { - "gov.irs.credits.ctc.amount.base": {"2024-01-01": 3000}, -} - -# Empty policy data -HOUSEHOLD_EMPTY_POLICY_DATA = {"parameter_values": []} - -# None policy data -HOUSEHOLD_NONE_POLICY_DATA = None - -# Policy data with missing fields -HOUSEHOLD_INCOMPLETE_POLICY_DATA = { - "parameter_values": [ - { - "parameter_name": None, # Missing - "value": 3000, - "start_date": "2024-01-01", - }, - { - "parameter_name": "gov.irs.credits.ctc.amount.base", - "value": 3000, - "start_date": None, # Missing - }, - { - "parameter_name": "gov.irs.credits.eitc.max.0", - "value": 600, - "start_date": "2024-01-01", # Valid - }, - ] -} - -HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED = { - "gov.irs.credits.eitc.max.0": {"2024-01-01": 600}, -} diff --git a/tests/test_policy_reform.py b/tests/test_policy_reform.py deleted file mode 100644 index cfee3b8..0000000 --- a/tests/test_policy_reform.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Tests for policy reform conversion logic. - -Tests the helper functions that convert policy objects to reform dict format -for use with Microsimulation. These are critical for fixing the bug where -reforms weren't being applied to economy-wide and household simulations. -""" - -import sys -from unittest.mock import MagicMock - -import pytest - -# Mock modal before importing modal_app -sys.modules["modal"] = MagicMock() - -from test_fixtures.fixtures_policy_reform import ( - DATETIME_POLICY, - DATETIME_POLICY_EXPECTED, - EMPTY_POLICY, - HOUSEHOLD_EMPTY_POLICY_DATA, - HOUSEHOLD_INCOMPLETE_POLICY_DATA, - HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED, - HOUSEHOLD_NONE_POLICY_DATA, - HOUSEHOLD_POLICY_DATA, - HOUSEHOLD_POLICY_DATA_DATETIME, - HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED, - HOUSEHOLD_POLICY_DATA_EXPECTED, - INVALID_ENTRIES_POLICY, - INVALID_ENTRIES_POLICY_EXPECTED, - ISO_STRING_POLICY, - ISO_STRING_POLICY_EXPECTED, - MERGED_EXPECTED, - MERGED_MULTI_DATE_EXPECTED, - MULTI_DATE_POLICY, - MULTI_DATE_POLICY_EXPECTED, - MULTI_PARAM_POLICY, - MULTI_PARAM_POLICY_EXPECTED, - NONE_PARAM_VALUES_POLICY, - NONE_POLICY, - REFORM_DICT_1, - REFORM_DICT_2, - REFORM_DICT_3, - REFORM_DICT_4, - SIMPLE_POLICY, - SIMPLE_POLICY_EXPECTED, -) - -# Import after mocking modal -from policyengine_api.modal_app import _merge_reform_dicts, _pe_policy_to_reform_dict - - -class TestPePolicyToReformDict: - """Tests for _pe_policy_to_reform_dict function.""" - - # ========================================================================= - # Given: Valid policy with single parameter - # ========================================================================= - - def test__given_simple_policy_with_date_object__then_returns_correct_reform_dict( - self, - ): - """Given a policy with a single parameter using date object, - then returns correctly formatted reform dict.""" - # When - result = _pe_policy_to_reform_dict(SIMPLE_POLICY) - - # Then - assert result == SIMPLE_POLICY_EXPECTED - - def test__given_policy_with_datetime_object__then_extracts_date_correctly(self): - """Given a policy with datetime start_date (has time component), - then extracts just the date part for the reform dict.""" - # When - result = _pe_policy_to_reform_dict(DATETIME_POLICY) - - # Then - assert result == DATETIME_POLICY_EXPECTED - - def test__given_policy_with_iso_string_date__then_parses_date_correctly(self): - """Given a policy with ISO string start_date, - then parses and extracts the date correctly.""" - # When - result = _pe_policy_to_reform_dict(ISO_STRING_POLICY) - - # Then - assert result == ISO_STRING_POLICY_EXPECTED - - # ========================================================================= - # Given: Policy with multiple parameters - # ========================================================================= - - def test__given_policy_with_multiple_parameters__then_includes_all_in_dict(self): - """Given a policy with multiple parameter changes, - then includes all parameters in the reform dict.""" - # When - result = _pe_policy_to_reform_dict(MULTI_PARAM_POLICY) - - # Then - assert result == MULTI_PARAM_POLICY_EXPECTED - - def test__given_policy_with_same_param_multiple_dates__then_includes_all_dates( - self, - ): - """Given a policy with the same parameter changed at different dates, - then includes all date entries for that parameter.""" - # When - result = _pe_policy_to_reform_dict(MULTI_DATE_POLICY) - - # Then - assert result == MULTI_DATE_POLICY_EXPECTED - - # ========================================================================= - # Given: Empty or None policy - # ========================================================================= - - def test__given_none_policy__then_returns_none(self): - """Given None as policy, - then returns None.""" - # When - result = _pe_policy_to_reform_dict(NONE_POLICY) - - # Then - assert result is None - - def test__given_policy_with_empty_parameter_values__then_returns_none(self): - """Given a policy with empty parameter_values list, - then returns None.""" - # When - result = _pe_policy_to_reform_dict(EMPTY_POLICY) - - # Then - assert result is None - - def test__given_policy_with_none_parameter_values__then_returns_none(self): - """Given a policy with parameter_values=None, - then returns None.""" - # When - result = _pe_policy_to_reform_dict(NONE_PARAM_VALUES_POLICY) - - # Then - assert result is None - - # ========================================================================= - # Given: Policy with invalid entries - # ========================================================================= - - def test__given_policy_with_invalid_entries__then_skips_invalid_keeps_valid(self): - """Given a policy with some invalid entries (missing parameter or date), - then skips invalid entries and keeps valid ones.""" - # When - result = _pe_policy_to_reform_dict(INVALID_ENTRIES_POLICY) - - # Then - assert result == INVALID_ENTRIES_POLICY_EXPECTED - - -class TestMergeReformDicts: - """Tests for _merge_reform_dicts function.""" - - # ========================================================================= - # Given: Two valid reform dicts - # ========================================================================= - - def test__given_two_reform_dicts__then_merges_with_second_taking_precedence(self): - """Given two reform dicts with overlapping parameters, - then merges them with the second dict taking precedence.""" - # When - result = _merge_reform_dicts(REFORM_DICT_1, REFORM_DICT_2) - - # Then - assert result == MERGED_EXPECTED - - def test__given_dicts_with_multiple_dates__then_merges_date_entries_correctly(self): - """Given reform dicts with same parameter at multiple dates, - then merges date entries correctly with second taking precedence.""" - # When - result = _merge_reform_dicts(REFORM_DICT_3, REFORM_DICT_4) - - # Then - assert result == MERGED_MULTI_DATE_EXPECTED - - # ========================================================================= - # Given: None values - # ========================================================================= - - def test__given_both_none__then_returns_none(self): - """Given both reform dicts are None, - then returns None.""" - # When - result = _merge_reform_dicts(None, None) - - # Then - assert result is None - - def test__given_first_none__then_returns_second(self): - """Given first reform dict is None, - then returns the second dict.""" - # When - result = _merge_reform_dicts(None, REFORM_DICT_1) - - # Then - assert result == REFORM_DICT_1 - - def test__given_second_none__then_returns_first(self): - """Given second reform dict is None, - then returns the first dict.""" - # When - result = _merge_reform_dicts(REFORM_DICT_1, None) - - # Then - assert result == REFORM_DICT_1 - - # ========================================================================= - # Given: Original dict should not be mutated - # ========================================================================= - - def test__given_two_dicts__then_does_not_mutate_original_dicts(self): - """Given two reform dicts, - then merging does not mutate the original dicts.""" - # Given - original_dict1 = {"param.a": {"2024-01-01": 100}} - original_dict2 = {"param.b": {"2024-01-01": 200}} - dict1_copy = dict(original_dict1) - dict2_copy = dict(original_dict2) - - # When - _merge_reform_dicts(original_dict1, original_dict2) - - # Then - assert original_dict1 == dict1_copy - assert original_dict2 == dict2_copy - - -class TestHouseholdPolicyDataConversion: - """Tests for the policy data conversion logic used in household calculations. - - This tests the conversion logic as it appears in _calculate_household_us - and _calculate_household_uk functions. - """ - - def _convert_policy_data_to_reform(self, policy_data: dict | None) -> dict | None: - """Convert policy_data (from API) to reform dict format. - - This mirrors the conversion logic in _calculate_household_us. - """ - if not policy_data or not policy_data.get("parameter_values"): - return None - - reform = {} - for pv in policy_data["parameter_values"]: - param_name = pv.get("parameter_name") - value = pv.get("value") - start_date = pv.get("start_date") - - if param_name and start_date: - # Parse ISO date string to get just the date part - if "T" in start_date: - date_str = start_date.split("T")[0] - else: - date_str = start_date - - if param_name not in reform: - reform[param_name] = {} - reform[param_name][date_str] = value - - return reform if reform else None - - # ========================================================================= - # Given: Valid policy data from API - # ========================================================================= - - def test__given_valid_policy_data__then_converts_to_reform_dict(self): - """Given valid policy data from the API, - then converts it to the correct reform dict format.""" - # When - result = self._convert_policy_data_to_reform(HOUSEHOLD_POLICY_DATA) - - # Then - assert result == HOUSEHOLD_POLICY_DATA_EXPECTED - - def test__given_policy_data_with_datetime_strings__then_extracts_date_part(self): - """Given policy data with ISO datetime strings (with T and timezone), - then extracts just the date part.""" - # When - result = self._convert_policy_data_to_reform(HOUSEHOLD_POLICY_DATA_DATETIME) - - # Then - assert result == HOUSEHOLD_POLICY_DATA_DATETIME_EXPECTED - - # ========================================================================= - # Given: Empty or None policy data - # ========================================================================= - - def test__given_none_policy_data__then_returns_none(self): - """Given None policy data, - then returns None.""" - # When - result = self._convert_policy_data_to_reform(HOUSEHOLD_NONE_POLICY_DATA) - - # Then - assert result is None - - def test__given_empty_parameter_values__then_returns_none(self): - """Given policy data with empty parameter_values list, - then returns None.""" - # When - result = self._convert_policy_data_to_reform(HOUSEHOLD_EMPTY_POLICY_DATA) - - # Then - assert result is None - - # ========================================================================= - # Given: Incomplete policy data - # ========================================================================= - - def test__given_incomplete_entries__then_skips_invalid_keeps_valid(self): - """Given policy data with some entries missing required fields, - then skips invalid entries and keeps valid ones.""" - # When - result = self._convert_policy_data_to_reform(HOUSEHOLD_INCOMPLETE_POLICY_DATA) - - # Then - assert result == HOUSEHOLD_INCOMPLETE_POLICY_DATA_EXPECTED - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) From 6e9ca4121b3f295a5263bca040863bb1d51ec48f Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 11 Feb 2026 23:12:13 +0100 Subject: [PATCH 16/87] test: Add tests for US policy reform application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests to verify that US policy reforms are applied correctly to household calculations. These tests cover: - Integration tests via API endpoints (TestUSPolicyReform, TestUKPolicyReform) - Unit tests for the calculation functions directly (test_household_calculation.py) The tests verify: 1. Baseline calculations work correctly 2. Reforms change household net income as expected 3. Running a reform doesn't pollute subsequent baseline calculations (regression test for the singleton pollution bug) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- tests/test_household.py | 197 ++++++++++++++++++++++++++++ tests/test_household_calculation.py | 128 ++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 tests/test_household_calculation.py diff --git a/tests/test_household.py b/tests/test_household.py index a7248b3..eab15a5 100644 --- a/tests/test_household.py +++ b/tests/test_household.py @@ -289,5 +289,202 @@ def test_missing_people(self): assert response.status_code == 422 +class TestUSPolicyReform: + """Tests for US household calculations with policy reforms.""" + + def _get_us_model_id(self) -> str: + """Get the US tax benefit model ID.""" + response = client.get("/tax-benefit-models/") + assert response.status_code == 200 + models = response.json() + for model in models: + if "us" in model["name"].lower(): + return model["id"] + raise AssertionError("US model not found") + + def _get_parameter_id(self, model_id: str, param_name: str) -> str: + """Get a parameter ID by name.""" + response = client.get( + f"/parameters/?tax_benefit_model_id={model_id}&limit=10000" + ) + assert response.status_code == 200 + params = response.json() + for param in params: + if param["name"] == param_name: + return param["id"] + raise AssertionError(f"Parameter {param_name} not found") + + def _create_policy(self, param_id: str, value: float) -> str: + """Create a policy with a parameter value.""" + response = client.post( + "/policies/", + json={ + "name": "Test Reform", + "description": "Test reform for household calculation", + "parameter_values": [ + { + "parameter_id": param_id, + "value_json": value, + "start_date": "2024-01-01T00:00:00Z", + } + ], + }, + ) + assert response.status_code == 200 + return response.json()["id"] + + def test_us_reform_changes_household_net_income(self): + """Test that a US policy reform changes household net income. + + This test verifies the fix for the US reform application bug where + reforms were not being applied correctly due to the shared singleton + TaxBenefitSystem in policyengine-us. + """ + # Get the US model and a UBI parameter + model_id = self._get_us_model_id() + param_name = "gov.contrib.ubi_center.basic_income.amount.person.by_age[3].amount" + param_id = self._get_parameter_id(model_id, param_name) + + # Create a policy with $1000 UBI for older adults + policy_id = self._create_policy(param_id, 1000) + + # Run baseline calculation (no policy) + baseline_response = client.post( + "/household/calculate", + json={ + "tax_benefit_model_name": "policyengine_us", + "people": [{"age": 40, "employment_income": 70000}], + "tax_unit": [{"state_code": "CA"}], + "household": [{"state_fips": 6}], + "year": 2024, + }, + ) + assert baseline_response.status_code == 200 + baseline_data = _poll_job(baseline_response.json()["job_id"]) + baseline_net_income = baseline_data["result"]["household"][0][ + "household_net_income" + ] + + # Run reform calculation (with UBI policy) + reform_response = client.post( + "/household/calculate", + json={ + "tax_benefit_model_name": "policyengine_us", + "people": [{"age": 40, "employment_income": 70000}], + "tax_unit": [{"state_code": "CA"}], + "household": [{"state_fips": 6}], + "year": 2024, + "policy_id": policy_id, + }, + ) + assert reform_response.status_code == 200 + reform_data = _poll_job(reform_response.json()["job_id"]) + reform_net_income = reform_data["result"]["household"][0][ + "household_net_income" + ] + + # Verify the reform increased net income by approximately $1000 + difference = reform_net_income - baseline_net_income + assert abs(difference - 1000) < 1, ( + f"Expected ~$1000 difference, got ${difference:.2f}. " + f"Baseline: ${baseline_net_income:.2f}, Reform: ${reform_net_income:.2f}" + ) + + +class TestUKPolicyReform: + """Tests for UK household calculations with policy reforms.""" + + def _get_uk_model_id(self) -> str | None: + """Get the UK tax benefit model ID, or None if not seeded.""" + response = client.get("/tax-benefit-models/") + assert response.status_code == 200 + models = response.json() + for model in models: + if "uk" in model["name"].lower(): + return model["id"] + return None + + def _get_parameter_id(self, model_id: str, param_name: str) -> str: + """Get a parameter ID by name.""" + response = client.get( + f"/parameters/?tax_benefit_model_id={model_id}&limit=10000" + ) + assert response.status_code == 200 + params = response.json() + for param in params: + if param["name"] == param_name: + return param["id"] + raise AssertionError(f"Parameter {param_name} not found") + + def _create_policy(self, param_id: str, value: float) -> str: + """Create a policy with a parameter value.""" + response = client.post( + "/policies/", + json={ + "name": "Test UK Reform", + "description": "Test reform for UK household calculation", + "parameter_values": [ + { + "parameter_id": param_id, + "value_json": value, + "start_date": "2026-01-01T00:00:00Z", + } + ], + }, + ) + assert response.status_code == 200 + return response.json()["id"] + + def test_uk_reform_changes_household_net_income(self): + """Test that a UK policy reform changes household net income.""" + # Get the UK model and a UBI parameter + model_id = self._get_uk_model_id() + if model_id is None: + pytest.skip("UK model not seeded in database") + param_name = "gov.contrib.ubi_center.basic_income.adult" + param_id = self._get_parameter_id(model_id, param_name) + + # Create a policy with £1000 UBI for adults + policy_id = self._create_policy(param_id, 1000) + + # Run baseline calculation (no policy) + baseline_response = client.post( + "/household/calculate", + json={ + "tax_benefit_model_name": "policyengine_uk", + "people": [{"age": 30, "employment_income": 30000}], + "year": 2026, + }, + ) + assert baseline_response.status_code == 200 + baseline_data = _poll_job(baseline_response.json()["job_id"]) + baseline_net_income = baseline_data["result"]["household"][0][ + "household_net_income" + ] + + # Run reform calculation (with UBI policy) + reform_response = client.post( + "/household/calculate", + json={ + "tax_benefit_model_name": "policyengine_uk", + "people": [{"age": 30, "employment_income": 30000}], + "year": 2026, + "policy_id": policy_id, + }, + ) + assert reform_response.status_code == 200 + reform_data = _poll_job(reform_response.json()["job_id"]) + reform_net_income = reform_data["result"]["household"][0][ + "household_net_income" + ] + + # Verify the reform increased net income + difference = reform_net_income - baseline_net_income + assert difference > 0, ( + f"Expected positive difference, got £{difference:.2f}. " + f"Baseline: £{baseline_net_income:.2f}, Reform: £{reform_net_income:.2f}" + ) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_household_calculation.py b/tests/test_household_calculation.py new file mode 100644 index 0000000..e4fc2a5 --- /dev/null +++ b/tests/test_household_calculation.py @@ -0,0 +1,128 @@ +"""Unit tests for household calculation functions. + +These tests verify that the calculation functions work correctly with policy reforms, +without requiring database setup or API calls. +""" + +import pytest + +from policyengine_api.api.household import _calculate_household_us + + +class TestUSHouseholdCalculation: + """Unit tests for US household calculation with policy reforms.""" + + @pytest.mark.slow + def test_baseline_calculation(self): + """Test basic US household calculation without policy.""" + result = _calculate_household_us( + people=[{"employment_income": 70000, "age": 40}], + marital_unit=[], + family=[], + spm_unit=[], + tax_unit=[{"state_code": "CA"}], + household=[{"state_fips": 6}], + year=2024, + policy_data=None, + ) + + assert "person" in result + assert "household" in result + assert "tax_unit" in result + assert len(result["person"]) == 1 + assert result["tax_unit"][0]["income_tax"] > 0 + + @pytest.mark.slow + def test_reform_changes_net_income(self): + """Test that a US policy reform changes household net income. + + This test verifies the fix for the US reform application bug where + reforms were not being applied correctly due to the shared singleton + TaxBenefitSystem in policyengine-us. + """ + household_args = { + "people": [{"employment_income": 70000, "age": 40}], + "marital_unit": [], + "family": [], + "spm_unit": [], + "tax_unit": [{"state_code": "CA"}], + "household": [{"state_fips": 6}], + "year": 2024, + } + + # Calculate baseline (no policy) + baseline = _calculate_household_us(**household_args, policy_data=None) + baseline_net_income = baseline["household"][0]["household_net_income"] + + # Calculate with $1000 UBI reform + policy_data = { + "name": "Test UBI", + "description": "Test UBI reform", + "parameter_values": [ + { + "parameter_name": "gov.contrib.ubi_center.basic_income.amount.person.by_age[3].amount", + "value": 1000, + "start_date": "2024-01-01T00:00:00", + "end_date": None, + } + ], + } + reform = _calculate_household_us(**household_args, policy_data=policy_data) + reform_net_income = reform["household"][0]["household_net_income"] + + # Verify the reform increased net income by exactly $1000 + difference = reform_net_income - baseline_net_income + assert abs(difference - 1000) < 1, ( + f"Expected ~$1000 difference, got ${difference:.2f}. " + f"Baseline: ${baseline_net_income:.2f}, Reform: ${reform_net_income:.2f}" + ) + + @pytest.mark.slow + def test_reform_does_not_affect_baseline(self): + """Test that running reform doesn't pollute baseline calculations. + + This is a regression test for the singleton pollution bug where running + a reform calculation would affect subsequent baseline calculations. + """ + household_args = { + "people": [{"employment_income": 70000, "age": 40}], + "marital_unit": [], + "family": [], + "spm_unit": [], + "tax_unit": [{"state_code": "CA"}], + "household": [{"state_fips": 6}], + "year": 2024, + } + + # First baseline + baseline1 = _calculate_household_us(**household_args, policy_data=None) + baseline1_net_income = baseline1["household"][0]["household_net_income"] + + # Run reform + policy_data = { + "name": "Test UBI", + "description": "Test UBI reform", + "parameter_values": [ + { + "parameter_name": "gov.contrib.ubi_center.basic_income.amount.person.by_age[3].amount", + "value": 5000, + "start_date": "2024-01-01T00:00:00", + "end_date": None, + } + ], + } + _calculate_household_us(**household_args, policy_data=policy_data) + + # Second baseline - should be same as first + baseline2 = _calculate_household_us(**household_args, policy_data=None) + baseline2_net_income = baseline2["household"][0]["household_net_income"] + + # Verify baselines are identical + assert abs(baseline1_net_income - baseline2_net_income) < 0.01, ( + f"Baseline changed after reform calculation! " + f"Before: ${baseline1_net_income:.2f}, After: ${baseline2_net_income:.2f}" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 43135e26c25f348e5a82bf4688da768eea0fe4a6 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Mon, 16 Feb 2026 23:26:04 +0100 Subject: [PATCH 17/87] fix: Install policyengine.py from app-v2-migration branch The PyPI release (3.1.15) has a bug where US reforms silently fail due to the shared singleton TaxBenefitSystem (policyengine.py#232). The fix exists on the app-v2-migration branch but hasn't been released. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1fe9093..91ec88a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,11 @@ dependencies = [ "psycopg2-binary>=2.9.10", "supabase>=2.10.0", "storage3>=0.8.1", - "policyengine>=3.1.15", + # IMPORTANT: Before merging app-v2-migration into main, replace this git + # dependency with the production PyPI version of policyengine (e.g., "policyengine>=X.Y.Z"). + # The git ref is used here because the app-v2-migration branch contains fixes + # (US reform application, regions support) not yet released to PyPI. + "policyengine @ git+https://github.com/PolicyEngine/policyengine.py.git@app-v2-migration", "policyengine-uk>=2.0.0", "policyengine-us>=1.0.0", "pydantic>=2.9.2", From 6e77d4d8f224970a12646e9921c74bc162a9225b Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 17 Feb 2026 00:12:04 +0100 Subject: [PATCH 18/87] fix: Allow direct references in hatch metadata Hatchling rejects git URL dependencies unless explicitly opted in. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 91ec88a..d624a6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ dev = [ requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.targets.wheel] packages = ["src/policyengine_api"] From 754126e06d1a7b3d4c414121cc41d610210d51a0 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 17 Feb 2026 00:33:54 +0100 Subject: [PATCH 19/87] chore: Remove test scripts, Nevada seed, and archived Supabase migrations Test scripts in scripts/ were ad-hoc debugging aids, not part of the test suite. Nevada seed is no longer needed. Archived Supabase migrations are superseded by Alembic. Co-Authored-By: Claude Opus 4.6 --- scripts/seed_nevada.py | 128 ------- scripts/test_economy_simulation.py | 277 -------------- scripts/test_household_impact.py | 135 ------- scripts/test_household_scenarios.py | 344 ------------------ ...229000000_add_parameter_values_indexes.sql | 16 - .../20260103000000_add_poverty_inequality.sql | 33 -- .../20260111000000_add_aggregate_status.sql | 13 - .../20260203000000_create_households.sql | 14 - ...001_create_user_household_associations.sql | 14 - ...203000002_simulation_household_support.sql | 16 - 10 files changed, 990 deletions(-) delete mode 100644 scripts/seed_nevada.py delete mode 100644 scripts/test_economy_simulation.py delete mode 100644 scripts/test_household_impact.py delete mode 100644 scripts/test_household_scenarios.py delete mode 100644 supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql delete mode 100644 supabase/migrations_archived/20260103000000_add_poverty_inequality.sql delete mode 100644 supabase/migrations_archived/20260111000000_add_aggregate_status.sql delete mode 100644 supabase/migrations_archived/20260203000000_create_households.sql delete mode 100644 supabase/migrations_archived/20260203000001_create_user_household_associations.sql delete mode 100644 supabase/migrations_archived/20260203000002_simulation_household_support.sql diff --git a/scripts/seed_nevada.py b/scripts/seed_nevada.py deleted file mode 100644 index 0af2cb4..0000000 --- a/scripts/seed_nevada.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Seed Nevada datasets into local Supabase. - -This script seeds pre-created Nevada state and congressional district datasets -into the local Supabase database for testing purposes. - -Usage: - uv run python scripts/seed_nevada.py -""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from rich.console import Console -from sqlmodel import Session, create_engine, select - -from policyengine_api.config.settings import settings -from policyengine_api.models import Dataset, TaxBenefitModel -from policyengine_api.services.storage import upload_dataset_for_seeding - -console = Console() - -# Nevada datasets location -NEVADA_DATA_DIR = Path(__file__).parent.parent / "test_data" / "nevada_datasets" - - -def main(): - """Seed Nevada datasets.""" - console.print("[bold blue]Seeding Nevada datasets for testing...") - - engine = create_engine(settings.database_url, echo=False) - - with Session(engine) as session: - # Get or create US model - us_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") - ).first() - - if not us_model: - console.print(" Creating US tax-benefit model...") - us_model = TaxBenefitModel( - name="policyengine-us", - description="US tax-benefit system model", - ) - session.add(us_model) - session.commit() - session.refresh(us_model) - console.print(" [green]✓[/green] Created policyengine-us model") - - # Seed state datasets - states_dir = NEVADA_DATA_DIR / "states" - if states_dir.exists(): - console.print("\n [bold]Nevada State Datasets:[/bold]") - for h5_file in sorted(states_dir.glob("*.h5")): - name = h5_file.stem # e.g., "NV_year_2024" - year = int(name.split("_")[-1]) - - # Check if already exists - existing = session.exec( - select(Dataset).where(Dataset.name == name) - ).first() - - if existing: - console.print(f" [yellow]⏭[/yellow] {name} (already exists)") - continue - - # Upload to storage - console.print(f" Uploading {name}...", end=" ") - try: - object_name = upload_dataset_for_seeding(str(h5_file)) - - # Create database record - db_dataset = Dataset( - name=name, - description=f"Nevada state dataset for year {year}", - filepath=object_name, - year=year, - tax_benefit_model_id=us_model.id, - ) - session.add(db_dataset) - session.commit() - console.print("[green]✓[/green]") - except Exception as e: - console.print(f"[red]✗ {e}[/red]") - - # Seed district datasets - districts_dir = NEVADA_DATA_DIR / "districts" - if districts_dir.exists(): - console.print("\n [bold]Nevada Congressional District Datasets:[/bold]") - for h5_file in sorted(districts_dir.glob("*.h5")): - name = h5_file.stem # e.g., "NV-01_year_2024" - year = int(name.split("_")[-1]) - district = name.split("_")[0] # e.g., "NV-01" - - # Check if already exists - existing = session.exec( - select(Dataset).where(Dataset.name == name) - ).first() - - if existing: - console.print(f" [yellow]⏭[/yellow] {name} (already exists)") - continue - - # Upload to storage - console.print(f" Uploading {name}...", end=" ") - try: - object_name = upload_dataset_for_seeding(str(h5_file)) - - # Create database record - db_dataset = Dataset( - name=name, - description=f"{district} congressional district dataset for year {year}", - filepath=object_name, - year=year, - tax_benefit_model_id=us_model.id, - ) - session.add(db_dataset) - session.commit() - console.print("[green]✓[/green]") - except Exception as e: - console.print(f"[red]✗ {e}[/red]") - - console.print("\n[bold green]✓ Nevada datasets seeded successfully![/bold green]") - - -if __name__ == "__main__": - main() diff --git a/scripts/test_economy_simulation.py b/scripts/test_economy_simulation.py deleted file mode 100644 index 3845fc4..0000000 --- a/scripts/test_economy_simulation.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Test economy-wide simulation following the exact flow from modal_app.py. - -This script mimics the economy-wide simulation code path as closely as possible -to verify whether policy reforms are being applied correctly. -""" - -import tempfile -from datetime import datetime -from pathlib import Path - -import pandas as pd -from microdf import MicroDataFrame - -# Import exactly as modal_app.py does -from policyengine.core import Simulation as PESimulation -from policyengine.core.policy import ParameterValue as PEParameterValue -from policyengine.core.policy import Policy as PEPolicy -from policyengine.tax_benefit_models.us import us_latest -from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset, USYearData - - -def create_test_dataset(year: int) -> PolicyEngineUSDataset: - """Create a small test dataset similar to what would be loaded from storage. - - Uses the same structure as economy-wide datasets but with just a few households. - """ - # Create 3 test households with different income levels - # Each household has 2 adults + 2 children (to test CTC) - n_households = 3 - n_people = n_households * 4 # 4 people per household - - # Person data - person_data = { - "person_id": list(range(n_people)), - "person_household_id": [i // 4 for i in range(n_people)], - "person_marital_unit_id": [], - "person_family_id": [i // 4 for i in range(n_people)], - "person_spm_unit_id": [i // 4 for i in range(n_people)], - "person_tax_unit_id": [i // 4 for i in range(n_people)], - "person_weight": [1000.0] * n_people, # Weight for population scaling - "age": [], - "employment_income": [], - } - - # Build person details - marital_unit_counter = 0 - for hh in range(n_households): - base_income = 10000 + (hh * 20000) # 10k, 30k, 50k - # Adult 1 - person_data["age"].append(35) - person_data["employment_income"].append(base_income) - person_data["person_marital_unit_id"].append(marital_unit_counter) - # Adult 2 - person_data["age"].append(33) - person_data["employment_income"].append(0) - person_data["person_marital_unit_id"].append(marital_unit_counter) - marital_unit_counter += 1 - # Child 1 - person_data["age"].append(5) - person_data["employment_income"].append(0) - person_data["person_marital_unit_id"].append(marital_unit_counter) - marital_unit_counter += 1 - # Child 2 - person_data["age"].append(3) - person_data["employment_income"].append(0) - person_data["person_marital_unit_id"].append(marital_unit_counter) - marital_unit_counter += 1 - - n_marital_units = marital_unit_counter - - # Entity data - household_data = { - "household_id": list(range(n_households)), - "household_weight": [1000.0] * n_households, - "state_fips": [48] * n_households, # Texas - } - - marital_unit_data = { - "marital_unit_id": list(range(n_marital_units)), - "marital_unit_weight": [1000.0] * n_marital_units, - } - - family_data = { - "family_id": list(range(n_households)), - "family_weight": [1000.0] * n_households, - } - - spm_unit_data = { - "spm_unit_id": list(range(n_households)), - "spm_unit_weight": [1000.0] * n_households, - } - - tax_unit_data = { - "tax_unit_id": list(range(n_households)), - "tax_unit_weight": [1000.0] * n_households, - } - - # Create MicroDataFrames (same as economy datasets) - person_df = MicroDataFrame(pd.DataFrame(person_data), weights="person_weight") - household_df = MicroDataFrame(pd.DataFrame(household_data), weights="household_weight") - marital_unit_df = MicroDataFrame(pd.DataFrame(marital_unit_data), weights="marital_unit_weight") - family_df = MicroDataFrame(pd.DataFrame(family_data), weights="family_weight") - spm_unit_df = MicroDataFrame(pd.DataFrame(spm_unit_data), weights="spm_unit_weight") - tax_unit_df = MicroDataFrame(pd.DataFrame(tax_unit_data), weights="tax_unit_weight") - - # Create dataset file - tmpdir = tempfile.mkdtemp() - filepath = str(Path(tmpdir) / "test_economy.h5") - - return PolicyEngineUSDataset( - name="Test Economy Dataset", - description="Small test dataset for economy simulation", - filepath=filepath, - year=year, - data=USYearData( - person=person_df, - household=household_df, - marital_unit=marital_unit_df, - family=family_df, - spm_unit=spm_unit_df, - tax_unit=tax_unit_df, - ), - ) - - -def create_policy_like_modal_app(model_version) -> PEPolicy: - """Create a policy exactly like _get_pe_policy_us does in modal_app.py. - - This mimics the exact flow: - 1. Look up parameter by name from model_version.parameters - 2. Create PEParameterValue with the parameter, value, start_date, end_date - 3. Create PEPolicy with the parameter values - """ - param_lookup = {p.name: p for p in model_version.parameters} - - # This is exactly what _get_pe_policy_us does - pe_param = param_lookup.get("gov.irs.credits.ctc.refundable.fully_refundable") - if not pe_param: - raise ValueError("Parameter not found!") - - pe_pv = PEParameterValue( - parameter=pe_param, - value=True, # Make CTC fully refundable - start_date=datetime(2024, 1, 1), - end_date=None, - ) - - return PEPolicy( - name="CTC Fully Refundable", - description="Makes CTC fully refundable", - parameter_values=[pe_pv], - ) - - -def run_economy_simulation(dataset: PolicyEngineUSDataset, policy: PEPolicy | None, label: str) -> dict: - """Run an economy simulation exactly like modal_app.py does. - - This follows the exact flow from simulate_economy_us: - 1. Create PESimulation with dataset, model version, policy, dynamic - 2. Call pe_sim.ensure() (which calls run() internally) - 3. Access output via pe_sim.output_dataset - """ - print(f"\n=== {label} ===") - print(f" Policy: {policy.name if policy else 'None (baseline)'}") - if policy: - print(f" Policy parameter_values: {len(policy.parameter_values)}") - for pv in policy.parameter_values: - print(f" - {pv.parameter.name}: {pv.value} (start: {pv.start_date})") - - pe_model_version = us_latest - - # Create and run simulation - EXACTLY like modal_app.py lines 1006-1012 - pe_sim = PESimulation( - dataset=dataset, - tax_benefit_model_version=pe_model_version, - policy=policy, - dynamic=None, - ) - pe_sim.ensure() - - # Extract results from output dataset - output_data = pe_sim.output_dataset.data - - # Sum up key metrics across all tax units (weighted) - tax_unit_df = pd.DataFrame(output_data.tax_unit) - - # Get the variables we care about - total_ctc = 0 - total_income_tax = 0 - total_eitc = 0 - - for var in ["ctc", "income_tax", "eitc"]: - if var in tax_unit_df.columns: - # Weighted sum - weights = tax_unit_df.get("tax_unit_weight", pd.Series([1.0] * len(tax_unit_df))) - if var == "ctc": - total_ctc = (tax_unit_df[var] * weights).sum() - elif var == "income_tax": - total_income_tax = (tax_unit_df[var] * weights).sum() - elif var == "eitc": - total_eitc = (tax_unit_df[var] * weights).sum() - - print(f" Results (weighted totals across {len(tax_unit_df)} tax units):") - print(f" Total CTC: ${total_ctc:,.0f}") - print(f" Total Income Tax: ${total_income_tax:,.0f}") - print(f" Total EITC: ${total_eitc:,.0f}") - - # Also show per-household breakdown - print(f" Per tax unit breakdown:") - for i in range(len(tax_unit_df)): - ctc = tax_unit_df["ctc"].iloc[i] if "ctc" in tax_unit_df.columns else 0 - income_tax = tax_unit_df["income_tax"].iloc[i] if "income_tax" in tax_unit_df.columns else 0 - print(f" Tax Unit {i}: CTC=${ctc:,.0f}, Income Tax=${income_tax:,.0f}") - - return { - "total_ctc": total_ctc, - "total_income_tax": total_income_tax, - "total_eitc": total_eitc, - "tax_unit_df": tax_unit_df, - } - - -def main(): - print("=" * 60) - print("ECONOMY-WIDE SIMULATION TEST") - print("Following the exact code path from modal_app.py") - print("=" * 60) - - year = 2024 - - # Create test dataset (same for both simulations) - print("\nCreating test dataset...") - - # Run baseline simulation - baseline_dataset = create_test_dataset(year) - baseline_results = run_economy_simulation(baseline_dataset, None, "BASELINE (no policy)") - - # Create policy exactly like modal_app.py does - policy = create_policy_like_modal_app(us_latest) - - # Run reform simulation - reform_dataset = create_test_dataset(year) - reform_results = run_economy_simulation(reform_dataset, policy, "REFORM (CTC fully refundable)") - - # Compare results - print("\n" + "=" * 60) - print("COMPARISON") - print("=" * 60) - - ctc_diff = reform_results["total_ctc"] - baseline_results["total_ctc"] - tax_diff = reform_results["total_income_tax"] - baseline_results["total_income_tax"] - - print(f"\nTotal CTC:") - print(f" Baseline: ${baseline_results['total_ctc']:,.0f}") - print(f" Reform: ${reform_results['total_ctc']:,.0f}") - print(f" Change: ${ctc_diff:,.0f}") - - print(f"\nTotal Income Tax:") - print(f" Baseline: ${baseline_results['total_income_tax']:,.0f}") - print(f" Reform: ${reform_results['total_income_tax']:,.0f}") - print(f" Change: ${tax_diff:,.0f}") - - # Verdict - print("\n" + "=" * 60) - print("VERDICT") - print("=" * 60) - - if baseline_results["total_income_tax"] == reform_results["total_income_tax"]: - print("\n❌ BUG CONFIRMED: Results are IDENTICAL!") - print(" The policy reform is NOT being applied to economy simulations.") - else: - print("\n✓ NO BUG: Results differ as expected!") - print(f" The fully refundable CTC reform changed income tax by ${tax_diff:,.0f}") - - -if __name__ == "__main__": - main() diff --git a/scripts/test_household_impact.py b/scripts/test_household_impact.py deleted file mode 100644 index 81c85b0..0000000 --- a/scripts/test_household_impact.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Test household impact analysis end-to-end. - -This script tests the async household impact analysis workflow: -1. Create a stored household -2. Run household impact analysis (returns immediately with report_id) -3. Poll until completed -4. Verify results - -Usage: - uv run python scripts/test_household_impact.py -""" - -import sys -import time - -import requests - -BASE_URL = "http://127.0.0.1:8000" - - -def main(): - print("=" * 60) - print("Testing Household Impact Analysis (Async)") - print("=" * 60) - - # Step 1: Create a US household - print("\n1. Creating US household...") - household_data = { - "tax_benefit_model_name": "policyengine_us", - "year": 2024, - "label": "Test household for impact analysis", - "people": [ - { - "age": 35, - "employment_income": 50000, - } - ], - "tax_unit": {}, - "family": {}, - "spm_unit": {}, - "marital_unit": {}, - "household": {"state_code": "NV"}, - } - - resp = requests.post(f"{BASE_URL}/households/", json=household_data) - if resp.status_code != 201: - print(f" FAILED: {resp.status_code} - {resp.text}") - sys.exit(1) - - household = resp.json() - household_id = household["id"] - print(f" Created household: {household_id}") - - # Step 2: Run household impact analysis - print("\n2. Starting household impact analysis...") - impact_request = { - "household_id": household_id, - "policy_id": None, # Single run under current law - } - - resp = requests.post(f"{BASE_URL}/analysis/household-impact", json=impact_request) - if resp.status_code != 200: - print(f" FAILED: {resp.status_code} - {resp.text}") - sys.exit(1) - - result = resp.json() - report_id = result["report_id"] - status = result["status"] - print(f" Report ID: {report_id}") - print(f" Initial status: {status}") - - # Step 3: Poll until completed - print("\n3. Polling for results...") - max_attempts = 30 - for attempt in range(max_attempts): - resp = requests.get(f"{BASE_URL}/analysis/household-impact/{report_id}") - if resp.status_code != 200: - print(f" FAILED: {resp.status_code} - {resp.text}") - sys.exit(1) - - result = resp.json() - status = result["status"].upper() # Normalize to uppercase - print(f" Attempt {attempt + 1}: status={status}") - - if status == "COMPLETED": - break - elif status == "FAILED": - print(f" FAILED: {result.get('error_message', 'Unknown error')}") - sys.exit(1) - - time.sleep(0.5) - else: - print(f" FAILED: Timed out after {max_attempts} attempts") - sys.exit(1) - - # Step 4: Verify results - print("\n4. Verifying results...") - baseline_result = result.get("baseline_result") - if not baseline_result: - print(" FAILED: No baseline result") - sys.exit(1) - - print(f" Baseline result keys: {list(baseline_result.keys())}") - - # Check for expected entity types - expected_entities = ["person", "tax_unit", "spm_unit", "family", "marital_unit", "household"] - for entity in expected_entities: - if entity in baseline_result: - print(f" ✓ {entity}: {len(baseline_result[entity])} entities") - else: - print(f" ✗ {entity}: missing") - - # Look for net_income in person output - if "person" in baseline_result and baseline_result["person"]: - person = baseline_result["person"][0] - if "household_net_income" in person: - print(f" household_net_income: ${person['household_net_income']:,.2f}") - elif "spm_unit_net_income" in person: - print(f" spm_unit_net_income: ${person['spm_unit_net_income']:,.2f}") - - # Step 5: Cleanup - delete household - print("\n5. Cleaning up...") - resp = requests.delete(f"{BASE_URL}/households/{household_id}") - if resp.status_code == 204: - print(f" Deleted household: {household_id}") - else: - print(f" Warning: Failed to delete household: {resp.status_code}") - - print("\n" + "=" * 60) - print("SUCCESS: Household impact analysis working correctly!") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/scripts/test_household_scenarios.py b/scripts/test_household_scenarios.py deleted file mode 100644 index fb418a4..0000000 --- a/scripts/test_household_scenarios.py +++ /dev/null @@ -1,344 +0,0 @@ -"""Test household calculation scenarios. - -Tests: -1. US California household under current law -2. Scotland household under current law -3. US household: current law vs CTC fully refundable reform -""" - -import sys -import time -import requests - -BASE_URL = "http://127.0.0.1:8000" - - -def poll_for_completion(report_id: str, max_attempts: int = 60) -> dict: - """Poll until report is completed or failed.""" - for attempt in range(max_attempts): - resp = requests.get(f"{BASE_URL}/analysis/household-impact/{report_id}") - if resp.status_code != 200: - raise Exception(f"Failed to get report: {resp.status_code} - {resp.text}") - - result = resp.json() - status = result["status"].upper() - - if status == "COMPLETED": - return result - elif status == "FAILED": - raise Exception(f"Report failed: {result.get('error_message', 'Unknown error')}") - - time.sleep(0.5) - - raise Exception(f"Timed out after {max_attempts} attempts") - - -def print_household_summary(result: dict, label: str): - """Print summary of household calculation result.""" - print(f"\n {label}:") - - baseline = result.get("baseline_result", {}) - reform = result.get("reform_result", {}) - - # Get key metrics from person/household - if "person" in baseline and baseline["person"]: - person = baseline["person"][0] - if "household_net_income" in person: - baseline_income = person["household_net_income"] - print(f" Baseline net income: ${baseline_income:,.2f}") - - if reform and "person" in reform and reform["person"]: - reform_income = reform["person"][0].get("household_net_income", 0) - print(f" Reform net income: ${reform_income:,.2f}") - print(f" Difference: ${reform_income - baseline_income:,.2f}") - - # Show some tax/benefit info if available - for key in ["income_tax", "federal_income_tax", "state_income_tax", "ctc", "refundable_ctc"]: - if key in person: - print(f" {key}: ${person[key]:,.2f}") - - -def test_us_california(): - """Test 1: US California household under current law.""" - print("\n" + "=" * 60) - print("TEST 1: US California Household - Current Law") - print("=" * 60) - - # Create California household - household_data = { - "tax_benefit_model_name": "policyengine_us", - "year": 2024, - "label": "California test household", - "people": [ - {"age": 35, "employment_income": 75000}, - {"age": 33, "employment_income": 45000}, - {"age": 8}, # Child - ], - "tax_unit": {}, - "family": {}, - "spm_unit": {}, - "marital_unit": {}, - "household": {"state_code": "CA"}, - } - - print("\n Creating household...") - resp = requests.post(f"{BASE_URL}/households/", json=household_data) - if resp.status_code != 201: - print(f" FAILED: {resp.status_code} - {resp.text}") - return None - - household = resp.json() - household_id = household["id"] - print(f" Household ID: {household_id}") - - # Run analysis under current law (no policy_id) - print(" Running analysis...") - resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ - "household_id": household_id, - "policy_id": None, - }) - - if resp.status_code != 200: - print(f" FAILED: {resp.status_code} - {resp.text}") - return household_id - - report_id = resp.json()["report_id"] - print(f" Report ID: {report_id}") - - # Poll for results - try: - result = poll_for_completion(report_id) - print(" Status: COMPLETED") - print_household_summary(result, "Results") - except Exception as e: - print(f" FAILED: {e}") - - return household_id - - -def test_scotland(): - """Test 2: Scotland household under current law.""" - print("\n" + "=" * 60) - print("TEST 2: Scotland Household - Current Law") - print("=" * 60) - - # Create Scotland household - household_data = { - "tax_benefit_model_name": "policyengine_uk", - "year": 2024, - "label": "Scotland test household", - "people": [ - {"age": 40, "employment_income": 45000}, - ], - "benunit": {}, - "household": {"region": "SCOTLAND"}, - } - - print("\n Creating household...") - resp = requests.post(f"{BASE_URL}/households/", json=household_data) - if resp.status_code != 201: - print(f" FAILED: {resp.status_code} - {resp.text}") - return None - - household = resp.json() - household_id = household["id"] - print(f" Household ID: {household_id}") - - # Run analysis under current law - print(" Running analysis...") - resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ - "household_id": household_id, - "policy_id": None, - }) - - if resp.status_code != 200: - print(f" FAILED: {resp.status_code} - {resp.text}") - return household_id - - report_id = resp.json()["report_id"] - print(f" Report ID: {report_id}") - - # Poll for results - try: - result = poll_for_completion(report_id) - print(" Status: COMPLETED") - print_household_summary(result, "Results") - except Exception as e: - print(f" FAILED: {e}") - - return household_id - - -def test_us_ctc_reform(): - """Test 3: US household - current law vs CTC fully refundable.""" - print("\n" + "=" * 60) - print("TEST 3: US Household - Current Law vs CTC Fully Refundable") - print("=" * 60) - - # First, find the CTC refundability parameter - print("\n Finding CTC refundability parameter...") - resp = requests.get(f"{BASE_URL}/parameters", params={"search": "ctc", "limit": 50}) - if resp.status_code != 200: - print(f" FAILED to search parameters: {resp.status_code}") - return None, None - - params = resp.json() - ctc_param = None - for p in params: - # Look for the refundable portion parameter - if "refundable" in p["name"].lower() and "ctc" in p["name"].lower(): - print(f" Found: {p['name']} (label: {p.get('label')})") - ctc_param = p - break - - if not ctc_param: - # Try searching for child tax credit parameters - print(" Searching for child_tax_credit parameters...") - resp = requests.get(f"{BASE_URL}/parameters", params={"search": "child_tax_credit", "limit": 50}) - params = resp.json() - for p in params: - print(f" - {p['name']}") - if "refundable" in p["name"].lower(): - ctc_param = p - break - - if not ctc_param: - print(" Could not find CTC refundability parameter") - print(" Continuing with household creation anyway...") - - # Create household with children (needed for CTC) - household_data = { - "tax_benefit_model_name": "policyengine_us", - "year": 2024, - "label": "CTC test household", - "people": [ - {"age": 35, "employment_income": 30000}, # Lower income to see CTC effect - {"age": 33, "employment_income": 0}, - {"age": 5}, # Child 1 - {"age": 3}, # Child 2 - ], - "tax_unit": {}, - "family": {}, - "spm_unit": {}, - "marital_unit": {}, - "household": {"state_code": "TX"}, # Texas - no state income tax - } - - print("\n Creating household...") - resp = requests.post(f"{BASE_URL}/households/", json=household_data) - if resp.status_code != 201: - print(f" FAILED: {resp.status_code} - {resp.text}") - return None, None - - household = resp.json() - household_id = household["id"] - print(f" Household ID: {household_id}") - - # Create a policy that makes CTC fully refundable - policy_id = None - if ctc_param: - print("\n Creating CTC fully refundable policy...") - policy_data = { - "name": "CTC Fully Refundable", - "description": "Makes the Child Tax Credit fully refundable", - } - resp = requests.post(f"{BASE_URL}/policies/", json=policy_data) - if resp.status_code == 201: - policy = resp.json() - policy_id = policy["id"] - print(f" Policy ID: {policy_id}") - - # Add parameter value to make CTC fully refundable - # The parameter should set refundable portion to 100% or max amount - pv_data = { - "parameter_id": ctc_param["id"], - "value_json": 1.0, # 100% refundable - "start_date": "2024-01-01T00:00:00Z", - "end_date": None, - "policy_id": policy_id, - } - resp = requests.post(f"{BASE_URL}/parameter-values/", json=pv_data) - if resp.status_code == 201: - print(" Added parameter value for full refundability") - else: - print(f" Warning: Failed to add parameter value: {resp.status_code} - {resp.text}") - else: - print(f" Warning: Failed to create policy: {resp.status_code}") - - # Run analysis with reform policy - print("\n Running analysis (baseline vs reform)...") - resp = requests.post(f"{BASE_URL}/analysis/household-impact", json={ - "household_id": household_id, - "policy_id": policy_id, - }) - - if resp.status_code != 200: - print(f" FAILED: {resp.status_code} - {resp.text}") - return household_id, policy_id - - report_id = resp.json()["report_id"] - print(f" Report ID: {report_id}") - - # Poll for results - try: - result = poll_for_completion(report_id) - print(" Status: COMPLETED") - print_household_summary(result, "Results") - except Exception as e: - print(f" FAILED: {e}") - - return household_id, policy_id - - -def main(): - print("=" * 60) - print("HOUSEHOLD CALCULATION SCENARIO TESTS") - print("=" * 60) - - # Track created resources for cleanup - households = [] - policies = [] - - # Test 1: US California - hh_id = test_us_california() - if hh_id: - households.append(hh_id) - - # Test 2: Scotland - hh_id = test_scotland() - if hh_id: - households.append(hh_id) - - # Test 3: CTC Reform - hh_id, policy_id = test_us_ctc_reform() - if hh_id: - households.append(hh_id) - if policy_id: - policies.append(policy_id) - - # Cleanup - print("\n" + "=" * 60) - print("CLEANUP") - print("=" * 60) - - for hh_id in households: - resp = requests.delete(f"{BASE_URL}/households/{hh_id}") - if resp.status_code == 204: - print(f" Deleted household: {hh_id}") - else: - print(f" Warning: Failed to delete household {hh_id}: {resp.status_code}") - - for policy_id in policies: - resp = requests.delete(f"{BASE_URL}/policies/{policy_id}") - if resp.status_code == 204: - print(f" Deleted policy: {policy_id}") - else: - print(f" Warning: Failed to delete policy {policy_id}: {resp.status_code}") - - print("\n" + "=" * 60) - print("TESTS COMPLETE") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql b/supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql deleted file mode 100644 index c1713d5..0000000 --- a/supabase/migrations_archived/20251229000000_add_parameter_values_indexes.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Add indexes to parameter_values table for query optimization --- This migration improves query performance for filtering by parameter_id and policy_id - --- Composite index for the most common query pattern (filtering by both) -CREATE INDEX IF NOT EXISTS idx_parameter_values_parameter_policy -ON parameter_values(parameter_id, policy_id); - --- Single index on policy_id for filtering by policy alone -CREATE INDEX IF NOT EXISTS idx_parameter_values_policy -ON parameter_values(policy_id); - --- Partial index for baseline values (policy_id IS NULL) --- This optimizes the common "get current law values" query -CREATE INDEX IF NOT EXISTS idx_parameter_values_baseline -ON parameter_values(parameter_id) -WHERE policy_id IS NULL; diff --git a/supabase/migrations_archived/20260103000000_add_poverty_inequality.sql b/supabase/migrations_archived/20260103000000_add_poverty_inequality.sql deleted file mode 100644 index f315d93..0000000 --- a/supabase/migrations_archived/20260103000000_add_poverty_inequality.sql +++ /dev/null @@ -1,33 +0,0 @@ --- Add poverty and inequality tables for economic analysis - -CREATE TABLE IF NOT EXISTS poverty ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - simulation_id UUID NOT NULL REFERENCES simulations(id) ON DELETE CASCADE, - report_id UUID REFERENCES reports(id) ON DELETE CASCADE, - poverty_type VARCHAR NOT NULL, - entity VARCHAR NOT NULL DEFAULT 'person', - filter_variable VARCHAR, - headcount FLOAT, - total_population FLOAT, - rate FLOAT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE IF NOT EXISTS inequality ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - simulation_id UUID NOT NULL REFERENCES simulations(id) ON DELETE CASCADE, - report_id UUID REFERENCES reports(id) ON DELETE CASCADE, - income_variable VARCHAR NOT NULL, - entity VARCHAR NOT NULL DEFAULT 'household', - gini FLOAT, - top_10_share FLOAT, - top_1_share FLOAT, - bottom_50_share FLOAT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Indexes for efficient querying -CREATE INDEX IF NOT EXISTS idx_poverty_simulation_id ON poverty(simulation_id); -CREATE INDEX IF NOT EXISTS idx_poverty_report_id ON poverty(report_id); -CREATE INDEX IF NOT EXISTS idx_inequality_simulation_id ON inequality(simulation_id); -CREATE INDEX IF NOT EXISTS idx_inequality_report_id ON inequality(report_id); diff --git a/supabase/migrations_archived/20260111000000_add_aggregate_status.sql b/supabase/migrations_archived/20260111000000_add_aggregate_status.sql deleted file mode 100644 index b190620..0000000 --- a/supabase/migrations_archived/20260111000000_add_aggregate_status.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Add status and error_message columns to aggregates table -ALTER TABLE aggregates -ADD COLUMN IF NOT EXISTS status VARCHAR(20) DEFAULT 'pending', -ADD COLUMN IF NOT EXISTS error_message TEXT; - --- Add status and error_message columns to change_aggregates table -ALTER TABLE change_aggregates -ADD COLUMN IF NOT EXISTS status VARCHAR(20) DEFAULT 'pending', -ADD COLUMN IF NOT EXISTS error_message TEXT; - --- Create indexes for status filtering -CREATE INDEX IF NOT EXISTS idx_aggregates_status ON aggregates(status); -CREATE INDEX IF NOT EXISTS idx_change_aggregates_status ON change_aggregates(status); diff --git a/supabase/migrations_archived/20260203000000_create_households.sql b/supabase/migrations_archived/20260203000000_create_households.sql deleted file mode 100644 index cc1907f..0000000 --- a/supabase/migrations_archived/20260203000000_create_households.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Create stored households table for persisting household definitions. - -CREATE TABLE IF NOT EXISTS households ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tax_benefit_model_name TEXT NOT NULL, - year INTEGER NOT NULL, - label TEXT, - household_data JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX idx_households_model_name ON households (tax_benefit_model_name); -CREATE INDEX idx_households_year ON households (year); diff --git a/supabase/migrations_archived/20260203000001_create_user_household_associations.sql b/supabase/migrations_archived/20260203000001_create_user_household_associations.sql deleted file mode 100644 index 3fdcb03..0000000 --- a/supabase/migrations_archived/20260203000001_create_user_household_associations.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Create user-household associations table for linking users to saved households. - -CREATE TABLE IF NOT EXISTS user_household_associations ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - household_id UUID NOT NULL REFERENCES households(id) ON DELETE CASCADE, - country_id TEXT NOT NULL, - label TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX idx_user_household_assoc_user ON user_household_associations (user_id); -CREATE INDEX idx_user_household_assoc_household ON user_household_associations (household_id); diff --git a/supabase/migrations_archived/20260203000002_simulation_household_support.sql b/supabase/migrations_archived/20260203000002_simulation_household_support.sql deleted file mode 100644 index 6813f07..0000000 --- a/supabase/migrations_archived/20260203000002_simulation_household_support.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Add simulation_type as TEXT (SQLModel enum maps to text) -ALTER TABLE simulations ADD COLUMN simulation_type TEXT NOT NULL DEFAULT 'economy'; - --- Make dataset_id nullable (was required) -ALTER TABLE simulations ALTER COLUMN dataset_id DROP NOT NULL; - --- Add household support columns -ALTER TABLE simulations ADD COLUMN household_id UUID REFERENCES households(id); -ALTER TABLE simulations ADD COLUMN household_result JSONB; - --- Indexes -CREATE INDEX idx_simulations_household ON simulations (household_id); -CREATE INDEX idx_simulations_type ON simulations (simulation_type); - --- Add report_type to reports -ALTER TABLE reports ADD COLUMN report_type TEXT; From 01e18069a5430a8b4a5868373035f4df9b7aa55e Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 10 Feb 2026 21:01:26 +0100 Subject: [PATCH 20/87] feat: Add regions support for geographic analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Region SQLModel with filtering fields (code, label, region_type, requires_filter, filter_field, filter_value, dataset_id, etc.) - Add Alembic migration for regions table - Add GET /regions/ endpoint with filters by model and region type - Add GET /regions/{region_id} and GET /regions/by-code/{code} endpoints - Add region parameter to analysis endpoint with dataset/region resolution 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .../versions/20260210_add_regions_table.py | 63 ++++++++++ src/policyengine_api/api/__init__.py | 2 + src/policyengine_api/api/analysis.py | 112 ++++++++++++++++-- src/policyengine_api/api/regions.py | 102 ++++++++++++++++ src/policyengine_api/models/__init__.py | 4 + src/policyengine_api/models/region.py | 60 ++++++++++ 6 files changed, 330 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/20260210_add_regions_table.py create mode 100644 src/policyengine_api/api/regions.py create mode 100644 src/policyengine_api/models/region.py diff --git a/alembic/versions/20260210_add_regions_table.py b/alembic/versions/20260210_add_regions_table.py new file mode 100644 index 0000000..effeab2 --- /dev/null +++ b/alembic/versions/20260210_add_regions_table.py @@ -0,0 +1,63 @@ +"""add regions table + +Revision ID: a1b2c3d4e5f6 +Revises: f419b5f4acba +Create Date: 2026-02-10 12:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes + + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, Sequence[str], None] = "f419b5f4acba" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create regions table.""" + op.create_table( + "regions", + sa.Column("code", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("region_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("requires_filter", sa.Boolean(), nullable=False, default=False), + sa.Column("filter_field", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("filter_value", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("parent_code", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("state_code", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("state_name", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("dataset_id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + # Create unique constraint on (code, tax_benefit_model_id) + op.create_index( + "ix_regions_code_model", + "regions", + ["code", "tax_benefit_model_id"], + unique=True, + ) + + +def downgrade() -> None: + """Drop regions table.""" + op.drop_index("ix_regions_code_model", table_name="regions") + op.drop_table("regions") diff --git a/src/policyengine_api/api/__init__.py b/src/policyengine_api/api/__init__.py index c3e0353..f135b14 100644 --- a/src/policyengine_api/api/__init__.py +++ b/src/policyengine_api/api/__init__.py @@ -15,6 +15,7 @@ parameter_values, parameters, policies, + regions, simulations, tax_benefit_model_versions, tax_benefit_models, @@ -26,6 +27,7 @@ api_router.include_router(datasets.router) api_router.include_router(policies.router) +api_router.include_router(regions.router) api_router.include_router(simulations.router) api_router.include_router(outputs.router) api_router.include_router(variables.router) diff --git a/src/policyengine_api/api/analysis.py b/src/policyengine_api/api/analysis.py index 10e6fc5..101669b 100644 --- a/src/policyengine_api/api/analysis.py +++ b/src/policyengine_api/api/analysis.py @@ -31,6 +31,7 @@ DecileImpactRead, ProgramStatistics, ProgramStatisticsRead, + Region, Report, ReportStatus, Simulation, @@ -68,19 +69,31 @@ def _safe_float(value: float | None) -> float | None: class EconomicImpactRequest(BaseModel): """Request body for economic impact analysis. - Example: + Example with dataset_id: { "tax_benefit_model_name": "policyengine_uk", "dataset_id": "uuid-from-datasets-endpoint", "policy_id": "uuid-of-reform-policy" } + + Example with region: + { + "tax_benefit_model_name": "policyengine_us", + "region": "state/ca", + "policy_id": "uuid-of-reform-policy" + } """ tax_benefit_model_name: Literal["policyengine_uk", "policyengine_us"] = Field( description="Which country model to use" ) - dataset_id: UUID = Field( - description="Dataset ID from /datasets endpoint containing population microdata" + dataset_id: UUID | None = Field( + default=None, + description="Dataset ID from /datasets endpoint. Either dataset_id or region must be provided.", + ) + region: str | None = Field( + default=None, + description="Region code (e.g., 'state/ca', 'us'). Either dataset_id or region must be provided.", ) policy_id: UUID | None = Field( default=None, @@ -99,6 +112,17 @@ class SimulationInfo(BaseModel): error_message: str | None = None +class RegionInfo(BaseModel): + """Region information used in analysis.""" + + code: str + label: str + region_type: str + requires_filter: bool + filter_field: str | None = None + filter_value: str | None = None + + class EconomicImpactResponse(BaseModel): """Response from economic impact analysis.""" @@ -106,6 +130,7 @@ class EconomicImpactResponse(BaseModel): status: ReportStatus baseline_simulation: SimulationInfo reform_simulation: SimulationInfo + region: RegionInfo | None = None error_message: str | None = None decile_impacts: list[DecileImpactRead] | None = None program_statistics: list[ProgramStatisticsRead] | None = None @@ -235,6 +260,7 @@ def _build_response( baseline_sim: Simulation, reform_sim: Simulation, session: Session, + region: Region | None = None, ) -> EconomicImpactResponse: """Build response from report and simulations.""" decile_impacts = None @@ -292,6 +318,17 @@ def _build_response( for s in stats ] + region_info = None + if region: + region_info = RegionInfo( + code=region.code, + label=region.label, + region_type=region.region_type, + requires_filter=region.requires_filter, + filter_field=region.filter_field, + filter_value=region.filter_value, + ) + return EconomicImpactResponse( report_id=report.id, status=report.status, @@ -305,6 +342,7 @@ def _build_response( status=reform_sim.status, error_message=reform_sim.error_message, ), + region=region_info, error_message=report.error_message, decile_impacts=decile_impacts, program_statistics=program_statistics, @@ -571,6 +609,54 @@ def _trigger_economy_comparison( fn.spawn(job_id=job_id, traceparent=traceparent) +def _resolve_dataset_and_region( + request: EconomicImpactRequest, + session: Session, +) -> tuple[Dataset, Region | None]: + """Resolve dataset from request, optionally via region lookup. + + Returns: + Tuple of (dataset, region) where region is None if dataset_id was provided directly. + """ + if request.region: + # Look up region by code + model_name = request.tax_benefit_model_name.replace("_", "-") + region = session.exec( + select(Region) + .join(TaxBenefitModel) + .where(Region.code == request.region) + .where(TaxBenefitModel.name == model_name) + ).first() + + if not region: + raise HTTPException( + status_code=404, + detail=f"Region '{request.region}' not found for model {model_name}", + ) + + dataset = session.get(Dataset, region.dataset_id) + if not dataset: + raise HTTPException( + status_code=404, + detail=f"Dataset for region '{request.region}' not found", + ) + return dataset, region + + elif request.dataset_id: + dataset = session.get(Dataset, request.dataset_id) + if not dataset: + raise HTTPException( + status_code=404, detail=f"Dataset {request.dataset_id} not found" + ) + return dataset, None + + else: + raise HTTPException( + status_code=400, + detail="Either dataset_id or region must be provided", + ) + + @router.post("/economic-impact", response_model=EconomicImpactResponse) def economic_impact( request: EconomicImpactRequest, @@ -584,25 +670,25 @@ def economic_impact( Results include decile impacts (income changes by income group) and program statistics (budgetary effects of tax/benefit programs). + + You can specify the geographic scope either by: + - dataset_id: Direct dataset reference + - region: Region code (e.g., "state/ca", "us") which resolves to a dataset """ - # Validate dataset exists - dataset = session.get(Dataset, request.dataset_id) - if not dataset: - raise HTTPException( - status_code=404, detail=f"Dataset {request.dataset_id} not found" - ) + # Resolve dataset (and optionally region) + dataset, region = _resolve_dataset_and_region(request, session) # Get model version model_version = _get_model_version(request.tax_benefit_model_name, session) - # Get or create simulations + # Get or create simulations using the resolved dataset baseline_sim = _get_or_create_simulation( simulation_type=SimulationType.ECONOMY, model_version_id=model_version.id, policy_id=None, dynamic_id=request.dynamic_id, session=session, - dataset_id=request.dataset_id, + dataset_id=dataset.id, ) reform_sim = _get_or_create_simulation( @@ -611,7 +697,7 @@ def economic_impact( policy_id=request.policy_id, dynamic_id=request.dynamic_id, session=session, - dataset_id=request.dataset_id, + dataset_id=dataset.id, ) # Get or create report @@ -630,7 +716,7 @@ def economic_impact( str(report.id), request.tax_benefit_model_name, session ) - return _build_response(report, baseline_sim, reform_sim, session) + return _build_response(report, baseline_sim, reform_sim, session, region) @router.get("/economic-impact/{report_id}", response_model=EconomicImpactResponse) diff --git a/src/policyengine_api/api/regions.py b/src/policyengine_api/api/regions.py new file mode 100644 index 0000000..1d0a34e --- /dev/null +++ b/src/policyengine_api/api/regions.py @@ -0,0 +1,102 @@ +"""Region endpoints for geographic areas used in analysis. + +Regions represent geographic areas from countries down to states, +congressional districts, cities, etc. Each region has an associated +dataset for running simulations. +""" + +from typing import List +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlmodel import Session, select + +from policyengine_api.models import Region, RegionRead, TaxBenefitModel +from policyengine_api.services.database import get_session + +router = APIRouter(prefix="/regions", tags=["regions"]) + + +@router.get("/", response_model=List[RegionRead]) +def list_regions( + tax_benefit_model_id: UUID | None = Query( + None, description="Filter by tax-benefit model ID" + ), + tax_benefit_model_name: str | None = Query( + None, description="Filter by tax-benefit model name (e.g., 'policyengine-us')" + ), + region_type: str | None = Query( + None, + description="Filter by region type (e.g., 'state', 'congressional_district')", + ), + session: Session = Depends(get_session), +): + """List available regions. + + Returns regions that can be used with the /analysis/economic-impact endpoint. + Each region represents a geographic area with an associated dataset. + + Args: + tax_benefit_model_id: Filter by tax-benefit model UUID. + tax_benefit_model_name: Filter by model name (e.g., "policyengine-us"). + region_type: Filter by region type (e.g., "state", "congressional_district"). + """ + query = select(Region) + + if tax_benefit_model_id: + query = query.where(Region.tax_benefit_model_id == tax_benefit_model_id) + elif tax_benefit_model_name: + query = query.join(TaxBenefitModel).where( + TaxBenefitModel.name == tax_benefit_model_name + ) + + if region_type: + query = query.where(Region.region_type == region_type) + + regions = session.exec(query).all() + return regions + + +@router.get("/{region_id}", response_model=RegionRead) +def get_region(region_id: UUID, session: Session = Depends(get_session)): + """Get a specific region by ID.""" + region = session.get(Region, region_id) + if not region: + raise HTTPException(status_code=404, detail="Region not found") + return region + + +@router.get("/by-code/{region_code:path}", response_model=RegionRead) +def get_region_by_code( + region_code: str, + tax_benefit_model_id: UUID | None = Query( + None, + description="Tax-benefit model ID (required if multiple models have same region code)", + ), + tax_benefit_model_name: str | None = Query( + None, description="Tax-benefit model name (e.g., 'policyengine-us')" + ), + session: Session = Depends(get_session), +): + """Get a specific region by code. + + Region codes use a prefix format like "state/ca" or "constituency/Sheffield Central". + + Args: + region_code: The region code (e.g., "state/ca", "us"). + tax_benefit_model_id: Filter by tax-benefit model UUID. + tax_benefit_model_name: Filter by model name. + """ + query = select(Region).where(Region.code == region_code) + + if tax_benefit_model_id: + query = query.where(Region.tax_benefit_model_id == tax_benefit_model_id) + elif tax_benefit_model_name: + query = query.join(TaxBenefitModel).where( + TaxBenefitModel.name == tax_benefit_model_name + ) + + region = session.exec(query).first() + if not region: + raise HTTPException(status_code=404, detail="Region not found") + return region diff --git a/src/policyengine_api/models/__init__.py b/src/policyengine_api/models/__init__.py index c49b457..7361979 100644 --- a/src/policyengine_api/models/__init__.py +++ b/src/policyengine_api/models/__init__.py @@ -30,6 +30,7 @@ from .parameter_value import ParameterValue, ParameterValueCreate, ParameterValueRead from .policy import Policy, PolicyCreate, PolicyRead from .poverty import Poverty, PovertyCreate, PovertyRead +from .region import Region, RegionCreate, RegionRead from .program_statistics import ( ProgramStatistics, ProgramStatisticsCreate, @@ -107,6 +108,9 @@ "Poverty", "PovertyCreate", "PovertyRead", + "Region", + "RegionCreate", + "RegionRead", "ProgramStatistics", "ProgramStatisticsCreate", "ProgramStatisticsRead", diff --git a/src/policyengine_api/models/region.py b/src/policyengine_api/models/region.py new file mode 100644 index 0000000..7c87a00 --- /dev/null +++ b/src/policyengine_api/models/region.py @@ -0,0 +1,60 @@ +"""Region model for geographic areas used in analysis.""" + +from datetime import datetime, timezone +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlmodel import Field, Relationship, SQLModel + +if TYPE_CHECKING: + from .dataset import Dataset + from .tax_benefit_model import TaxBenefitModel + + +class RegionBase(SQLModel): + """Base region fields.""" + + code: str # e.g., "state/ca", "constituency/Sheffield Central" + label: str # e.g., "California", "Sheffield Central" + region_type: str # e.g., "state", "congressional_district", "constituency" + requires_filter: bool = False + filter_field: str | None = None # e.g., "state_code", "place_fips" + filter_value: str | None = None # e.g., "CA", "44000" + parent_code: str | None = None # e.g., "us", "state/ca" + state_code: str | None = None # For US regions + state_name: str | None = None # For US regions + dataset_id: UUID = Field(foreign_key="datasets.id") + tax_benefit_model_id: UUID = Field(foreign_key="tax_benefit_models.id") + + +class Region(RegionBase, table=True): + """Region database model. + + Regions represent geographic areas for analysis, from countries + down to states, congressional districts, cities, etc. + Each region has a dataset (either dedicated or filtered from parent). + """ + + __tablename__ = "regions" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + # Relationships + dataset: "Dataset" = Relationship() + tax_benefit_model: "TaxBenefitModel" = Relationship() + + +class RegionCreate(RegionBase): + """Schema for creating regions.""" + + pass + + +class RegionRead(RegionBase): + """Schema for reading regions.""" + + id: UUID + created_at: datetime + updated_at: datetime From 3751dc6cb926dabc4ffcca0f10c9f9098f6e0005 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 10 Feb 2026 21:07:37 +0100 Subject: [PATCH 21/87] feat: Wire filter_field/filter_value through to policyengine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add filter_field and filter_value to Simulation model - Include filter params in deterministic simulation ID generation - Pass filter params from region to simulation creation - Pass filter params to policyengine.py PESimulation when running 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/policyengine_api/api/analysis.py | 24 +++++++++++++++++++++-- src/policyengine_api/models/simulation.py | 10 ++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/policyengine_api/api/analysis.py b/src/policyengine_api/api/analysis.py index 101669b..17539d7 100644 --- a/src/policyengine_api/api/analysis.py +++ b/src/policyengine_api/api/analysis.py @@ -170,10 +170,12 @@ def _get_deterministic_simulation_id( dynamic_id: UUID | None, dataset_id: UUID | None = None, household_id: UUID | None = None, + filter_field: str | None = None, + filter_value: str | None = None, ) -> UUID: """Generate a deterministic UUID from simulation parameters.""" if simulation_type == SimulationType.ECONOMY: - key = f"economy:{dataset_id}:{model_version_id}:{policy_id}:{dynamic_id}" + key = f"economy:{dataset_id}:{model_version_id}:{policy_id}:{dynamic_id}:{filter_field}:{filter_value}" else: key = f"household:{household_id}:{model_version_id}:{policy_id}:{dynamic_id}" return uuid5(SIMULATION_NAMESPACE, key) @@ -196,6 +198,8 @@ def _get_or_create_simulation( session: Session, dataset_id: UUID | None = None, household_id: UUID | None = None, + filter_field: str | None = None, + filter_value: str | None = None, ) -> Simulation: """Get existing simulation or create a new one.""" sim_id = _get_deterministic_simulation_id( @@ -205,6 +209,8 @@ def _get_or_create_simulation( dynamic_id, dataset_id=dataset_id, household_id=household_id, + filter_field=filter_field, + filter_value=filter_value, ) existing = session.get(Simulation, sim_id) @@ -220,6 +226,8 @@ def _get_or_create_simulation( policy_id=policy_id, dynamic_id=dynamic_id, status=SimulationStatus.PENDING, + filter_field=filter_field, + filter_value=filter_value, ) session.add(simulation) session.commit() @@ -487,12 +495,14 @@ def build_dynamic(dynamic_id): year=dataset.year, ) - # Run simulations + # Run simulations (with optional regional filtering) pe_baseline_sim = PESimulation( dataset=pe_dataset, tax_benefit_model_version=pe_model_version, policy=baseline_policy, dynamic=baseline_dynamic, + filter_field=baseline_sim.filter_field, + filter_value=baseline_sim.filter_value, ) pe_baseline_sim.ensure() @@ -501,6 +511,8 @@ def build_dynamic(dynamic_id): tax_benefit_model_version=pe_model_version, policy=reform_policy, dynamic=reform_dynamic, + filter_field=reform_sim.filter_field, + filter_value=reform_sim.filter_value, ) pe_reform_sim.ensure() @@ -678,6 +690,10 @@ def economic_impact( # Resolve dataset (and optionally region) dataset, region = _resolve_dataset_and_region(request, session) + # Extract filter parameters from region (if present) + filter_field = region.filter_field if region and region.requires_filter else None + filter_value = region.filter_value if region and region.requires_filter else None + # Get model version model_version = _get_model_version(request.tax_benefit_model_name, session) @@ -689,6 +705,8 @@ def economic_impact( dynamic_id=request.dynamic_id, session=session, dataset_id=dataset.id, + filter_field=filter_field, + filter_value=filter_value, ) reform_sim = _get_or_create_simulation( @@ -698,6 +716,8 @@ def economic_impact( dynamic_id=request.dynamic_id, session=session, dataset_id=dataset.id, + filter_field=filter_field, + filter_value=filter_value, ) # Get or create report diff --git a/src/policyengine_api/models/simulation.py b/src/policyengine_api/models/simulation.py index 985db3e..176f12e 100644 --- a/src/policyengine_api/models/simulation.py +++ b/src/policyengine_api/models/simulation.py @@ -46,6 +46,16 @@ class SimulationBase(SQLModel): status: SimulationStatus = SimulationStatus.PENDING error_message: str | None = None + # Regional filtering parameters (passed to policyengine.py) + filter_field: str | None = Field( + default=None, + description="Household-level variable to filter dataset by (e.g., 'place_fips', 'country')", + ) + filter_value: str | None = Field( + default=None, + description="Value to match when filtering (e.g., '44000', 'ENGLAND')", + ) + class Simulation(SimulationBase, table=True): """Simulation database model.""" From 1015daeb86025f0b92129d8451f5999b02df0154 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 10 Feb 2026 23:41:08 +0100 Subject: [PATCH 22/87] feat: Add filter pass-through to Modal functions + region unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire filter_field/filter_value through Modal functions to policyengine.py: - simulate_economy_uk, simulate_economy_us - economy_comparison_uk, economy_comparison_us - Add fixtures_regions.py with factory functions for test data - Add 25 unit tests for region resolution and filtering: - test__given_region_with_filter__then_filter_params_included.py - test__given_region_without_filter__then_filter_params_none.py - test__given_dataset_id__then_region_is_none.py - test__given_same_params__then_deterministic_id.py - test__given_invalid_region__then_404_error.py - test__given_existing_simulation__then_reuses_existing.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/policyengine_api/modal_app.py | 27 +- test_fixtures/fixtures_regions.py | 260 ++++++++++++++++++ ...__given_dataset_id__then_region_is_none.py | 94 +++++++ ...isting_simulation__then_reuses_existing.py | 144 ++++++++++ ...t__given_invalid_region__then_404_error.py | 107 +++++++ ...ith_filter__then_filter_params_included.py | 170 ++++++++++++ ...without_filter__then_filter_params_none.py | 110 ++++++++ ...iven_same_params__then_deterministic_id.py | 171 ++++++++++++ 8 files changed, 1078 insertions(+), 5 deletions(-) create mode 100644 test_fixtures/fixtures_regions.py create mode 100644 tests/test__given_dataset_id__then_region_is_none.py create mode 100644 tests/test__given_existing_simulation__then_reuses_existing.py create mode 100644 tests/test__given_invalid_region__then_404_error.py create mode 100644 tests/test__given_region_with_filter__then_filter_params_included.py create mode 100644 tests/test__given_region_without_filter__then_filter_params_none.py create mode 100644 tests/test__given_same_params__then_deterministic_id.py diff --git a/src/policyengine_api/modal_app.py b/src/policyengine_api/modal_app.py index 1aa8119..332c349 100644 --- a/src/policyengine_api/modal_app.py +++ b/src/policyengine_api/modal_app.py @@ -841,6 +841,8 @@ def simulate_economy_uk(simulation_id: str, traceparent: str | None = None) -> N tax_benefit_model_version=pe_model_version, policy=policy, dynamic=dynamic, + filter_field=simulation.filter_field, + filter_value=simulation.filter_value, ) pe_sim.ensure() @@ -1007,6 +1009,8 @@ def simulate_economy_us(simulation_id: str, traceparent: str | None = None) -> N tax_benefit_model_version=pe_model_version, policy=policy, dynamic=dynamic, + filter_field=simulation.filter_field, + filter_value=simulation.filter_value, ) pe_sim.ensure() @@ -1112,11 +1116,12 @@ def economy_comparison_uk(job_id: str, traceparent: str | None = None) -> None: # Debug: log the key role import base64 import json + try: - payload = supabase_key.split('.')[1] - payload += '=' * (4 - len(payload) % 4) + payload = supabase_key.split(".")[1] + payload += "=" * (4 - len(payload) % 4) decoded = json.loads(base64.urlsafe_b64decode(payload)) - logfire.info("Supabase key info", role=decoded.get('role', 'unknown')) + logfire.info("Supabase key info", role=decoded.get("role", "unknown")) except Exception as e: logfire.warn("Could not decode key", error=str(e)) @@ -1213,6 +1218,8 @@ def economy_comparison_uk(job_id: str, traceparent: str | None = None) -> None: tax_benefit_model_version=pe_model_version, policy=baseline_policy, dynamic=baseline_dynamic, + filter_field=baseline_sim.filter_field, + filter_value=baseline_sim.filter_value, ) pe_baseline_sim.ensure() @@ -1222,6 +1229,8 @@ def economy_comparison_uk(job_id: str, traceparent: str | None = None) -> None: tax_benefit_model_version=pe_model_version, policy=reform_policy, dynamic=reform_dynamic, + filter_field=reform_sim.filter_field, + filter_value=reform_sim.filter_value, ) pe_reform_sim.ensure() @@ -1535,6 +1544,8 @@ def economy_comparison_us(job_id: str, traceparent: str | None = None) -> None: tax_benefit_model_version=pe_model_version, policy=baseline_policy, dynamic=baseline_dynamic, + filter_field=baseline_sim.filter_field, + filter_value=baseline_sim.filter_value, ) pe_baseline_sim.ensure() @@ -1544,6 +1555,8 @@ def economy_comparison_us(job_id: str, traceparent: str | None = None) -> None: tax_benefit_model_version=pe_model_version, policy=reform_policy, dynamic=reform_dynamic, + filter_field=reform_sim.filter_field, + filter_value=reform_sim.filter_value, ) pe_reform_sim.ensure() @@ -1930,7 +1943,9 @@ def compute_aggregate_uk(aggregate_id: str, traceparent: str | None = None) -> N pe_aggregate = PEAggregate( simulation=pe_sim, variable=aggregate.variable, - aggregate_type=PEAggregateType(aggregate.aggregate_type.value), + aggregate_type=PEAggregateType( + aggregate.aggregate_type.value + ), entity=aggregate.entity, ) pe_aggregate.run() @@ -2074,7 +2089,9 @@ def compute_aggregate_us(aggregate_id: str, traceparent: str | None = None) -> N pe_aggregate = PEAggregate( simulation=pe_sim, variable=aggregate.variable, - aggregate_type=PEAggregateType(aggregate.aggregate_type.value), + aggregate_type=PEAggregateType( + aggregate.aggregate_type.value + ), entity=aggregate.entity, ) pe_aggregate.run() diff --git a/test_fixtures/fixtures_regions.py b/test_fixtures/fixtures_regions.py new file mode 100644 index 0000000..e95e0d8 --- /dev/null +++ b/test_fixtures/fixtures_regions.py @@ -0,0 +1,260 @@ +"""Fixtures and helpers for region-related tests.""" + +from uuid import uuid4 + +import pytest + +from policyengine_api.models import ( + Dataset, + Region, + Simulation, + SimulationStatus, + TaxBenefitModel, + TaxBenefitModelVersion, +) + + +# ----------------------------------------------------------------------------- +# Constants +# ----------------------------------------------------------------------------- + +TEST_UUIDS = { + "DATASET": uuid4(), + "DATASET_UK": uuid4(), + "DATASET_US": uuid4(), + "MODEL_UK": uuid4(), + "MODEL_US": uuid4(), + "MODEL_VERSION_UK": uuid4(), + "MODEL_VERSION_US": uuid4(), + "REGION_UK": uuid4(), + "REGION_US_STATE": uuid4(), + "REGION_US_NATIONAL": uuid4(), + "POLICY": uuid4(), + "DYNAMIC": uuid4(), +} + +REGION_CODES = { + "UK_ENGLAND": "country/england", + "US_CALIFORNIA": "state/ca", + "US_NATIONAL": "us", + "UK_NATIONAL": "uk", +} + +FILTER_FIELDS = { + "UK_COUNTRY": "country", + "US_STATE": "state_code", + "US_FIPS": "place_fips", +} + +FILTER_VALUES = { + "ENGLAND": "ENGLAND", + "CALIFORNIA": "CA", + "CA_FIPS": "06000", +} + + +# ----------------------------------------------------------------------------- +# Factory Functions +# ----------------------------------------------------------------------------- + + +def create_tax_benefit_model( + session, name: str = "policyengine-uk", description: str = "UK model" +) -> TaxBenefitModel: + """Create and persist a TaxBenefitModel.""" + model = TaxBenefitModel(name=name, description=description) + session.add(model) + session.commit() + session.refresh(model) + return model + + +def create_tax_benefit_model_version( + session, model: TaxBenefitModel, version: str = "1.0.0" +) -> TaxBenefitModelVersion: + """Create and persist a TaxBenefitModelVersion.""" + model_version = TaxBenefitModelVersion( + model_id=model.id, + version=version, + description=f"Version {version}", + ) + session.add(model_version) + session.commit() + session.refresh(model_version) + return model_version + + +def create_dataset( + session, + model: TaxBenefitModel, + name: str = "test_dataset", + filepath: str = "test/path/dataset.h5", + year: int = 2024, +) -> Dataset: + """Create and persist a Dataset.""" + dataset = Dataset( + name=name, + description=f"Test dataset: {name}", + filepath=filepath, + year=year, + tax_benefit_model_id=model.id, + ) + session.add(dataset) + session.commit() + session.refresh(dataset) + return dataset + + +def create_region( + session, + model: TaxBenefitModel, + dataset: Dataset, + code: str, + label: str, + region_type: str, + requires_filter: bool = False, + filter_field: str | None = None, + filter_value: str | None = None, +) -> Region: + """Create and persist a Region.""" + region = Region( + code=code, + label=label, + region_type=region_type, + requires_filter=requires_filter, + filter_field=filter_field, + filter_value=filter_value, + dataset_id=dataset.id, + tax_benefit_model_id=model.id, + ) + session.add(region) + session.commit() + session.refresh(region) + return region + + +def create_simulation( + session, + dataset: Dataset, + model_version: TaxBenefitModelVersion, + filter_field: str | None = None, + filter_value: str | None = None, + status: SimulationStatus = SimulationStatus.PENDING, +) -> Simulation: + """Create and persist a Simulation with optional filter parameters.""" + simulation = Simulation( + dataset_id=dataset.id, + tax_benefit_model_version_id=model_version.id, + status=status, + filter_field=filter_field, + filter_value=filter_value, + ) + session.add(simulation) + session.commit() + session.refresh(simulation) + return simulation + + +# ----------------------------------------------------------------------------- +# Composite Fixtures +# ----------------------------------------------------------------------------- + + +@pytest.fixture +def uk_model_and_version(session): + """Create UK model with version.""" + model = create_tax_benefit_model( + session, name="policyengine-uk", description="UK model" + ) + version = create_tax_benefit_model_version(session, model) + return model, version + + +@pytest.fixture +def us_model_and_version(session): + """Create US model with version.""" + model = create_tax_benefit_model( + session, name="policyengine-us", description="US model" + ) + version = create_tax_benefit_model_version(session, model) + return model, version + + +@pytest.fixture +def uk_dataset(session, uk_model_and_version): + """Create a UK dataset.""" + model, _ = uk_model_and_version + return create_dataset( + session, model, name="uk_enhanced_frs", filepath="uk/enhanced_frs_2024.h5" + ) + + +@pytest.fixture +def us_dataset(session, us_model_and_version): + """Create a US dataset.""" + model, _ = us_model_and_version + return create_dataset(session, model, name="us_cps", filepath="us/cps_2024.h5") + + +@pytest.fixture +def uk_region_national(session, uk_model_and_version, uk_dataset): + """Create UK national region (no filter required).""" + model, _ = uk_model_and_version + return create_region( + session, + model=model, + dataset=uk_dataset, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + + +@pytest.fixture +def uk_region_england(session, uk_model_and_version, uk_dataset): + """Create England region (filter required).""" + model, _ = uk_model_and_version + return create_region( + session, + model=model, + dataset=uk_dataset, + code="country/england", + label="England", + region_type="country", + requires_filter=True, + filter_field="country", + filter_value="ENGLAND", + ) + + +@pytest.fixture +def us_region_national(session, us_model_and_version, us_dataset): + """Create US national region (no filter required).""" + model, _ = us_model_and_version + return create_region( + session, + model=model, + dataset=us_dataset, + code="us", + label="United States", + region_type="national", + requires_filter=False, + ) + + +@pytest.fixture +def us_region_california(session, us_model_and_version, us_dataset): + """Create California state region (filter required).""" + model, _ = us_model_and_version + return create_region( + session, + model=model, + dataset=us_dataset, + code="state/ca", + label="California", + region_type="state", + requires_filter=True, + filter_field="state_code", + filter_value="CA", + ) diff --git a/tests/test__given_dataset_id__then_region_is_none.py b/tests/test__given_dataset_id__then_region_is_none.py new file mode 100644 index 0000000..ee3c1d5 --- /dev/null +++ b/tests/test__given_dataset_id__then_region_is_none.py @@ -0,0 +1,94 @@ +"""Tests for dataset resolution when dataset_id is provided directly. + +When a dataset_id is provided instead of a region code, +the resolved region should be None. +""" + +import pytest +from sqlmodel import Session + +from policyengine_api.api.analysis import ( + EconomicImpactRequest, + _resolve_dataset_and_region, +) +from test_fixtures.fixtures_regions import ( + create_dataset, + create_tax_benefit_model, +) + + +class TestResolveDatasetWithDatasetId: + """Tests for _resolve_dataset_and_region when dataset_id is provided.""" + + def test_given_dataset_id_then_region_is_none(self, session: Session): + """Given a dataset_id, then region is None in the response.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset.id, + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_region is None + + def test_given_dataset_id_then_dataset_is_returned(self, session: Session): + """Given a dataset_id, then the correct dataset is returned.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset.id, + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" + + def test_given_dataset_id_and_region_then_region_takes_precedence( + self, session: Session + ): + """Given both dataset_id and region, then region takes precedence.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset1 = create_dataset(session, model, name="dataset_from_id") + dataset2 = create_dataset(session, model, name="dataset_from_region") + from test_fixtures.fixtures_regions import create_region + + region = create_region( + session, + model=model, + dataset=dataset2, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset1.id, + region="uk", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + # Region code takes precedence, so we get dataset2 + assert resolved_dataset.id == dataset2.id + assert resolved_region is not None + assert resolved_region.code == "uk" diff --git a/tests/test__given_existing_simulation__then_reuses_existing.py b/tests/test__given_existing_simulation__then_reuses_existing.py new file mode 100644 index 0000000..77731f1 --- /dev/null +++ b/tests/test__given_existing_simulation__then_reuses_existing.py @@ -0,0 +1,144 @@ +"""Tests for simulation reuse with filter parameters. + +When a simulation with the same parameters already exists, +it should be reused instead of creating a new one. +""" + +import pytest +from sqlmodel import Session + +from policyengine_api.api.analysis import _get_or_create_simulation +from policyengine_api.models import SimulationStatus +from test_fixtures.fixtures_regions import ( + create_dataset, + create_simulation, + create_tax_benefit_model, + create_tax_benefit_model_version, +) + + +class TestSimulationReuse: + """Tests for simulation reuse behavior.""" + + def test_given_existing_simulation_with_filter_then_reuses(self, session: Session): + """Given an existing simulation with filter params, then it is reused.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # Create initial simulation with filter params + first_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # When - request same simulation again + second_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # Then + assert first_sim.id == second_sim.id + + def test_given_different_filter_then_creates_new_simulation(self, session: Session): + """Given different filter params, then a new simulation is created.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # Create simulation for England + england_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # When - request simulation for Scotland + scotland_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="SCOTLAND", + ) + + # Then + assert england_sim.id != scotland_sim.id + assert england_sim.filter_value == "ENGLAND" + assert scotland_sim.filter_value == "SCOTLAND" + + def test_given_no_filter_vs_filter_then_creates_separate_simulations( + self, session: Session + ): + """Given national vs filtered, then separate simulations are created.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # Create national (no filter) simulation + national_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field=None, + filter_value=None, + ) + + # When - request filtered simulation + filtered_sim = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # Then + assert national_sim.id != filtered_sim.id + assert national_sim.filter_field is None + assert filtered_sim.filter_field == "country" + + def test_given_new_simulation_then_status_is_pending(self, session: Session): + """Given a new simulation request, then status is PENDING.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # When + simulation = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # Then + assert simulation.status == SimulationStatus.PENDING diff --git a/tests/test__given_invalid_region__then_404_error.py b/tests/test__given_invalid_region__then_404_error.py new file mode 100644 index 0000000..562a5c9 --- /dev/null +++ b/tests/test__given_invalid_region__then_404_error.py @@ -0,0 +1,107 @@ +"""Tests for region resolution error cases. + +When an invalid region code is provided or required parameters are missing, +appropriate HTTP errors should be raised. +""" + +import pytest +from fastapi import HTTPException +from sqlmodel import Session + +from policyengine_api.api.analysis import ( + EconomicImpactRequest, + _resolve_dataset_and_region, +) +from test_fixtures.fixtures_regions import ( + create_dataset, + create_region, + create_tax_benefit_model, +) + + +class TestInvalidRegionCode: + """Tests for invalid region code handling.""" + + def test_given_nonexistent_region_code_then_raises_404(self, session: Session): + """Given a region code that doesn't exist, then raises 404.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + # Note: No region is created for this code + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="nonexistent/region", + ) + + # When/Then + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + assert "not found" in exc_info.value.detail.lower() + + def test_given_region_for_wrong_model_then_raises_404(self, session: Session): + """Given a region code for wrong model, then raises 404.""" + # Given + uk_model = create_tax_benefit_model(session, name="policyengine-uk") + uk_dataset = create_dataset(session, uk_model, name="uk_enhanced_frs") + create_region( + session, + model=uk_model, + dataset=uk_dataset, + code="uk", + label="United Kingdom", + region_type="national", + ) + # Request uses US model but UK region code + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="uk", + ) + + # When/Then + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + + +class TestMissingRequiredParams: + """Tests for missing required parameters.""" + + def test_given_neither_dataset_nor_region_then_raises_400(self, session: Session): + """Given neither dataset_id nor region, then raises 400.""" + # Given + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + # Neither dataset_id nor region provided + ) + + # When/Then + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 400 + assert "either dataset_id or region" in exc_info.value.detail.lower() + + +class TestNonexistentDataset: + """Tests for nonexistent dataset handling.""" + + def test_given_nonexistent_dataset_id_then_raises_404(self, session: Session): + """Given a dataset_id that doesn't exist, then raises 404.""" + # Given + from uuid import uuid4 + + nonexistent_id = uuid4() + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=nonexistent_id, + ) + + # When/Then + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + assert "not found" in exc_info.value.detail.lower() diff --git a/tests/test__given_region_with_filter__then_filter_params_included.py b/tests/test__given_region_with_filter__then_filter_params_included.py new file mode 100644 index 0000000..c84a372 --- /dev/null +++ b/tests/test__given_region_with_filter__then_filter_params_included.py @@ -0,0 +1,170 @@ +"""Tests for region resolution with filter parameters. + +When a region requires filtering (e.g., England from UK dataset, +California from US dataset), the filter_field and filter_value +should be extracted and passed through to simulations. +""" + +import pytest +from sqlmodel import Session + +from policyengine_api.api.analysis import ( + EconomicImpactRequest, + _get_or_create_simulation, + _resolve_dataset_and_region, +) +from test_fixtures.fixtures_regions import ( + create_dataset, + create_region, + create_tax_benefit_model, + create_tax_benefit_model_version, +) + + +class TestResolveDatasetAndRegionWithFilter: + """Tests for _resolve_dataset_and_region when region requires filtering.""" + + def test_given_region_requires_filter_then_returns_filter_field( + self, session: Session + ): + """Given a region that requires filtering, then filter_field is populated.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + region = create_region( + session, + model=model, + dataset=dataset, + code="country/england", + label="England", + region_type="country", + requires_filter=True, + filter_field="country", + filter_value="ENGLAND", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="country/england", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_region is not None + assert resolved_region.filter_field == "country" + assert resolved_region.filter_value == "ENGLAND" + assert resolved_region.requires_filter is True + + def test_given_us_state_region_then_returns_state_filter(self, session: Session): + """Given a US state region, then returns state code filter.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-us") + dataset = create_dataset(session, model, name="us_cps") + region = create_region( + session, + model=model, + dataset=dataset, + code="state/ca", + label="California", + region_type="state", + requires_filter=True, + filter_field="state_code", + filter_value="CA", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="state/ca", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_region is not None + assert resolved_region.filter_field == "state_code" + assert resolved_region.filter_value == "CA" + + def test_given_region_with_filter_then_dataset_is_resolved(self, session: Session): + """Given a region code, then the associated dataset is returned.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + region = create_region( + session, + model=model, + dataset=dataset, + code="country/england", + label="England", + region_type="country", + requires_filter=True, + filter_field="country", + filter_value="ENGLAND", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="country/england", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" + + +class TestSimulationCreationWithFilter: + """Tests for creating simulations with filter parameters.""" + + def test_given_filter_params_then_simulation_has_filter_fields( + self, session: Session + ): + """Given filter parameters, then created simulation has filter fields populated.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # When + simulation = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + # Then + assert simulation.filter_field == "country" + assert simulation.filter_value == "ENGLAND" + + def test_given_no_filter_params_then_simulation_has_null_filter_fields( + self, session: Session + ): + """Given no filter parameters, then created simulation has null filter fields.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + # When + simulation = _get_or_create_simulation( + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + ) + + # Then + assert simulation.filter_field is None + assert simulation.filter_value is None diff --git a/tests/test__given_region_without_filter__then_filter_params_none.py b/tests/test__given_region_without_filter__then_filter_params_none.py new file mode 100644 index 0000000..e81d7a8 --- /dev/null +++ b/tests/test__given_region_without_filter__then_filter_params_none.py @@ -0,0 +1,110 @@ +"""Tests for region resolution without filter parameters. + +When a region does not require filtering (e.g., national UK or US), +the filter_field and filter_value should be None. +""" + +import pytest +from sqlmodel import Session + +from policyengine_api.api.analysis import ( + EconomicImpactRequest, + _resolve_dataset_and_region, +) +from test_fixtures.fixtures_regions import ( + create_dataset, + create_region, + create_tax_benefit_model, +) + + +class TestResolveDatasetAndRegionWithoutFilter: + """Tests for _resolve_dataset_and_region when region does not require filtering.""" + + def test_given_national_uk_region_then_filter_params_none(self, session: Session): + """Given UK national region, then filter_field and filter_value are None.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + region = create_region( + session, + model=model, + dataset=dataset, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="uk", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_region is not None + assert resolved_region.requires_filter is False + assert resolved_region.filter_field is None + assert resolved_region.filter_value is None + + def test_given_national_us_region_then_filter_params_none(self, session: Session): + """Given US national region, then filter_field and filter_value are None.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-us") + dataset = create_dataset(session, model, name="us_cps") + region = create_region( + session, + model=model, + dataset=dataset, + code="us", + label="United States", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="us", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_region is not None + assert resolved_region.requires_filter is False + assert resolved_region.filter_field is None + assert resolved_region.filter_value is None + + def test_given_national_region_then_dataset_still_resolved(self, session: Session): + """Given national region without filter, then dataset is still correctly resolved.""" + # Given + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + region = create_region( + session, + model=model, + dataset=dataset, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="uk", + ) + + # When + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + # Then + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" diff --git a/tests/test__given_same_params__then_deterministic_id.py b/tests/test__given_same_params__then_deterministic_id.py new file mode 100644 index 0000000..d393f53 --- /dev/null +++ b/tests/test__given_same_params__then_deterministic_id.py @@ -0,0 +1,171 @@ +"""Tests for deterministic simulation ID generation. + +The simulation ID is generated deterministically from the simulation +parameters (dataset, model version, policy, dynamic, filter params). +This ensures that re-running the same simulation reuses existing results. +""" + +from uuid import uuid4 + +import pytest + +from policyengine_api.api.analysis import _get_deterministic_simulation_id + + +class TestDeterministicSimulationId: + """Tests for _get_deterministic_simulation_id function.""" + + def test_given_same_params_then_same_id_returned(self): + """Given identical parameters, then the same ID is returned.""" + # Given + dataset_id = uuid4() + model_version_id = uuid4() + policy_id = uuid4() + dynamic_id = uuid4() + filter_field = "country" + filter_value = "ENGLAND" + + # When + id1 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field, + filter_value, + ) + id2 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field, + filter_value, + ) + + # Then + assert id1 == id2 + + def test_given_different_filter_field_then_different_id(self): + """Given different filter_field, then a different ID is returned.""" + # Given + dataset_id = uuid4() + model_version_id = uuid4() + policy_id = None + dynamic_id = None + + # When + id1 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field="state_code", + filter_value="ENGLAND", + ) + + # Then + assert id1 != id2 + + def test_given_different_filter_value_then_different_id(self): + """Given different filter_value, then a different ID is returned.""" + # Given + dataset_id = uuid4() + model_version_id = uuid4() + policy_id = None + dynamic_id = None + + # When + id1 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field="country", + filter_value="SCOTLAND", + ) + + # Then + assert id1 != id2 + + def test_given_filter_none_vs_filter_set_then_different_id(self): + """Given None filter vs set filter, then different IDs are returned.""" + # Given + dataset_id = uuid4() + model_version_id = uuid4() + policy_id = None + dynamic_id = None + + # When + id_no_filter = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field=None, + filter_value=None, + ) + id_with_filter = _get_deterministic_simulation_id( + dataset_id, + model_version_id, + policy_id, + dynamic_id, + filter_field="country", + filter_value="ENGLAND", + ) + + # Then + assert id_no_filter != id_with_filter + + def test_given_different_dataset_then_different_id(self): + """Given different dataset_id, then a different ID is returned.""" + # Given + model_version_id = uuid4() + policy_id = None + dynamic_id = None + filter_field = "country" + filter_value = "ENGLAND" + + # When + id1 = _get_deterministic_simulation_id( + uuid4(), model_version_id, policy_id, dynamic_id, filter_field, filter_value + ) + id2 = _get_deterministic_simulation_id( + uuid4(), model_version_id, policy_id, dynamic_id, filter_field, filter_value + ) + + # Then + assert id1 != id2 + + def test_given_null_optional_params_then_consistent_id(self): + """Given null optional parameters, then consistent ID is generated.""" + # Given + dataset_id = uuid4() + model_version_id = uuid4() + + # When + id1 = _get_deterministic_simulation_id( + dataset_id, model_version_id, None, None, None, None + ) + id2 = _get_deterministic_simulation_id( + dataset_id, model_version_id, None, None, None, None + ) + + # Then + assert id1 == id2 From ca580fb8d01ef8938fb560bd4d5b4836caf8bfac Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 11 Feb 2026 23:45:13 +0100 Subject: [PATCH 23/87] feat: Add seed script for US and UK regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add seed_regions.py to populate the regions table with geographic data from policyengine.py's region registries: - US: National + 51 states (DC included) - UK: National + 4 countries (England, Scotland, Wales, NI) Optional flags: - --include-places: Add US cities (333 places over 100K population) - --include-districts: Add US congressional districts (436) - --us-only / --uk-only: Seed only one country 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/seed_regions.py | 280 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 scripts/seed_regions.py diff --git a/scripts/seed_regions.py b/scripts/seed_regions.py new file mode 100644 index 0000000..f4f9954 --- /dev/null +++ b/scripts/seed_regions.py @@ -0,0 +1,280 @@ +"""Seed regions for US and UK geographic analysis. + +This script populates the regions table with: +- US: National, 51 states (incl. DC), and optionally places/cities +- UK: National and 4 countries (England, Scotland, Wales, Northern Ireland) + +Regions are sourced from policyengine.py's region registries and linked +to the appropriate datasets in the database. + +Usage: + python scripts/seed_regions.py # Seed US and UK regions + python scripts/seed_regions.py --us-only # Seed only US regions + python scripts/seed_regions.py --uk-only # Seed only UK regions + python scripts/seed_regions.py --include-places # Include US places (cities) +""" + +import argparse +import sys +import time +from pathlib import Path + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn +from sqlmodel import Session, create_engine, select + +from policyengine_api.config.settings import settings +from policyengine_api.models import Dataset, Region, TaxBenefitModel + +console = Console() + + +def get_session() -> Session: + """Get database session.""" + engine = create_engine(settings.database_url) + return Session(engine) + + +def seed_us_regions( + session: Session, + include_places: bool = False, + include_districts: bool = False, +) -> tuple[int, int]: + """Seed US regions from policyengine.py registry. + + Args: + session: Database session + include_places: Include US places (cities over 100K population) + include_districts: Include congressional districts + + Returns: + Tuple of (created_count, skipped_count) + """ + from policyengine.countries.us.regions import us_region_registry + + # Get US model + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not us_model: + console.print("[red]Error: US model not found. Run seed.py first.[/red]") + return 0, 0 + + # Get US national dataset (CPS) + us_dataset = session.exec( + select(Dataset) + .where(Dataset.tax_benefit_model_id == us_model.id) + .where(Dataset.name.contains("cps")) # type: ignore + .order_by(Dataset.year.desc()) # type: ignore + ).first() + + if not us_dataset: + console.print("[red]Error: US dataset not found. Run seed.py first.[/red]") + return 0, 0 + + created = 0 + skipped = 0 + + # Filter regions based on options + regions_to_seed = [] + for region in us_region_registry.regions: + if region.region_type == "national": + regions_to_seed.append(region) + elif region.region_type == "state": + regions_to_seed.append(region) + elif region.region_type == "congressional_district" and include_districts: + regions_to_seed.append(region) + elif region.region_type == "place" and include_places: + regions_to_seed.append(region) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("US regions", total=len(regions_to_seed)) + + for pe_region in regions_to_seed: + progress.update(task, description=f"US: {pe_region.label}") + + # Check if region already exists + existing = session.exec( + select(Region).where(Region.code == pe_region.code) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Create region record + db_region = Region( + code=pe_region.code, + label=pe_region.label, + region_type=pe_region.region_type, + requires_filter=pe_region.requires_filter, + filter_field=pe_region.filter_field, + filter_value=pe_region.filter_value, + parent_code=pe_region.parent_code, + state_code=pe_region.state_code, + state_name=pe_region.state_name, + dataset_id=us_dataset.id, # All US regions use the national dataset + tax_benefit_model_id=us_model.id, + ) + session.add(db_region) + created += 1 + progress.advance(task) + + session.commit() + + return created, skipped + + +def seed_uk_regions(session: Session) -> tuple[int, int]: + """Seed UK regions from policyengine.py registry. + + Args: + session: Database session + + Returns: + Tuple of (created_count, skipped_count) + """ + from policyengine.countries.uk.regions import uk_region_registry + + # Get UK model + uk_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") + ).first() + + if not uk_model: + console.print( + "[yellow]Warning: UK model not found. Skipping UK regions.[/yellow]" + ) + return 0, 0 + + # Get UK national dataset (FRS) + uk_dataset = session.exec( + select(Dataset) + .where(Dataset.tax_benefit_model_id == uk_model.id) + .where(Dataset.name.contains("frs")) # type: ignore + .order_by(Dataset.year.desc()) # type: ignore + ).first() + + if not uk_dataset: + console.print( + "[yellow]Warning: UK dataset not found. Skipping UK regions.[/yellow]" + ) + return 0, 0 + + created = 0 + skipped = 0 + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("UK regions", total=len(uk_region_registry.regions)) + + for pe_region in uk_region_registry.regions: + progress.update(task, description=f"UK: {pe_region.label}") + + # Check if region already exists + existing = session.exec( + select(Region).where(Region.code == pe_region.code) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Create region record + db_region = Region( + code=pe_region.code, + label=pe_region.label, + region_type=pe_region.region_type, + requires_filter=pe_region.requires_filter, + filter_field=pe_region.filter_field, + filter_value=pe_region.filter_value, + parent_code=pe_region.parent_code, + state_code=None, # UK regions don't have state_code + state_name=None, + dataset_id=uk_dataset.id, # All UK regions use the national dataset + tax_benefit_model_id=uk_model.id, + ) + session.add(db_region) + created += 1 + progress.advance(task) + + session.commit() + + return created, skipped + + +def main(): + parser = argparse.ArgumentParser(description="Seed US and UK regions") + parser.add_argument( + "--us-only", + action="store_true", + help="Only seed US regions", + ) + parser.add_argument( + "--uk-only", + action="store_true", + help="Only seed UK regions", + ) + parser.add_argument( + "--include-places", + action="store_true", + help="Include US places (cities over 100K population)", + ) + parser.add_argument( + "--include-districts", + action="store_true", + help="Include US congressional districts", + ) + args = parser.parse_args() + + console.print("[bold green]Seeding regions...[/bold green]\n") + + start = time.time() + total_created = 0 + total_skipped = 0 + + with get_session() as session: + # Seed US regions + if not args.uk_only: + console.print("[bold]US Regions[/bold]") + us_created, us_skipped = seed_us_regions( + session, + include_places=args.include_places, + include_districts=args.include_districts, + ) + total_created += us_created + total_skipped += us_skipped + console.print( + f"[green]✓[/green] US regions: {us_created} created, {us_skipped} skipped\n" + ) + + # Seed UK regions + if not args.us_only: + console.print("[bold]UK Regions[/bold]") + uk_created, uk_skipped = seed_uk_regions(session) + total_created += uk_created + total_skipped += uk_skipped + console.print( + f"[green]✓[/green] UK regions: {uk_created} created, {uk_skipped} skipped\n" + ) + + elapsed = time.time() - start + console.print(f"[bold]Total: {total_created} created, {total_skipped} skipped[/bold]") + console.print(f"[bold]Time: {elapsed:.1f}s[/bold]") + + +if __name__ == "__main__": + main() From df422aaaea36ed76340e75aae58072cca484f59c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 12 Feb 2026 16:56:18 +0100 Subject: [PATCH 24/87] feat: Integrate regions seeding into main seed script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --skip-regions, --include-places, and --include-districts CLI options to seed.py. Regions are now seeded as part of the standard database setup process, sourcing region definitions from policyengine.py's registries. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/seed.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/seed.py b/scripts/seed.py index 4274528..2568d26 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -631,6 +631,21 @@ def main(): action="store_true", help="Skip UK datasets (useful when HuggingFace token is not available)", ) + parser.add_argument( + "--skip-regions", + action="store_true", + help="Skip seeding regions", + ) + parser.add_argument( + "--include-places", + action="store_true", + help="Include US places (cities over 100K population) when seeding regions", + ) + parser.add_argument( + "--include-districts", + action="store_true", + help="Include US congressional districts when seeding regions", + ) args = parser.parse_args() with logfire.span("database_seeding"): @@ -652,6 +667,26 @@ def main(): # Seed example policies seed_example_policies(session) + # Seed regions + if not args.skip_regions: + from seed_regions import seed_us_regions, seed_uk_regions + + console.print("\n[bold]Seeding regions...[/bold]") + us_created, us_skipped = seed_us_regions( + session, + include_places=args.include_places, + include_districts=args.include_districts, + ) + console.print( + f"[green]✓[/green] US regions: {us_created} created, {us_skipped} skipped" + ) + + if not args.skip_uk_datasets: + uk_created, uk_skipped = seed_uk_regions(session) + console.print( + f"[green]✓[/green] UK regions: {uk_created} created, {uk_skipped} skipped" + ) + console.print("\n[bold green]✓ Database seeding complete![/bold green]") From 473265058adeae9063927d4de65e67361cd89954 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 12 Feb 2026 17:48:17 +0100 Subject: [PATCH 25/87] refactor: Change regions CLI to use skip flags instead of include flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default behavior now seeds all US regions (national, states, districts, places). Use --skip-places and --skip-districts to exclude specific region types. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/seed.py | 12 ++++++------ scripts/seed_regions.py | 31 ++++++++++++++++--------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/scripts/seed.py b/scripts/seed.py index 2568d26..072b7bb 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -637,14 +637,14 @@ def main(): help="Skip seeding regions", ) parser.add_argument( - "--include-places", + "--skip-places", action="store_true", - help="Include US places (cities over 100K population) when seeding regions", + help="Skip US places (cities over 100K population) when seeding regions", ) parser.add_argument( - "--include-districts", + "--skip-districts", action="store_true", - help="Include US congressional districts when seeding regions", + help="Skip US congressional districts when seeding regions", ) args = parser.parse_args() @@ -674,8 +674,8 @@ def main(): console.print("\n[bold]Seeding regions...[/bold]") us_created, us_skipped = seed_us_regions( session, - include_places=args.include_places, - include_districts=args.include_districts, + skip_places=args.skip_places, + skip_districts=args.skip_districts, ) console.print( f"[green]✓[/green] US regions: {us_created} created, {us_skipped} skipped" diff --git a/scripts/seed_regions.py b/scripts/seed_regions.py index f4f9954..c8cc9d8 100644 --- a/scripts/seed_regions.py +++ b/scripts/seed_regions.py @@ -1,17 +1,18 @@ """Seed regions for US and UK geographic analysis. This script populates the regions table with: -- US: National, 51 states (incl. DC), and optionally places/cities +- US: National, 51 states (incl. DC), 436 congressional districts, 333 places/cities - UK: National and 4 countries (England, Scotland, Wales, Northern Ireland) Regions are sourced from policyengine.py's region registries and linked to the appropriate datasets in the database. Usage: - python scripts/seed_regions.py # Seed US and UK regions + python scripts/seed_regions.py # Seed all US and UK regions python scripts/seed_regions.py --us-only # Seed only US regions python scripts/seed_regions.py --uk-only # Seed only UK regions - python scripts/seed_regions.py --include-places # Include US places (cities) + python scripts/seed_regions.py --skip-places # Exclude US places (cities) + python scripts/seed_regions.py --skip-districts # Exclude US congressional districts """ import argparse @@ -40,15 +41,15 @@ def get_session() -> Session: def seed_us_regions( session: Session, - include_places: bool = False, - include_districts: bool = False, + skip_places: bool = False, + skip_districts: bool = False, ) -> tuple[int, int]: """Seed US regions from policyengine.py registry. Args: session: Database session - include_places: Include US places (cities over 100K population) - include_districts: Include congressional districts + skip_places: Skip US places (cities over 100K population) + skip_districts: Skip congressional districts Returns: Tuple of (created_count, skipped_count) @@ -86,9 +87,9 @@ def seed_us_regions( regions_to_seed.append(region) elif region.region_type == "state": regions_to_seed.append(region) - elif region.region_type == "congressional_district" and include_districts: + elif region.region_type == "congressional_district" and not skip_districts: regions_to_seed.append(region) - elif region.region_type == "place" and include_places: + elif region.region_type == "place" and not skip_places: regions_to_seed.append(region) with Progress( @@ -229,14 +230,14 @@ def main(): help="Only seed UK regions", ) parser.add_argument( - "--include-places", + "--skip-places", action="store_true", - help="Include US places (cities over 100K population)", + help="Skip US places (cities over 100K population)", ) parser.add_argument( - "--include-districts", + "--skip-districts", action="store_true", - help="Include US congressional districts", + help="Skip US congressional districts", ) args = parser.parse_args() @@ -252,8 +253,8 @@ def main(): console.print("[bold]US Regions[/bold]") us_created, us_skipped = seed_us_regions( session, - include_places=args.include_places, - include_districts=args.include_districts, + skip_places=args.skip_places, + skip_districts=args.skip_districts, ) total_created += us_created total_skipped += us_skipped From dd523ef7d27bf4d243a35556be4af0c53c70e9c2 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 13 Feb 2026 00:40:03 +0100 Subject: [PATCH 26/87] refactor: Split seed.py into modular subscripts with presets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Break up monolithic seed.py into focused subscripts: - seed_utils.py: Shared utilities (get_session, bulk_insert, console) - seed_models.py: TaxBenefitModel, Version, Variables, Parameters, ParameterValues - seed_datasets.py: Dataset seeding and S3 upload - seed_policies.py: Example policy reforms - seed_regions.py: Geographic regions (updated to use seed_utils) Main seed.py is now an orchestrator with preset configurations: - full: Everything (default) - lite: Both countries, 2026 only, skip state params, core regions - minimal: Both countries, 2026 only, no policies/regions - uk-lite, uk-minimal: UK-only variants - us-lite, us-minimal: US-only variants Each subscript can also run standalone with its own CLI. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- scripts/seed.py | 880 ++++++--------------- scripts/seed_datasets.py | 227 ++++++ scripts/{seed_common.py => seed_models.py} | 211 +++-- scripts/seed_policies.py | 303 ++++--- scripts/seed_regions.py | 20 +- scripts/seed_uk_datasets.py | 113 --- scripts/seed_uk_model.py | 33 - scripts/seed_us_datasets.py | 108 --- scripts/seed_us_model.py | 33 - scripts/seed_utils.py | 72 ++ 10 files changed, 800 insertions(+), 1200 deletions(-) create mode 100644 scripts/seed_datasets.py rename scripts/{seed_common.py => seed_models.py} (68%) delete mode 100644 scripts/seed_uk_datasets.py delete mode 100644 scripts/seed_uk_model.py delete mode 100644 scripts/seed_us_datasets.py delete mode 100644 scripts/seed_us_model.py create mode 100644 scripts/seed_utils.py diff --git a/scripts/seed.py b/scripts/seed.py index 072b7bb..a1e1c35 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -1,693 +1,259 @@ -"""Seed database with UK and US models, variables, parameters, datasets.""" +"""Seed PolicyEngine database with models, datasets, policies, and regions. + +This is the main orchestrator script that calls individual seed scripts +based on the selected preset. + +Presets: + full - Everything (default) + lite - Both countries, 2026 datasets only, skip state params, core regions + minimal - Both countries, 2026 datasets only, skip state params, no policies/regions + uk-lite - UK only, 2026 datasets, skip state params + uk-minimal - UK only, 2026 datasets, skip state params, no policies/regions + us-lite - US only, 2026 datasets, skip state params, core regions only + us-minimal - US only, 2026 datasets, skip state params, no policies/regions + +Usage: + python scripts/seed.py # Full seed (default) + python scripts/seed.py --preset=lite # Lite mode for both countries + python scripts/seed.py --preset=us-lite # US only, lite mode + python scripts/seed.py --preset=minimal # Minimal seed (no policies/regions) +""" import argparse -import json -import logging -import math -import sys -import warnings -from datetime import datetime, timezone -from pathlib import Path -from uuid import uuid4 - -import logfire - -# Disable all SQLAlchemy and database logging BEFORE any imports -logging.basicConfig(level=logging.ERROR) -logging.getLogger("sqlalchemy").setLevel(logging.ERROR) -warnings.filterwarnings("ignore") - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from policyengine.tax_benefit_models.uk import uk_latest # noqa: E402 -from policyengine.tax_benefit_models.uk.datasets import ( # noqa: E402 - ensure_datasets as ensure_uk_datasets, -) -from policyengine.tax_benefit_models.us import us_latest # noqa: E402 -from policyengine.tax_benefit_models.us.datasets import ( # noqa: E402 - ensure_datasets as ensure_us_datasets, -) -from rich.console import Console # noqa: E402 -from rich.progress import Progress, SpinnerColumn, TextColumn # noqa: E402 -from sqlmodel import Session, create_engine, select # noqa: E402 - -from policyengine_api.config.settings import settings # noqa: E402 -from policyengine_api.models import ( # noqa: E402 - Dataset, - Parameter, - ParameterValue, - Policy, - TaxBenefitModel, - TaxBenefitModelVersion, -) -from policyengine_api.services.storage import ( # noqa: E402 - upload_dataset_for_seeding, -) - -# Configure logfire -if settings.logfire_token: - logfire.configure( - token=settings.logfire_token, - environment=settings.logfire_environment, - console=False, - ) - -console = Console() - - -def get_quiet_session(): - """Get database session with logging disabled.""" - engine = create_engine(settings.database_url, echo=False) - with Session(engine) as session: - yield session - - -def bulk_insert(session, table: str, columns: list[str], rows: list[dict]): - """Fast bulk insert using PostgreSQL COPY via StringIO.""" - if not rows: - return - - import io - - # Get raw psycopg2 connection - need to use the connection from session - # but not commit separately to avoid transaction issues - connection = session.connection() - raw_conn = connection.connection.dbapi_connection - cursor = raw_conn.cursor() - - # Build CSV-like data in memory - output = io.StringIO() - for row in rows: - values = [] - for col in columns: - val = row[col] - if val is None: - values.append("\\N") - elif isinstance(val, str): - # Escape special characters for COPY - val = ( - val.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n") - ) - values.append(val) - else: - values.append(str(val)) - output.write("\t".join(values) + "\n") - - output.seek(0) - - # COPY is the fastest way to bulk load PostgreSQL - cursor.copy_from(output, table, columns=columns, null="\\N") - # Let SQLAlchemy handle the commit via session - session.commit() - - -def seed_model(model_version, session, lite: bool = False) -> TaxBenefitModelVersion: - """Seed a tax-benefit model with its variables and parameters.""" - - with logfire.span( - "seed_model", - model=model_version.model.id, - version=model_version.version, - ): - # Create or get the model - console.print(f"[bold blue]Seeding {model_version.model.id}...") - - existing_model = session.exec( - select(TaxBenefitModel).where( - TaxBenefitModel.name == model_version.model.id - ) - ).first() - - if existing_model: - db_model = existing_model - console.print(f" Using existing model: {db_model.id}") - else: - db_model = TaxBenefitModel( - name=model_version.model.id, - description=model_version.model.description, - ) - session.add(db_model) - session.commit() - session.refresh(db_model) - console.print(f" Created model: {db_model.id}") - - # Create model version - existing_version = session.exec( - select(TaxBenefitModelVersion).where( - TaxBenefitModelVersion.model_id == db_model.id, - TaxBenefitModelVersion.version == model_version.version, - ) - ).first() - - if existing_version: - console.print( - f" Model version {model_version.version} already exists, skipping" - ) - return existing_version - - db_version = TaxBenefitModelVersion( - model_id=db_model.id, - version=model_version.version, - description=f"Version {model_version.version}", - ) - session.add(db_version) - session.commit() - session.refresh(db_version) - console.print(f" Created version: {db_version.version}") - - # Add variables - with logfire.span("add_variables", count=len(model_version.variables)): - var_rows = [] - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task( - f"Preparing {len(model_version.variables)} variables", - total=len(model_version.variables), - ) - for var in model_version.variables: - var_rows.append( - { - "id": uuid4(), - "name": var.name, - "entity": var.entity, - "description": var.description or "", - "data_type": var.data_type.__name__ - if hasattr(var.data_type, "__name__") - else str(var.data_type), - "possible_values": None, - "tax_benefit_model_version_id": db_version.id, - "created_at": datetime.now(timezone.utc), - } - ) - progress.advance(task) - - console.print(f" Inserting {len(var_rows)} variables...") - bulk_insert( - session, - "variables", - [ - "id", - "name", - "entity", - "description", - "data_type", - "possible_values", - "tax_benefit_model_version_id", - "created_at", - ], - var_rows, +import time +from dataclasses import dataclass + +from seed_utils import console, get_session + +# Import seed functions from subscripts +from seed_datasets import seed_uk_datasets, seed_us_datasets +from seed_models import seed_uk_model, seed_us_model +from seed_policies import seed_uk_policy, seed_us_policy +from seed_regions import seed_uk_regions, seed_us_regions + + +@dataclass +class SeedConfig: + """Configuration for database seeding.""" + + # Countries + seed_uk: bool = True + seed_us: bool = True + + # Models + skip_state_params: bool = False + + # Datasets + dataset_year: int | None = None # None = all years + + # Policies + seed_policies: bool = True + + # Regions + seed_regions: bool = True + skip_places: bool = False + skip_districts: bool = False + + +# Preset configurations +PRESETS: dict[str, SeedConfig] = { + "full": SeedConfig( + seed_uk=True, + seed_us=True, + skip_state_params=False, + dataset_year=None, + seed_policies=True, + seed_regions=True, + skip_places=False, + skip_districts=False, + ), + "lite": SeedConfig( + seed_uk=True, + seed_us=True, + skip_state_params=True, + dataset_year=2026, + seed_policies=True, + seed_regions=True, + skip_places=True, + skip_districts=True, + ), + "minimal": SeedConfig( + seed_uk=True, + seed_us=True, + skip_state_params=True, + dataset_year=2026, + seed_policies=False, + seed_regions=False, + ), + "uk-lite": SeedConfig( + seed_uk=True, + seed_us=False, + skip_state_params=True, + dataset_year=2026, + seed_policies=True, + seed_regions=True, + ), + "uk-minimal": SeedConfig( + seed_uk=True, + seed_us=False, + skip_state_params=True, + dataset_year=2026, + seed_policies=False, + seed_regions=False, + ), + "us-lite": SeedConfig( + seed_uk=False, + seed_us=True, + skip_state_params=True, + dataset_year=2026, + seed_policies=True, + seed_regions=True, + skip_places=True, + skip_districts=True, + ), + "us-minimal": SeedConfig( + seed_uk=False, + seed_us=True, + skip_state_params=True, + dataset_year=2026, + seed_policies=False, + seed_regions=False, + ), +} + + +def run_seed(config: SeedConfig): + """Run database seeding with the given configuration.""" + start = time.time() + + with get_session() as session: + # Step 1: Seed models + console.print("[bold blue]Step 1: Seeding models...[/bold blue]\n") + + if config.seed_uk: + seed_uk_model(session, skip_state_params=config.skip_state_params) + + if config.seed_us: + seed_us_model(session, skip_state_params=config.skip_state_params) + + # Step 2: Seed datasets + console.print("[bold blue]Step 2: Seeding datasets...[/bold blue]\n") + + if config.seed_uk: + console.print("[bold]UK Datasets[/bold]") + uk_created, uk_skipped = seed_uk_datasets( + session, year=config.dataset_year ) - console.print( - f" [green]✓[/green] Added {len(model_version.variables)} variables" + f"[green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped\n" ) - # Add parameters (only user-facing ones: those with labels) - # Deduplicate by name - keep first occurrence - # In lite mode, exclude US state parameters (gov.states.*) - seen_names = set() - parameters_to_add = [] - skipped_state_params = 0 - for p in model_version.parameters: - if p.label is None or p.name in seen_names: - continue - # In lite mode, skip state-level parameters for faster seeding - if lite and p.name.startswith("gov.states."): - skipped_state_params += 1 - continue - parameters_to_add.append(p) - seen_names.add(p.name) - - filter_msg = f" Filtered to {len(parameters_to_add)} user-facing parameters" - filter_msg += f" (from {len(model_version.parameters)} total, deduplicated by name)" - if lite and skipped_state_params > 0: - filter_msg += f", skipped {skipped_state_params} state params (lite mode)" - console.print(filter_msg) - - with logfire.span("add_parameters", count=len(parameters_to_add)): - # Build list of parameter dicts for bulk insert - param_rows = [] - param_names = [] # Track (pe_id, name, generated_uuid) - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task( - f"Preparing {len(parameters_to_add)} parameters", - total=len(parameters_to_add), - ) - for param in parameters_to_add: - param_uuid = uuid4() - param_rows.append( - { - "id": param_uuid, - "name": param.name, - "label": param.label if hasattr(param, "label") else None, - "description": param.description or "", - "data_type": param.data_type.__name__ - if hasattr(param.data_type, "__name__") - else str(param.data_type), - "unit": param.unit, - "tax_benefit_model_version_id": db_version.id, - "created_at": datetime.now(timezone.utc), - } - ) - param_names.append((param.id, param.name, param_uuid)) - progress.advance(task) - - console.print(f" Inserting {len(param_rows)} parameters...") - bulk_insert( - session, - "parameters", - [ - "id", - "name", - "label", - "description", - "data_type", - "unit", - "tax_benefit_model_version_id", - "created_at", - ], - param_rows, + if config.seed_us: + console.print("[bold]US Datasets[/bold]") + us_created, us_skipped = seed_us_datasets( + session, year=config.dataset_year ) - - # Build param_id_map from pre-generated UUIDs - param_id_map = {pe_id: db_uuid for pe_id, name, db_uuid in param_names} - console.print( - f" [green]✓[/green] Added {len(parameters_to_add)} parameters" + f"[green]✓[/green] US: {us_created} created, {us_skipped} skipped\n" ) - # Add parameter values - # Filter to only include values for parameters we added - parameter_values_to_add = [ - pv - for pv in model_version.parameter_values - if pv.parameter.id in param_id_map - ] - console.print(f" Found {len(parameter_values_to_add)} parameter values to add") - - with logfire.span("add_parameter_values", count=len(parameter_values_to_add)): - pv_rows = [] - skipped = 0 - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task( - f"Preparing {len(parameter_values_to_add)} parameter values", - total=len(parameter_values_to_add), - ) - for pv in parameter_values_to_add: - # Handle Infinity values - skip them as they can't be stored in JSON - if isinstance(pv.value, float) and ( - math.isinf(pv.value) or math.isnan(pv.value) - ): - skipped += 1 - progress.advance(task) - continue - - # Source data has dates swapped (start > end), fix ordering - # Only swap if both dates are set, otherwise keep original - if pv.start_date and pv.end_date: - start = pv.end_date # Swap: source end is our start - end = pv.start_date # Swap: source start is our end - else: - start = pv.start_date - end = pv.end_date - pv_rows.append( - { - "id": uuid4(), - "parameter_id": param_id_map[pv.parameter.id], - "value_json": json.dumps(pv.value), - "start_date": start, - "end_date": end, - "policy_id": None, - "dynamic_id": None, - "created_at": datetime.now(timezone.utc), - } - ) - progress.advance(task) - - console.print(f" Inserting {len(pv_rows)} parameter values...") - bulk_insert( - session, - "parameter_values", - [ - "id", - "parameter_id", - "value_json", - "start_date", - "end_date", - "policy_id", - "dynamic_id", - "created_at", - ], - pv_rows, - ) + # Step 3: Seed policies + if config.seed_policies: + console.print("[bold blue]Step 3: Seeding policies...[/bold blue]\n") - console.print( - f" [green]✓[/green] Added {len(pv_rows)} parameter values" - + (f" (skipped {skipped} invalid)" if skipped else "") - ) - - return db_version - - -def seed_datasets(session, lite: bool = False, skip_uk_datasets: bool = False): - """Seed datasets and upload to S3.""" - with logfire.span("seed_datasets"): - mode_str = " (lite mode - 2026 only)" if lite else "" - console.print(f"[bold blue]Seeding datasets{mode_str}...") + if config.seed_uk: + seed_uk_policy(session) - # Get UK and US models - uk_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") - ).first() - us_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") - ).first() - - if not uk_model or not us_model: - console.print( - "[red]Error: UK or US model not found. Run seed_model first.[/red]" - ) - return - - data_folder = str(Path(__file__).parent.parent / "data") - - # UK datasets - uk_created = 0 - uk_skipped = 0 - - if skip_uk_datasets: - console.print(" [yellow]Skipping UK datasets (--skip-uk-datasets)[/yellow]") - else: - console.print(" Creating UK datasets...") - uk_datasets = ensure_uk_datasets(data_folder=data_folder) - - # In lite mode, only upload FRS 2026 - if lite: - uk_datasets = { - k: v for k, v in uk_datasets.items() if v.year == 2026 and "frs" in k - } - console.print(f" Lite mode: filtered to {len(uk_datasets)} dataset(s)") - - with logfire.span("seed_uk_datasets", count=len(uk_datasets)): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task("UK datasets", total=len(uk_datasets)) - for _, pe_dataset in uk_datasets.items(): - progress.update(task, description=f"UK: {pe_dataset.name}") - - # Check if dataset already exists - existing = session.exec( - select(Dataset).where(Dataset.name == pe_dataset.name) - ).first() - - if existing: - uk_skipped += 1 - progress.advance(task) - continue - - # Upload to S3 - object_name = upload_dataset_for_seeding(pe_dataset.filepath) - - # Create database record - db_dataset = Dataset( - name=pe_dataset.name, - description=pe_dataset.description, - filepath=object_name, - year=pe_dataset.year, - tax_benefit_model_id=uk_model.id, - ) - session.add(db_dataset) - session.commit() - uk_created += 1 - progress.advance(task) + if config.seed_us: + seed_us_policy(session) - console.print( - f" [green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped" - ) + console.print() - # US datasets - console.print(" Creating US datasets...") - us_datasets = ensure_us_datasets(data_folder=data_folder) - - # In lite mode, only upload CPS 2026 - if lite: - us_datasets = { - k: v for k, v in us_datasets.items() if v.year == 2026 and "cps" in k - } - console.print(f" Lite mode: filtered to {len(us_datasets)} dataset(s)") - - us_created = 0 - us_skipped = 0 - - with logfire.span("seed_us_datasets", count=len(us_datasets)): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task("US datasets", total=len(us_datasets)) - for _, pe_dataset in us_datasets.items(): - progress.update(task, description=f"US: {pe_dataset.name}") - - # Check if dataset already exists - existing = session.exec( - select(Dataset).where(Dataset.name == pe_dataset.name) - ).first() - - if existing: - us_skipped += 1 - progress.advance(task) - continue - - # Upload to S3 - object_name = upload_dataset_for_seeding(pe_dataset.filepath) - - # Create database record - db_dataset = Dataset( - name=pe_dataset.name, - description=pe_dataset.description, - filepath=object_name, - year=pe_dataset.year, - tax_benefit_model_id=us_model.id, - ) - session.add(db_dataset) - session.commit() - us_created += 1 - progress.advance(task) - - console.print( - f" [green]✓[/green] US: {us_created} created, {us_skipped} skipped" - ) - console.print( - f"[green]✓[/green] Seeded {uk_created + us_created} datasets total\n" - ) - - -def seed_example_policies(session): - """Seed example policy reforms for UK and US.""" - with logfire.span("seed_example_policies"): - console.print("[bold blue]Seeding example policies...") - - # Get model versions - uk_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") - ).first() - us_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") - ).first() - - if not uk_model or not us_model: - console.print( - "[red]Error: UK or US model not found. Run seed_model first.[/red]" - ) - return - - uk_version = session.exec( - select(TaxBenefitModelVersion) - .where(TaxBenefitModelVersion.model_id == uk_model.id) - .order_by(TaxBenefitModelVersion.created_at.desc()) - ).first() - - us_version = session.exec( - select(TaxBenefitModelVersion) - .where(TaxBenefitModelVersion.model_id == us_model.id) - .order_by(TaxBenefitModelVersion.created_at.desc()) - ).first() - - # UK example policy: raise basic rate to 22p - uk_policy_name = "UK basic rate 22p" - existing_uk_policy = session.exec( - select(Policy).where(Policy.name == uk_policy_name) - ).first() - - if existing_uk_policy: - console.print(f" Policy '{uk_policy_name}' already exists, skipping") - else: - # Find the basic rate parameter - uk_basic_rate_param = session.exec( - select(Parameter).where( - Parameter.name == "gov.hmrc.income_tax.rates.uk[0].rate", - Parameter.tax_benefit_model_version_id == uk_version.id, - ) - ).first() + # Step 4: Seed regions + if config.seed_regions: + console.print("[bold blue]Step 4: Seeding regions...[/bold blue]\n") - if uk_basic_rate_param: - uk_policy = Policy( - name=uk_policy_name, - description="Raise the UK income tax basic rate from 20p to 22p", - ) - session.add(uk_policy) - session.commit() - session.refresh(uk_policy) - - # Add parameter value (22% = 0.22) - uk_param_value = ParameterValue( - parameter_id=uk_basic_rate_param.id, - value_json={"value": 0.22}, - start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), - end_date=None, - policy_id=uk_policy.id, + if config.seed_us: + console.print("[bold]US Regions[/bold]") + us_created, us_skipped = seed_us_regions( + session, + skip_places=config.skip_places, + skip_districts=config.skip_districts, ) - session.add(uk_param_value) - session.commit() - console.print(f" [green]✓[/green] Created UK policy: {uk_policy_name}") - else: console.print( - " [yellow]Warning: UK basic rate parameter not found[/yellow]" + f"[green]✓[/green] US: {us_created} created, {us_skipped} skipped\n" ) - # US example policy: raise first bracket rate to 12% - us_policy_name = "US 12% lowest bracket" - existing_us_policy = session.exec( - select(Policy).where(Policy.name == us_policy_name) - ).first() - - if existing_us_policy: - console.print(f" Policy '{us_policy_name}' already exists, skipping") - else: - # Find the first bracket rate parameter - us_first_bracket_param = session.exec( - select(Parameter).where( - Parameter.name == "gov.irs.income.bracket.rates.1", - Parameter.tax_benefit_model_version_id == us_version.id, - ) - ).first() - - if us_first_bracket_param: - us_policy = Policy( - name=us_policy_name, - description="Raise US federal income tax lowest bracket to 12%", - ) - session.add(us_policy) - session.commit() - session.refresh(us_policy) - - # Add parameter value (12% = 0.12) - us_param_value = ParameterValue( - parameter_id=us_first_bracket_param.id, - value_json={"value": 0.12}, - start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), - end_date=None, - policy_id=us_policy.id, - ) - session.add(us_param_value) - session.commit() - console.print(f" [green]✓[/green] Created US policy: {us_policy_name}") - else: + if config.seed_uk: + console.print("[bold]UK Regions[/bold]") + uk_created, uk_skipped = seed_uk_regions(session) console.print( - " [yellow]Warning: US first bracket parameter not found[/yellow]" + f"[green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped\n" ) - console.print("[green]✓[/green] Example policies seeded\n") + elapsed = time.time() - start + console.print(f"[bold green]✓ Database seeding complete![/bold green]") + console.print(f"[bold]Total time: {elapsed:.1f}s[/bold]") def main(): - """Main seed function.""" - parser = argparse.ArgumentParser(description="Seed PolicyEngine database") - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: skip US state parameters, only seed FRS 2026 and CPS 2026 datasets", - ) - parser.add_argument( - "--skip-uk-datasets", - action="store_true", - help="Skip UK datasets (useful when HuggingFace token is not available)", + parser = argparse.ArgumentParser( + description="Seed PolicyEngine database", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Presets: + full Everything (default) + lite Both countries, 2026 datasets only, skip state params, core regions + minimal Both countries, 2026 datasets only, skip state params, no policies/regions + uk-lite UK only, 2026 datasets, skip state params + uk-minimal UK only, 2026 datasets, skip state params, no policies/regions + us-lite US only, 2026 datasets, skip state params, core regions only + us-minimal US only, 2026 datasets, skip state params, no policies/regions +""", ) parser.add_argument( - "--skip-regions", - action="store_true", - help="Skip seeding regions", - ) - parser.add_argument( - "--skip-places", - action="store_true", - help="Skip US places (cities over 100K population) when seeding regions", - ) - parser.add_argument( - "--skip-districts", - action="store_true", - help="Skip US congressional districts when seeding regions", + "--preset", + choices=list(PRESETS.keys()), + default="full", + help="Seeding preset (default: full)", ) args = parser.parse_args() - with logfire.span("database_seeding"): - mode_str = " (lite mode)" if args.lite else "" - console.print(f"[bold green]PolicyEngine database seeding{mode_str}[/bold green]\n") + config = PRESETS[args.preset] - with next(get_quiet_session()) as session: - # Seed UK model - uk_version = seed_model(uk_latest, session, lite=args.lite) - console.print(f"[green]✓[/green] UK model seeded: {uk_version.id}\n") + # Build description of what we're doing + countries = [] + if config.seed_uk: + countries.append("UK") + if config.seed_us: + countries.append("US") + country_str = " + ".join(countries) - # Seed US model - us_version = seed_model(us_latest, session, lite=args.lite) - console.print(f"[green]✓[/green] US model seeded: {us_version.id}\n") + year_str = f", {config.dataset_year} only" if config.dataset_year else "" + state_str = ", skip state params" if config.skip_state_params else "" - # Seed datasets - seed_datasets(session, lite=args.lite, skip_uk_datasets=args.skip_uk_datasets) - - # Seed example policies - seed_example_policies(session) - - # Seed regions - if not args.skip_regions: - from seed_regions import seed_us_regions, seed_uk_regions - - console.print("\n[bold]Seeding regions...[/bold]") - us_created, us_skipped = seed_us_regions( - session, - skip_places=args.skip_places, - skip_districts=args.skip_districts, - ) - console.print( - f"[green]✓[/green] US regions: {us_created} created, {us_skipped} skipped" - ) - - if not args.skip_uk_datasets: - uk_created, uk_skipped = seed_uk_regions(session) - console.print( - f"[green]✓[/green] UK regions: {uk_created} created, {uk_skipped} skipped" - ) - - console.print("\n[bold green]✓ Database seeding complete![/bold green]") + console.print( + f"[bold green]PolicyEngine database seeding[/bold green] " + f"[dim](preset: {args.preset})[/dim]\n" + ) + console.print(f" Countries: {country_str}") + console.print(f" Datasets: {'all years' if not config.dataset_year else config.dataset_year}") + if config.skip_state_params: + console.print(" State params: skipped") + console.print(f" Policies: {'yes' if config.seed_policies else 'no'}") + if config.seed_regions: + region_details = [] + if config.skip_places: + region_details.append("no places") + if config.skip_districts: + region_details.append("no districts") + region_str = f"yes ({', '.join(region_details)})" if region_details else "yes (all)" + console.print(f" Regions: {region_str}") + else: + console.print(" Regions: no") + console.print() + + run_seed(config) if __name__ == "__main__": diff --git a/scripts/seed_datasets.py b/scripts/seed_datasets.py new file mode 100644 index 0000000..8a13130 --- /dev/null +++ b/scripts/seed_datasets.py @@ -0,0 +1,227 @@ +"""Seed datasets and upload to S3. + +This script downloads datasets from policyengine.py, uploads them to S3, +and creates database records. + +Usage: + python scripts/seed_datasets.py # Seed UK and US datasets + python scripts/seed_datasets.py --us-only # Seed only US datasets + python scripts/seed_datasets.py --uk-only # Seed only UK datasets + python scripts/seed_datasets.py --year=2026 # Seed only 2026 datasets +""" + +import argparse +from pathlib import Path + +import logfire +from rich.progress import Progress, SpinnerColumn, TextColumn +from sqlmodel import Session, select + +from seed_utils import console, get_session + +# Import after seed_utils sets up path +from policyengine_api.models import Dataset, TaxBenefitModel # noqa: E402 +from policyengine_api.services.storage import upload_dataset_for_seeding # noqa: E402 + + +def seed_uk_datasets(session: Session, year: int | None = None) -> tuple[int, int]: + """Seed UK datasets. + + Args: + session: Database session + year: If specified, only seed datasets for this year + + Returns: + Tuple of (created_count, skipped_count) + """ + from policyengine.tax_benefit_models.uk.datasets import ( + ensure_datasets as ensure_uk_datasets, + ) + + uk_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") + ).first() + + if not uk_model: + console.print("[red]Error: UK model not found. Run seed_models.py first.[/red]") + return 0, 0 + + data_folder = str(Path(__file__).parent.parent / "data") + uk_datasets = ensure_uk_datasets(data_folder=data_folder) + + # Filter by year if specified + if year: + uk_datasets = { + k: v for k, v in uk_datasets.items() if v.year == year and "frs" in k + } + console.print(f" Filtered to {len(uk_datasets)} dataset(s) for year {year}") + + created = 0 + skipped = 0 + + with logfire.span("seed_uk_datasets", count=len(uk_datasets)): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("UK datasets", total=len(uk_datasets)) + for _, pe_dataset in uk_datasets.items(): + progress.update(task, description=f"UK: {pe_dataset.name}") + + # Check if dataset already exists + existing = session.exec( + select(Dataset).where(Dataset.name == pe_dataset.name) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Upload to S3 + object_name = upload_dataset_for_seeding(pe_dataset.filepath) + + # Create database record + db_dataset = Dataset( + name=pe_dataset.name, + description=pe_dataset.description, + filepath=object_name, + year=pe_dataset.year, + tax_benefit_model_id=uk_model.id, + ) + session.add(db_dataset) + session.commit() + created += 1 + progress.advance(task) + + return created, skipped + + +def seed_us_datasets(session: Session, year: int | None = None) -> tuple[int, int]: + """Seed US datasets. + + Args: + session: Database session + year: If specified, only seed datasets for this year + + Returns: + Tuple of (created_count, skipped_count) + """ + from policyengine.tax_benefit_models.us.datasets import ( + ensure_datasets as ensure_us_datasets, + ) + + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not us_model: + console.print("[red]Error: US model not found. Run seed_models.py first.[/red]") + return 0, 0 + + data_folder = str(Path(__file__).parent.parent / "data") + us_datasets = ensure_us_datasets(data_folder=data_folder) + + # Filter by year if specified + if year: + us_datasets = { + k: v for k, v in us_datasets.items() if v.year == year and "cps" in k + } + console.print(f" Filtered to {len(us_datasets)} dataset(s) for year {year}") + + created = 0 + skipped = 0 + + with logfire.span("seed_us_datasets", count=len(us_datasets)): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("US datasets", total=len(us_datasets)) + for _, pe_dataset in us_datasets.items(): + progress.update(task, description=f"US: {pe_dataset.name}") + + # Check if dataset already exists + existing = session.exec( + select(Dataset).where(Dataset.name == pe_dataset.name) + ).first() + + if existing: + skipped += 1 + progress.advance(task) + continue + + # Upload to S3 + object_name = upload_dataset_for_seeding(pe_dataset.filepath) + + # Create database record + db_dataset = Dataset( + name=pe_dataset.name, + description=pe_dataset.description, + filepath=object_name, + year=pe_dataset.year, + tax_benefit_model_id=us_model.id, + ) + session.add(db_dataset) + session.commit() + created += 1 + progress.advance(task) + + return created, skipped + + +def main(): + parser = argparse.ArgumentParser(description="Seed datasets") + parser.add_argument( + "--us-only", + action="store_true", + help="Only seed US datasets", + ) + parser.add_argument( + "--uk-only", + action="store_true", + help="Only seed UK datasets", + ) + parser.add_argument( + "--year", + type=int, + default=None, + help="Only seed datasets for this year (e.g., 2026)", + ) + args = parser.parse_args() + + year_str = f" (year {args.year})" if args.year else "" + console.print(f"[bold green]Seeding datasets{year_str}...[/bold green]\n") + + total_created = 0 + total_skipped = 0 + + with get_session() as session: + if not args.us_only: + console.print("[bold]UK Datasets[/bold]") + uk_created, uk_skipped = seed_uk_datasets(session, year=args.year) + total_created += uk_created + total_skipped += uk_skipped + console.print( + f"[green]✓[/green] UK: {uk_created} created, {uk_skipped} skipped\n" + ) + + if not args.uk_only: + console.print("[bold]US Datasets[/bold]") + us_created, us_skipped = seed_us_datasets(session, year=args.year) + total_created += us_created + total_skipped += us_skipped + console.print( + f"[green]✓[/green] US: {us_created} created, {us_skipped} skipped\n" + ) + + console.print( + f"[bold green]✓ Dataset seeding complete! " + f"{total_created} created, {total_skipped} skipped[/bold green]" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_common.py b/scripts/seed_models.py similarity index 68% rename from scripts/seed_common.py rename to scripts/seed_models.py index 49797cb..d970df9 100644 --- a/scripts/seed_common.py +++ b/scripts/seed_models.py @@ -1,106 +1,59 @@ -"""Shared utilities for seed scripts.""" +"""Seed tax-benefit models with variables and parameters. -import io +This script seeds TaxBenefitModel, TaxBenefitModelVersion, Variables, +Parameters, and ParameterValues from policyengine.py. + +Usage: + python scripts/seed_models.py # Seed UK and US models + python scripts/seed_models.py --us-only # Seed only US model + python scripts/seed_models.py --uk-only # Seed only UK model + python scripts/seed_models.py --skip-state-params # Skip US state parameters +""" + +import argparse import json -import logging import math -import sys -import warnings from datetime import datetime, timezone -from pathlib import Path from uuid import uuid4 import logfire -from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from sqlmodel import Session, create_engine - -# Disable all SQLAlchemy and database logging BEFORE any imports -logging.basicConfig(level=logging.ERROR) -logging.getLogger("sqlalchemy").setLevel(logging.ERROR) -warnings.filterwarnings("ignore") - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from policyengine_api.config.settings import settings # noqa: E402 - -# Configure logfire -if settings.logfire_token: - logfire.configure( - token=settings.logfire_token, - environment=settings.logfire_environment, - console=False, - ) - -console = Console() - - -def get_session(): - """Get database session with logging disabled.""" - engine = create_engine(settings.database_url, echo=False) - return Session(engine) - - -def bulk_insert(session, table: str, columns: list[str], rows: list[dict]): - """Fast bulk insert using PostgreSQL COPY via StringIO.""" - if not rows: - return - - # Get raw psycopg2 connection - connection = session.connection() - raw_conn = connection.connection.dbapi_connection - cursor = raw_conn.cursor() - - # Build CSV-like data in memory - output = io.StringIO() - for row in rows: - values = [] - for col in columns: - val = row[col] - if val is None: - values.append("\\N") - elif isinstance(val, str): - # Escape special characters for COPY - val = ( - val.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n") - ) - values.append(val) - else: - values.append(str(val)) - output.write("\t".join(values) + "\n") +from sqlmodel import Session, select - output.seek(0) +from seed_utils import bulk_insert, console, get_session - # COPY is the fastest way to bulk load PostgreSQL - cursor.copy_from(output, table, columns=columns, null="\\N") - session.commit() +# Import after seed_utils sets up path +from policyengine_api.models import ( # noqa: E402 + Parameter, + ParameterValue, + TaxBenefitModel, + TaxBenefitModelVersion, +) -def seed_model(model_version, session, lite: bool = False): +def seed_model( + model_version, + session: Session, + skip_state_params: bool = False, +) -> TaxBenefitModelVersion: """Seed a tax-benefit model with its variables and parameters. Args: - model_version: The policyengine package model version + model_version: The policyengine.py model version object session: Database session - lite: If True, skip state-level parameters + skip_state_params: Skip US state-level parameters (gov.states.*) - Returns the TaxBenefitModelVersion that was created or found. + Returns: + The created or existing TaxBenefitModelVersion """ - from policyengine_api.models import ( - TaxBenefitModel, - TaxBenefitModelVersion, - ) - from sqlmodel import select - with logfire.span( "seed_model", model=model_version.model.id, version=model_version.version, ): - # Create or get the model console.print(f"[bold blue]Seeding {model_version.model.id}...") + # Create or get the model existing_model = session.exec( select(TaxBenefitModel).where( TaxBenefitModel.name == model_version.model.id @@ -157,10 +110,6 @@ def seed_model(model_version, session, lite: bool = False): total=len(model_version.variables), ) for var in model_version.variables: - # default_value is pre-serialized by policyengine.py: - # - Enum values are converted to their name (e.g., "SINGLE") - # - datetime.date values are converted to ISO format - # - Primitives (bool, int, float, str) are kept as-is var_rows.append( { "id": uuid4(), @@ -171,7 +120,6 @@ def seed_model(model_version, session, lite: bool = False): if hasattr(var.data_type, "__name__") else str(var.data_type), "possible_values": None, - "default_value": json.dumps(var.default_value), "tax_benefit_model_version_id": db_version.id, "created_at": datetime.now(timezone.utc), } @@ -189,7 +137,6 @@ def seed_model(model_version, session, lite: bool = False): "description", "data_type", "possible_values", - "default_value", "tax_benefit_model_version_id", "created_at", ], @@ -200,43 +147,30 @@ def seed_model(model_version, session, lite: bool = False): f" [green]✓[/green] Added {len(model_version.variables)} variables" ) - # Add parameters - deduplicate by name (keep first occurrence) - # - # WHY DEDUPLICATION IS NEEDED: - # The policyengine package can provide multiple parameter entries with the same - # name. This happens because parameters can have multiple bracket entries or - # state-specific variants that share the same base name. We keep only the first - # occurrence to avoid database unique constraint violations and reduce redundancy. - # - # NOTE: We do NOT filter by label. Parameters without labels (bracket params, - # breakdown params) are still valid and needed for policy analysis. - # - # In lite mode, exclude US state parameters (gov.states.*) + # Add parameters (only user-facing ones: those with labels) + # Deduplicate by name - keep first occurrence seen_names = set() parameters_to_add = [] - skipped_state_params = 0 - skipped_duplicate = 0 - + skipped_state_params_count = 0 for p in model_version.parameters: - if p.name in seen_names: - skipped_duplicate += 1 + if p.label is None or p.name in seen_names: continue - # In lite mode, skip state-level parameters for faster seeding - if lite and p.name.startswith("gov.states."): - skipped_state_params += 1 + # Skip state-level parameters if requested + if skip_state_params and p.name.startswith("gov.states."): + skipped_state_params_count += 1 continue parameters_to_add.append(p) seen_names.add(p.name) - console.print(f" Parameter filtering:") - console.print(f" - Total from source: {len(model_version.parameters)}") - console.print(f" - Skipped (duplicate name): {skipped_duplicate}") - if lite and skipped_state_params > 0: - console.print(f" - Skipped (state params, lite mode): {skipped_state_params}") - console.print(f" - To add: {len(parameters_to_add)}") + filter_msg = f" Filtered to {len(parameters_to_add)} user-facing parameters" + filter_msg += ( + f" (from {len(model_version.parameters)} total, deduplicated by name)" + ) + if skip_state_params and skipped_state_params_count > 0: + filter_msg += f", skipped {skipped_state_params_count} state params" + console.print(filter_msg) with logfire.span("add_parameters", count=len(parameters_to_add)): - # Build list of parameter dicts for bulk insert param_rows = [] param_names = [] # Track (pe_id, name, generated_uuid) @@ -293,7 +227,6 @@ def seed_model(model_version, session, lite: bool = False): ) # Add parameter values - # Filter to only include values for parameters we added parameter_values_to_add = [ pv for pv in model_version.parameter_values @@ -324,7 +257,6 @@ def seed_model(model_version, session, lite: bool = False): continue # Source data has dates swapped (start > end), fix ordering - # Only swap if both dates are set, otherwise keep original if pv.start_date and pv.end_date: start = pv.end_date # Swap: source end is our start end = pv.start_date # Swap: source start is our end @@ -368,3 +300,56 @@ def seed_model(model_version, session, lite: bool = False): ) return db_version + + +def seed_uk_model(session: Session, skip_state_params: bool = False): + """Seed UK model.""" + from policyengine.tax_benefit_models.uk import uk_latest + + version = seed_model(uk_latest, session, skip_state_params=skip_state_params) + console.print(f"[green]✓[/green] UK model seeded: {version.id}\n") + return version + + +def seed_us_model(session: Session, skip_state_params: bool = False): + """Seed US model.""" + from policyengine.tax_benefit_models.us import us_latest + + version = seed_model(us_latest, session, skip_state_params=skip_state_params) + console.print(f"[green]✓[/green] US model seeded: {version.id}\n") + return version + + +def main(): + parser = argparse.ArgumentParser(description="Seed tax-benefit models") + parser.add_argument( + "--us-only", + action="store_true", + help="Only seed US model", + ) + parser.add_argument( + "--uk-only", + action="store_true", + help="Only seed UK model", + ) + parser.add_argument( + "--skip-state-params", + action="store_true", + help="Skip US state-level parameters (gov.states.*)", + ) + args = parser.parse_args() + + console.print("[bold green]Seeding tax-benefit models...[/bold green]\n") + + with get_session() as session: + if not args.us_only: + seed_uk_model(session, skip_state_params=args.skip_state_params) + + if not args.uk_only: + seed_us_model(session, skip_state_params=args.skip_state_params) + + console.print("[bold green]✓ Model seeding complete![/bold green]") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_policies.py b/scripts/seed_policies.py index e57b964..c3212d6 100644 --- a/scripts/seed_policies.py +++ b/scripts/seed_policies.py @@ -1,142 +1,191 @@ -"""Seed example policy reforms for UK and US.""" +"""Seed example policy reforms. -import time +This script creates example policy reforms for UK and US models. + +Usage: + python scripts/seed_policies.py # Seed UK and US example policies + python scripts/seed_policies.py --us-only # Seed only US example policy + python scripts/seed_policies.py --uk-only # Seed only UK example policy +""" + +import argparse from datetime import datetime, timezone import logfire -from sqlmodel import select - -from seed_common import console, get_session +from sqlmodel import Session, select + +from seed_utils import console, get_session + +# Import after seed_utils sets up path +from policyengine_api.models import ( # noqa: E402 + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) + + +def seed_uk_policy(session: Session) -> bool: + """Seed UK example policy: raise basic rate to 22p. + + Returns: + True if created, False if skipped + """ + uk_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") + ).first() + + if not uk_model: + console.print("[red]Error: UK model not found. Run seed_models.py first.[/red]") + return False + + uk_version = session.exec( + select(TaxBenefitModelVersion) + .where(TaxBenefitModelVersion.model_id == uk_model.id) + .order_by(TaxBenefitModelVersion.created_at.desc()) + ).first() + + if not uk_version: + console.print( + "[red]Error: UK model version not found. Run seed_models.py first.[/red]" + ) + return False + + policy_name = "UK basic rate 22p" + existing = session.exec(select(Policy).where(Policy.name == policy_name)).first() + + if existing: + console.print(f" Policy '{policy_name}' already exists, skipping") + return False + + # Find the basic rate parameter + uk_basic_rate_param = session.exec( + select(Parameter).where( + Parameter.name == "gov.hmrc.income_tax.rates.uk[0].rate", + Parameter.tax_benefit_model_version_id == uk_version.id, + ) + ).first() + + if not uk_basic_rate_param: + console.print(" [yellow]Warning: UK basic rate parameter not found[/yellow]") + return False + + uk_policy = Policy( + name=policy_name, + description="Raise the UK income tax basic rate from 20p to 22p", + ) + session.add(uk_policy) + session.commit() + session.refresh(uk_policy) + + # Add parameter value (22% = 0.22) + uk_param_value = ParameterValue( + parameter_id=uk_basic_rate_param.id, + value_json={"value": 0.22}, + start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), + end_date=None, + policy_id=uk_policy.id, + ) + session.add(uk_param_value) + session.commit() + console.print(f" [green]✓[/green] Created UK policy: {policy_name}") + return True + + +def seed_us_policy(session: Session) -> bool: + """Seed US example policy: raise first bracket to 12%. + + Returns: + True if created, False if skipped + """ + us_model = session.exec( + select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") + ).first() + + if not us_model: + console.print("[red]Error: US model not found. Run seed_models.py first.[/red]") + return False + + us_version = session.exec( + select(TaxBenefitModelVersion) + .where(TaxBenefitModelVersion.model_id == us_model.id) + .order_by(TaxBenefitModelVersion.created_at.desc()) + ).first() + + if not us_version: + console.print( + "[red]Error: US model version not found. Run seed_models.py first.[/red]" + ) + return False + + policy_name = "US 12% lowest bracket" + existing = session.exec(select(Policy).where(Policy.name == policy_name)).first() + + if existing: + console.print(f" Policy '{policy_name}' already exists, skipping") + return False + + # Find the first bracket rate parameter + us_first_bracket_param = session.exec( + select(Parameter).where( + Parameter.name == "gov.irs.income.bracket.rates.1", + Parameter.tax_benefit_model_version_id == us_version.id, + ) + ).first() + + if not us_first_bracket_param: + console.print( + " [yellow]Warning: US first bracket parameter not found[/yellow]" + ) + return False + + us_policy = Policy( + name=policy_name, + description="Raise US federal income tax lowest bracket to 12%", + ) + session.add(us_policy) + session.commit() + session.refresh(us_policy) + + # Add parameter value (12% = 0.12) + us_param_value = ParameterValue( + parameter_id=us_first_bracket_param.id, + value_json={"value": 0.12}, + start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), + end_date=None, + policy_id=us_policy.id, + ) + session.add(us_param_value) + session.commit() + console.print(f" [green]✓[/green] Created US policy: {policy_name}") + return True def main(): - from policyengine_api.models import ( - Parameter, - ParameterValue, - Policy, - TaxBenefitModel, - TaxBenefitModelVersion, + parser = argparse.ArgumentParser(description="Seed example policies") + parser.add_argument( + "--us-only", + action="store_true", + help="Only seed US example policy", + ) + parser.add_argument( + "--uk-only", + action="store_true", + help="Only seed UK example policy", ) + args = parser.parse_args() console.print("[bold green]Seeding example policies...[/bold green]\n") - start = time.time() with get_session() as session: - with logfire.span("seed_example_policies"): - # Get model versions - uk_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") - ).first() - us_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") - ).first() - - if not uk_model or not us_model: - console.print( - "[red]Error: UK or US model not found. Run seed_*_model.py first.[/red]" - ) - return - - uk_version = session.exec( - select(TaxBenefitModelVersion) - .where(TaxBenefitModelVersion.model_id == uk_model.id) - .order_by(TaxBenefitModelVersion.created_at.desc()) - ).first() - - us_version = session.exec( - select(TaxBenefitModelVersion) - .where(TaxBenefitModelVersion.model_id == us_model.id) - .order_by(TaxBenefitModelVersion.created_at.desc()) - ).first() - - # UK example policy: raise basic rate to 22p - uk_policy_name = "UK basic rate 22p" - existing_uk_policy = session.exec( - select(Policy).where(Policy.name == uk_policy_name) - ).first() - - if existing_uk_policy: - console.print(f" Policy '{uk_policy_name}' already exists, skipping") - else: - # Find the basic rate parameter - uk_basic_rate_param = session.exec( - select(Parameter).where( - Parameter.name == "gov.hmrc.income_tax.rates.uk[0].rate", - Parameter.tax_benefit_model_version_id == uk_version.id, - ) - ).first() - - if uk_basic_rate_param: - uk_policy = Policy( - name=uk_policy_name, - description="Raise the UK income tax basic rate from 20p to 22p", - ) - session.add(uk_policy) - session.commit() - session.refresh(uk_policy) - - # Add parameter value (22% = 0.22) - uk_param_value = ParameterValue( - parameter_id=uk_basic_rate_param.id, - value_json={"value": 0.22}, - start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), - end_date=None, - policy_id=uk_policy.id, - ) - session.add(uk_param_value) - session.commit() - console.print(f" [green]✓[/green] Created UK policy: {uk_policy_name}") - else: - console.print( - " [yellow]Warning: UK basic rate parameter not found[/yellow]" - ) - - # US example policy: raise first bracket rate to 12% - us_policy_name = "US 12% lowest bracket" - existing_us_policy = session.exec( - select(Policy).where(Policy.name == us_policy_name) - ).first() - - if existing_us_policy: - console.print(f" Policy '{us_policy_name}' already exists, skipping") - else: - # Find the first bracket rate parameter - us_first_bracket_param = session.exec( - select(Parameter).where( - Parameter.name == "gov.irs.income.bracket.rates.1", - Parameter.tax_benefit_model_version_id == us_version.id, - ) - ).first() - - if us_first_bracket_param: - us_policy = Policy( - name=us_policy_name, - description="Raise US federal income tax lowest bracket to 12%", - ) - session.add(us_policy) - session.commit() - session.refresh(us_policy) - - # Add parameter value (12% = 0.12) - us_param_value = ParameterValue( - parameter_id=us_first_bracket_param.id, - value_json={"value": 0.12}, - start_date=datetime(2024, 1, 1, tzinfo=timezone.utc), - end_date=None, - policy_id=us_policy.id, - ) - session.add(us_param_value) - session.commit() - console.print(f" [green]✓[/green] Created US policy: {us_policy_name}") - else: - console.print( - " [yellow]Warning: US first bracket parameter not found[/yellow]" - ) - - console.print("[green]✓[/green] Example policies seeded") - - elapsed = time.time() - start - console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") + if not args.us_only: + seed_uk_policy(session) + + if not args.uk_only: + seed_us_policy(session) + + console.print("\n[bold green]✓ Policy seeding complete![/bold green]") if __name__ == "__main__": diff --git a/scripts/seed_regions.py b/scripts/seed_regions.py index c8cc9d8..060fb2f 100644 --- a/scripts/seed_regions.py +++ b/scripts/seed_regions.py @@ -16,27 +16,15 @@ """ import argparse -import sys import time -from pathlib import Path -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from sqlmodel import Session, create_engine, select - -from policyengine_api.config.settings import settings -from policyengine_api.models import Dataset, Region, TaxBenefitModel - -console = Console() +from sqlmodel import Session, select +from seed_utils import console, get_session -def get_session() -> Session: - """Get database session.""" - engine = create_engine(settings.database_url) - return Session(engine) +# Import after seed_utils sets up path +from policyengine_api.models import Dataset, Region, TaxBenefitModel # noqa: E402 def seed_us_regions( diff --git a/scripts/seed_uk_datasets.py b/scripts/seed_uk_datasets.py deleted file mode 100644 index 1754454..0000000 --- a/scripts/seed_uk_datasets.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Seed UK datasets (FRS) and upload to S3. - -NOTE: Requires HUGGING_FACE_TOKEN environment variable to be set, -as UK FRS datasets are hosted on a private HuggingFace repository. -""" - -import argparse -import time -from pathlib import Path - -import logfire -from rich.progress import Progress, SpinnerColumn, TextColumn -from sqlmodel import select - -from seed_common import console, get_session - - -def main(): - parser = argparse.ArgumentParser(description="Seed UK datasets") - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: only seed FRS 2026", - ) - args = parser.parse_args() - - # Import here to avoid slow import at module level - from policyengine.tax_benefit_models.uk.datasets import ( - ensure_datasets as ensure_uk_datasets, - ) - - from policyengine_api.models import Dataset, TaxBenefitModel - from policyengine_api.services.storage import upload_dataset_for_seeding - - console.print("[bold green]Seeding UK datasets...[/bold green]\n") - console.print("[yellow]Note: Requires HUGGING_FACE_TOKEN environment variable[/yellow]\n") - - start = time.time() - with get_session() as session: - # Get UK model - uk_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-uk") - ).first() - - if not uk_model: - console.print("[red]Error: UK model not found. Run seed_uk_model.py first.[/red]") - return - - data_folder = str(Path(__file__).parent.parent / "data") - console.print(f" Data folder: {data_folder}") - - # Get datasets - console.print(" Loading UK datasets from policyengine package...") - ds_start = time.time() - uk_datasets = ensure_uk_datasets(data_folder=data_folder) - console.print(f" Loaded {len(uk_datasets)} datasets in {time.time() - ds_start:.1f}s") - - # In lite mode, only upload FRS 2026 - if args.lite: - uk_datasets = { - k: v for k, v in uk_datasets.items() if v.year == 2026 and "frs" in k - } - console.print(f" Lite mode: filtered to {len(uk_datasets)} dataset(s)") - - created = 0 - skipped = 0 - - with logfire.span("seed_uk_datasets", count=len(uk_datasets)): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task("UK datasets", total=len(uk_datasets)) - for name, pe_dataset in uk_datasets.items(): - progress.update(task, description=f"UK: {pe_dataset.name}") - - # Check if dataset already exists - existing = session.exec( - select(Dataset).where(Dataset.name == pe_dataset.name) - ).first() - - if existing: - skipped += 1 - progress.advance(task) - continue - - # Upload to S3 - upload_start = time.time() - object_name = upload_dataset_for_seeding(pe_dataset.filepath) - console.print(f" Uploaded {pe_dataset.name} in {time.time() - upload_start:.1f}s") - - # Create database record - db_dataset = Dataset( - name=pe_dataset.name, - description=pe_dataset.description, - filepath=object_name, - year=pe_dataset.year, - tax_benefit_model_id=uk_model.id, - ) - session.add(db_dataset) - session.commit() - created += 1 - progress.advance(task) - - console.print(f"[green]✓[/green] UK datasets: {created} created, {skipped} skipped") - - elapsed = time.time() - start - console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") - - -if __name__ == "__main__": - main() diff --git a/scripts/seed_uk_model.py b/scripts/seed_uk_model.py deleted file mode 100644 index 07543bf..0000000 --- a/scripts/seed_uk_model.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Seed UK model (variables, parameters, parameter values).""" - -import argparse -import time - -from seed_common import console, get_session, seed_model - - -def main(): - parser = argparse.ArgumentParser(description="Seed UK model") - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: skip state parameters", - ) - args = parser.parse_args() - - # Import here to avoid slow import at module level - from policyengine.tax_benefit_models.uk import uk_latest - - console.print("[bold green]Seeding UK model...[/bold green]\n") - - start = time.time() - with get_session() as session: - version = seed_model(uk_latest, session, lite=args.lite) - console.print(f"[green]✓[/green] UK model seeded: {version.id}") - - elapsed = time.time() - start - console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") - - -if __name__ == "__main__": - main() diff --git a/scripts/seed_us_datasets.py b/scripts/seed_us_datasets.py deleted file mode 100644 index abf1995..0000000 --- a/scripts/seed_us_datasets.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Seed US datasets (CPS) and upload to S3.""" - -import argparse -import time -from pathlib import Path - -import logfire -from rich.progress import Progress, SpinnerColumn, TextColumn -from sqlmodel import select - -from seed_common import console, get_session - - -def main(): - parser = argparse.ArgumentParser(description="Seed US datasets") - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: only seed CPS 2026", - ) - args = parser.parse_args() - - # Import here to avoid slow import at module level - from policyengine.tax_benefit_models.us.datasets import ( - ensure_datasets as ensure_us_datasets, - ) - - from policyengine_api.models import Dataset, TaxBenefitModel - from policyengine_api.services.storage import upload_dataset_for_seeding - - console.print("[bold green]Seeding US datasets...[/bold green]\n") - - start = time.time() - with get_session() as session: - # Get US model - us_model = session.exec( - select(TaxBenefitModel).where(TaxBenefitModel.name == "policyengine-us") - ).first() - - if not us_model: - console.print("[red]Error: US model not found. Run seed_us_model.py first.[/red]") - return - - data_folder = str(Path(__file__).parent.parent / "data") - console.print(f" Data folder: {data_folder}") - - # Get datasets - console.print(" Loading US datasets from policyengine package...") - ds_start = time.time() - us_datasets = ensure_us_datasets(data_folder=data_folder) - console.print(f" Loaded {len(us_datasets)} datasets in {time.time() - ds_start:.1f}s") - - # In lite mode, only upload CPS 2026 - if args.lite: - us_datasets = { - k: v for k, v in us_datasets.items() if v.year == 2026 and "cps" in k - } - console.print(f" Lite mode: filtered to {len(us_datasets)} dataset(s)") - - created = 0 - skipped = 0 - - with logfire.span("seed_us_datasets", count=len(us_datasets)): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console, - ) as progress: - task = progress.add_task("US datasets", total=len(us_datasets)) - for name, pe_dataset in us_datasets.items(): - progress.update(task, description=f"US: {pe_dataset.name}") - - # Check if dataset already exists - existing = session.exec( - select(Dataset).where(Dataset.name == pe_dataset.name) - ).first() - - if existing: - skipped += 1 - progress.advance(task) - continue - - # Upload to S3 - upload_start = time.time() - object_name = upload_dataset_for_seeding(pe_dataset.filepath) - console.print(f" Uploaded {pe_dataset.name} in {time.time() - upload_start:.1f}s") - - # Create database record - db_dataset = Dataset( - name=pe_dataset.name, - description=pe_dataset.description, - filepath=object_name, - year=pe_dataset.year, - tax_benefit_model_id=us_model.id, - ) - session.add(db_dataset) - session.commit() - created += 1 - progress.advance(task) - - console.print(f"[green]✓[/green] US datasets: {created} created, {skipped} skipped") - - elapsed = time.time() - start - console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") - - -if __name__ == "__main__": - main() diff --git a/scripts/seed_us_model.py b/scripts/seed_us_model.py deleted file mode 100644 index ce8a829..0000000 --- a/scripts/seed_us_model.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Seed US model (variables, parameters, parameter values).""" - -import argparse -import time - -from seed_common import console, get_session, seed_model - - -def main(): - parser = argparse.ArgumentParser(description="Seed US model") - parser.add_argument( - "--lite", - action="store_true", - help="Lite mode: skip state parameters", - ) - args = parser.parse_args() - - # Import here to avoid slow import at module level - from policyengine.tax_benefit_models.us import us_latest - - console.print("[bold green]Seeding US model...[/bold green]\n") - - start = time.time() - with get_session() as session: - version = seed_model(us_latest, session, lite=args.lite) - console.print(f"[green]✓[/green] US model seeded: {version.id}") - - elapsed = time.time() - start - console.print(f"\n[bold]Total time: {elapsed:.1f}s[/bold]") - - -if __name__ == "__main__": - main() diff --git a/scripts/seed_utils.py b/scripts/seed_utils.py new file mode 100644 index 0000000..624379f --- /dev/null +++ b/scripts/seed_utils.py @@ -0,0 +1,72 @@ +"""Shared utilities for seed scripts.""" + +import io +import logging +import sys +import warnings +from pathlib import Path + +import logfire +from rich.console import Console +from sqlmodel import Session, create_engine + +# Disable all SQLAlchemy and database logging +logging.basicConfig(level=logging.ERROR) +logging.getLogger("sqlalchemy").setLevel(logging.ERROR) +warnings.filterwarnings("ignore") + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from policyengine_api.config.settings import settings # noqa: E402 + +# Configure logfire +if settings.logfire_token: + logfire.configure( + token=settings.logfire_token, + environment=settings.logfire_environment, + console=False, + ) + +console = Console() + + +def get_session() -> Session: + """Get database session with logging disabled.""" + engine = create_engine(settings.database_url, echo=False) + return Session(engine) + + +def bulk_insert(session: Session, table: str, columns: list[str], rows: list[dict]): + """Fast bulk insert using PostgreSQL COPY via StringIO.""" + if not rows: + return + + # Get raw psycopg2 connection + connection = session.connection() + raw_conn = connection.connection.dbapi_connection + cursor = raw_conn.cursor() + + # Build CSV-like data in memory + output = io.StringIO() + for row in rows: + values = [] + for col in columns: + val = row[col] + if val is None: + values.append("\\N") + elif isinstance(val, str): + # Escape special characters for COPY + val = ( + val.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n") + ) + values.append(val) + else: + values.append(str(val)) + output.write("\t".join(values) + "\n") + + output.seek(0) + + # COPY is the fastest way to bulk load PostgreSQL + cursor.copy_from(output, table, columns=columns, null="\\N") + session.commit() From f48fe17492bb14a2975b981ea3a8e68f06442d45 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 17 Feb 2026 00:18:44 +0100 Subject: [PATCH 27/87] fix: Update region tests to pass simulation_type argument Tests were written before _get_or_create_simulation and _get_deterministic_simulation_id gained the simulation_type parameter. Add SimulationType.ECONOMY and use keyword args for dataset_id/filter params to match the current function signatures. Co-Authored-By: Claude Opus 4.6 --- ...isting_simulation__then_reuses_existing.py | 9 ++- ...ith_filter__then_filter_params_included.py | 3 + ...iven_same_params__then_deterministic_id.py | 65 ++++++++++++++----- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/tests/test__given_existing_simulation__then_reuses_existing.py b/tests/test__given_existing_simulation__then_reuses_existing.py index 77731f1..09f4df0 100644 --- a/tests/test__given_existing_simulation__then_reuses_existing.py +++ b/tests/test__given_existing_simulation__then_reuses_existing.py @@ -8,7 +8,7 @@ from sqlmodel import Session from policyengine_api.api.analysis import _get_or_create_simulation -from policyengine_api.models import SimulationStatus +from policyengine_api.models import SimulationStatus, SimulationType from test_fixtures.fixtures_regions import ( create_dataset, create_simulation, @@ -29,6 +29,7 @@ def test_given_existing_simulation_with_filter_then_reuses(self, session: Sessio # Create initial simulation with filter params first_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -40,6 +41,7 @@ def test_given_existing_simulation_with_filter_then_reuses(self, session: Sessio # When - request same simulation again second_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -61,6 +63,7 @@ def test_given_different_filter_then_creates_new_simulation(self, session: Sessi # Create simulation for England england_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -72,6 +75,7 @@ def test_given_different_filter_then_creates_new_simulation(self, session: Sessi # When - request simulation for Scotland scotland_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -97,6 +101,7 @@ def test_given_no_filter_vs_filter_then_creates_separate_simulations( # Create national (no filter) simulation national_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -108,6 +113,7 @@ def test_given_no_filter_vs_filter_then_creates_separate_simulations( # When - request filtered simulation filtered_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -131,6 +137,7 @@ def test_given_new_simulation_then_status_is_pending(self, session: Session): # When simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, diff --git a/tests/test__given_region_with_filter__then_filter_params_included.py b/tests/test__given_region_with_filter__then_filter_params_included.py index c84a372..bc1d862 100644 --- a/tests/test__given_region_with_filter__then_filter_params_included.py +++ b/tests/test__given_region_with_filter__then_filter_params_included.py @@ -13,6 +13,7 @@ _get_or_create_simulation, _resolve_dataset_and_region, ) +from policyengine_api.models import SimulationType from test_fixtures.fixtures_regions import ( create_dataset, create_region, @@ -134,6 +135,7 @@ def test_given_filter_params_then_simulation_has_filter_fields( # When simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, @@ -158,6 +160,7 @@ def test_given_no_filter_params_then_simulation_has_null_filter_fields( # When simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, dataset_id=dataset.id, model_version_id=model_version.id, policy_id=None, diff --git a/tests/test__given_same_params__then_deterministic_id.py b/tests/test__given_same_params__then_deterministic_id.py index d393f53..cbac11c 100644 --- a/tests/test__given_same_params__then_deterministic_id.py +++ b/tests/test__given_same_params__then_deterministic_id.py @@ -10,6 +10,7 @@ import pytest from policyengine_api.api.analysis import _get_deterministic_simulation_id +from policyengine_api.models import SimulationType class TestDeterministicSimulationId: @@ -27,20 +28,22 @@ def test_given_same_params_then_same_id_returned(self): # When id1 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, - filter_field, - filter_value, + dataset_id=dataset_id, + filter_field=filter_field, + filter_value=filter_value, ) id2 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, - filter_field, - filter_value, + dataset_id=dataset_id, + filter_field=filter_field, + filter_value=filter_value, ) # Then @@ -56,18 +59,20 @@ def test_given_different_filter_field_then_different_id(self): # When id1 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field="country", filter_value="ENGLAND", ) id2 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field="state_code", filter_value="ENGLAND", ) @@ -85,18 +90,20 @@ def test_given_different_filter_value_then_different_id(self): # When id1 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field="country", filter_value="ENGLAND", ) id2 = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field="country", filter_value="SCOTLAND", ) @@ -114,18 +121,20 @@ def test_given_filter_none_vs_filter_set_then_different_id(self): # When id_no_filter = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field=None, filter_value=None, ) id_with_filter = _get_deterministic_simulation_id( - dataset_id, + SimulationType.ECONOMY, model_version_id, policy_id, dynamic_id, + dataset_id=dataset_id, filter_field="country", filter_value="ENGLAND", ) @@ -144,10 +153,22 @@ def test_given_different_dataset_then_different_id(self): # When id1 = _get_deterministic_simulation_id( - uuid4(), model_version_id, policy_id, dynamic_id, filter_field, filter_value + SimulationType.ECONOMY, + model_version_id, + policy_id, + dynamic_id, + dataset_id=uuid4(), + filter_field=filter_field, + filter_value=filter_value, ) id2 = _get_deterministic_simulation_id( - uuid4(), model_version_id, policy_id, dynamic_id, filter_field, filter_value + SimulationType.ECONOMY, + model_version_id, + policy_id, + dynamic_id, + dataset_id=uuid4(), + filter_field=filter_field, + filter_value=filter_value, ) # Then @@ -161,10 +182,22 @@ def test_given_null_optional_params_then_consistent_id(self): # When id1 = _get_deterministic_simulation_id( - dataset_id, model_version_id, None, None, None, None + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field=None, + filter_value=None, ) id2 = _get_deterministic_simulation_id( - dataset_id, model_version_id, None, None, None, None + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field=None, + filter_value=None, ) # Then From cdcfd72c86006f6163619ab53090bb9fd7f00918 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 17 Feb 2026 01:00:38 +0100 Subject: [PATCH 28/87] refactor: Consolidate region test files into test_analysis.py Merge 6 separate test__given_* files into test_analysis.py organized by function tested: TestResolveDatasetAndRegion, TestGetDeterministicSimulationId, TestGetOrCreateSimulation. Fix pre-existing test_missing_dataset_id assertion (400 not 422). Move @pytest.mark.integration from file-level to class-level. Co-Authored-By: Claude Opus 4.6 --- ...__given_dataset_id__then_region_is_none.py | 94 --- ...isting_simulation__then_reuses_existing.py | 151 ---- ...t__given_invalid_region__then_404_error.py | 107 --- ...ith_filter__then_filter_params_included.py | 173 ----- ...without_filter__then_filter_params_none.py | 110 --- ...iven_same_params__then_deterministic_id.py | 204 ------ tests/test_analysis.py | 679 +++++++++++++++++- 7 files changed, 661 insertions(+), 857 deletions(-) delete mode 100644 tests/test__given_dataset_id__then_region_is_none.py delete mode 100644 tests/test__given_existing_simulation__then_reuses_existing.py delete mode 100644 tests/test__given_invalid_region__then_404_error.py delete mode 100644 tests/test__given_region_with_filter__then_filter_params_included.py delete mode 100644 tests/test__given_region_without_filter__then_filter_params_none.py delete mode 100644 tests/test__given_same_params__then_deterministic_id.py diff --git a/tests/test__given_dataset_id__then_region_is_none.py b/tests/test__given_dataset_id__then_region_is_none.py deleted file mode 100644 index ee3c1d5..0000000 --- a/tests/test__given_dataset_id__then_region_is_none.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Tests for dataset resolution when dataset_id is provided directly. - -When a dataset_id is provided instead of a region code, -the resolved region should be None. -""" - -import pytest -from sqlmodel import Session - -from policyengine_api.api.analysis import ( - EconomicImpactRequest, - _resolve_dataset_and_region, -) -from test_fixtures.fixtures_regions import ( - create_dataset, - create_tax_benefit_model, -) - - -class TestResolveDatasetWithDatasetId: - """Tests for _resolve_dataset_and_region when dataset_id is provided.""" - - def test_given_dataset_id_then_region_is_none(self, session: Session): - """Given a dataset_id, then region is None in the response.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - dataset_id=dataset.id, - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_region is None - - def test_given_dataset_id_then_dataset_is_returned(self, session: Session): - """Given a dataset_id, then the correct dataset is returned.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - dataset_id=dataset.id, - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_dataset.id == dataset.id - assert resolved_dataset.name == "uk_enhanced_frs" - - def test_given_dataset_id_and_region_then_region_takes_precedence( - self, session: Session - ): - """Given both dataset_id and region, then region takes precedence.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset1 = create_dataset(session, model, name="dataset_from_id") - dataset2 = create_dataset(session, model, name="dataset_from_region") - from test_fixtures.fixtures_regions import create_region - - region = create_region( - session, - model=model, - dataset=dataset2, - code="uk", - label="United Kingdom", - region_type="national", - requires_filter=False, - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - dataset_id=dataset1.id, - region="uk", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - # Region code takes precedence, so we get dataset2 - assert resolved_dataset.id == dataset2.id - assert resolved_region is not None - assert resolved_region.code == "uk" diff --git a/tests/test__given_existing_simulation__then_reuses_existing.py b/tests/test__given_existing_simulation__then_reuses_existing.py deleted file mode 100644 index 09f4df0..0000000 --- a/tests/test__given_existing_simulation__then_reuses_existing.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Tests for simulation reuse with filter parameters. - -When a simulation with the same parameters already exists, -it should be reused instead of creating a new one. -""" - -import pytest -from sqlmodel import Session - -from policyengine_api.api.analysis import _get_or_create_simulation -from policyengine_api.models import SimulationStatus, SimulationType -from test_fixtures.fixtures_regions import ( - create_dataset, - create_simulation, - create_tax_benefit_model, - create_tax_benefit_model_version, -) - - -class TestSimulationReuse: - """Tests for simulation reuse behavior.""" - - def test_given_existing_simulation_with_filter_then_reuses(self, session: Session): - """Given an existing simulation with filter params, then it is reused.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # Create initial simulation with filter params - first_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # When - request same simulation again - second_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # Then - assert first_sim.id == second_sim.id - - def test_given_different_filter_then_creates_new_simulation(self, session: Session): - """Given different filter params, then a new simulation is created.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # Create simulation for England - england_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # When - request simulation for Scotland - scotland_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="SCOTLAND", - ) - - # Then - assert england_sim.id != scotland_sim.id - assert england_sim.filter_value == "ENGLAND" - assert scotland_sim.filter_value == "SCOTLAND" - - def test_given_no_filter_vs_filter_then_creates_separate_simulations( - self, session: Session - ): - """Given national vs filtered, then separate simulations are created.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # Create national (no filter) simulation - national_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field=None, - filter_value=None, - ) - - # When - request filtered simulation - filtered_sim = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # Then - assert national_sim.id != filtered_sim.id - assert national_sim.filter_field is None - assert filtered_sim.filter_field == "country" - - def test_given_new_simulation_then_status_is_pending(self, session: Session): - """Given a new simulation request, then status is PENDING.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # When - simulation = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # Then - assert simulation.status == SimulationStatus.PENDING diff --git a/tests/test__given_invalid_region__then_404_error.py b/tests/test__given_invalid_region__then_404_error.py deleted file mode 100644 index 562a5c9..0000000 --- a/tests/test__given_invalid_region__then_404_error.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Tests for region resolution error cases. - -When an invalid region code is provided or required parameters are missing, -appropriate HTTP errors should be raised. -""" - -import pytest -from fastapi import HTTPException -from sqlmodel import Session - -from policyengine_api.api.analysis import ( - EconomicImpactRequest, - _resolve_dataset_and_region, -) -from test_fixtures.fixtures_regions import ( - create_dataset, - create_region, - create_tax_benefit_model, -) - - -class TestInvalidRegionCode: - """Tests for invalid region code handling.""" - - def test_given_nonexistent_region_code_then_raises_404(self, session: Session): - """Given a region code that doesn't exist, then raises 404.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - # Note: No region is created for this code - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - region="nonexistent/region", - ) - - # When/Then - with pytest.raises(HTTPException) as exc_info: - _resolve_dataset_and_region(request, session) - - assert exc_info.value.status_code == 404 - assert "not found" in exc_info.value.detail.lower() - - def test_given_region_for_wrong_model_then_raises_404(self, session: Session): - """Given a region code for wrong model, then raises 404.""" - # Given - uk_model = create_tax_benefit_model(session, name="policyengine-uk") - uk_dataset = create_dataset(session, uk_model, name="uk_enhanced_frs") - create_region( - session, - model=uk_model, - dataset=uk_dataset, - code="uk", - label="United Kingdom", - region_type="national", - ) - # Request uses US model but UK region code - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_us", - region="uk", - ) - - # When/Then - with pytest.raises(HTTPException) as exc_info: - _resolve_dataset_and_region(request, session) - - assert exc_info.value.status_code == 404 - - -class TestMissingRequiredParams: - """Tests for missing required parameters.""" - - def test_given_neither_dataset_nor_region_then_raises_400(self, session: Session): - """Given neither dataset_id nor region, then raises 400.""" - # Given - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - # Neither dataset_id nor region provided - ) - - # When/Then - with pytest.raises(HTTPException) as exc_info: - _resolve_dataset_and_region(request, session) - - assert exc_info.value.status_code == 400 - assert "either dataset_id or region" in exc_info.value.detail.lower() - - -class TestNonexistentDataset: - """Tests for nonexistent dataset handling.""" - - def test_given_nonexistent_dataset_id_then_raises_404(self, session: Session): - """Given a dataset_id that doesn't exist, then raises 404.""" - # Given - from uuid import uuid4 - - nonexistent_id = uuid4() - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - dataset_id=nonexistent_id, - ) - - # When/Then - with pytest.raises(HTTPException) as exc_info: - _resolve_dataset_and_region(request, session) - - assert exc_info.value.status_code == 404 - assert "not found" in exc_info.value.detail.lower() diff --git a/tests/test__given_region_with_filter__then_filter_params_included.py b/tests/test__given_region_with_filter__then_filter_params_included.py deleted file mode 100644 index bc1d862..0000000 --- a/tests/test__given_region_with_filter__then_filter_params_included.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Tests for region resolution with filter parameters. - -When a region requires filtering (e.g., England from UK dataset, -California from US dataset), the filter_field and filter_value -should be extracted and passed through to simulations. -""" - -import pytest -from sqlmodel import Session - -from policyengine_api.api.analysis import ( - EconomicImpactRequest, - _get_or_create_simulation, - _resolve_dataset_and_region, -) -from policyengine_api.models import SimulationType -from test_fixtures.fixtures_regions import ( - create_dataset, - create_region, - create_tax_benefit_model, - create_tax_benefit_model_version, -) - - -class TestResolveDatasetAndRegionWithFilter: - """Tests for _resolve_dataset_and_region when region requires filtering.""" - - def test_given_region_requires_filter_then_returns_filter_field( - self, session: Session - ): - """Given a region that requires filtering, then filter_field is populated.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - region = create_region( - session, - model=model, - dataset=dataset, - code="country/england", - label="England", - region_type="country", - requires_filter=True, - filter_field="country", - filter_value="ENGLAND", - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - region="country/england", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_region is not None - assert resolved_region.filter_field == "country" - assert resolved_region.filter_value == "ENGLAND" - assert resolved_region.requires_filter is True - - def test_given_us_state_region_then_returns_state_filter(self, session: Session): - """Given a US state region, then returns state code filter.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-us") - dataset = create_dataset(session, model, name="us_cps") - region = create_region( - session, - model=model, - dataset=dataset, - code="state/ca", - label="California", - region_type="state", - requires_filter=True, - filter_field="state_code", - filter_value="CA", - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_us", - region="state/ca", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_region is not None - assert resolved_region.filter_field == "state_code" - assert resolved_region.filter_value == "CA" - - def test_given_region_with_filter_then_dataset_is_resolved(self, session: Session): - """Given a region code, then the associated dataset is returned.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - region = create_region( - session, - model=model, - dataset=dataset, - code="country/england", - label="England", - region_type="country", - requires_filter=True, - filter_field="country", - filter_value="ENGLAND", - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - region="country/england", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_dataset.id == dataset.id - assert resolved_dataset.name == "uk_enhanced_frs" - - -class TestSimulationCreationWithFilter: - """Tests for creating simulations with filter parameters.""" - - def test_given_filter_params_then_simulation_has_filter_fields( - self, session: Session - ): - """Given filter parameters, then created simulation has filter fields populated.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # When - simulation = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - filter_field="country", - filter_value="ENGLAND", - ) - - # Then - assert simulation.filter_field == "country" - assert simulation.filter_value == "ENGLAND" - - def test_given_no_filter_params_then_simulation_has_null_filter_fields( - self, session: Session - ): - """Given no filter parameters, then created simulation has null filter fields.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - model_version = create_tax_benefit_model_version(session, model) - dataset = create_dataset(session, model, name="uk_enhanced_frs") - - # When - simulation = _get_or_create_simulation( - simulation_type=SimulationType.ECONOMY, - dataset_id=dataset.id, - model_version_id=model_version.id, - policy_id=None, - dynamic_id=None, - session=session, - ) - - # Then - assert simulation.filter_field is None - assert simulation.filter_value is None diff --git a/tests/test__given_region_without_filter__then_filter_params_none.py b/tests/test__given_region_without_filter__then_filter_params_none.py deleted file mode 100644 index e81d7a8..0000000 --- a/tests/test__given_region_without_filter__then_filter_params_none.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Tests for region resolution without filter parameters. - -When a region does not require filtering (e.g., national UK or US), -the filter_field and filter_value should be None. -""" - -import pytest -from sqlmodel import Session - -from policyengine_api.api.analysis import ( - EconomicImpactRequest, - _resolve_dataset_and_region, -) -from test_fixtures.fixtures_regions import ( - create_dataset, - create_region, - create_tax_benefit_model, -) - - -class TestResolveDatasetAndRegionWithoutFilter: - """Tests for _resolve_dataset_and_region when region does not require filtering.""" - - def test_given_national_uk_region_then_filter_params_none(self, session: Session): - """Given UK national region, then filter_field and filter_value are None.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - region = create_region( - session, - model=model, - dataset=dataset, - code="uk", - label="United Kingdom", - region_type="national", - requires_filter=False, - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - region="uk", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_region is not None - assert resolved_region.requires_filter is False - assert resolved_region.filter_field is None - assert resolved_region.filter_value is None - - def test_given_national_us_region_then_filter_params_none(self, session: Session): - """Given US national region, then filter_field and filter_value are None.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-us") - dataset = create_dataset(session, model, name="us_cps") - region = create_region( - session, - model=model, - dataset=dataset, - code="us", - label="United States", - region_type="national", - requires_filter=False, - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_us", - region="us", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_region is not None - assert resolved_region.requires_filter is False - assert resolved_region.filter_field is None - assert resolved_region.filter_value is None - - def test_given_national_region_then_dataset_still_resolved(self, session: Session): - """Given national region without filter, then dataset is still correctly resolved.""" - # Given - model = create_tax_benefit_model(session, name="policyengine-uk") - dataset = create_dataset(session, model, name="uk_enhanced_frs") - region = create_region( - session, - model=model, - dataset=dataset, - code="uk", - label="United Kingdom", - region_type="national", - requires_filter=False, - ) - request = EconomicImpactRequest( - tax_benefit_model_name="policyengine_uk", - region="uk", - ) - - # When - resolved_dataset, resolved_region = _resolve_dataset_and_region( - request, session - ) - - # Then - assert resolved_dataset.id == dataset.id - assert resolved_dataset.name == "uk_enhanced_frs" diff --git a/tests/test__given_same_params__then_deterministic_id.py b/tests/test__given_same_params__then_deterministic_id.py deleted file mode 100644 index cbac11c..0000000 --- a/tests/test__given_same_params__then_deterministic_id.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Tests for deterministic simulation ID generation. - -The simulation ID is generated deterministically from the simulation -parameters (dataset, model version, policy, dynamic, filter params). -This ensures that re-running the same simulation reuses existing results. -""" - -from uuid import uuid4 - -import pytest - -from policyengine_api.api.analysis import _get_deterministic_simulation_id -from policyengine_api.models import SimulationType - - -class TestDeterministicSimulationId: - """Tests for _get_deterministic_simulation_id function.""" - - def test_given_same_params_then_same_id_returned(self): - """Given identical parameters, then the same ID is returned.""" - # Given - dataset_id = uuid4() - model_version_id = uuid4() - policy_id = uuid4() - dynamic_id = uuid4() - filter_field = "country" - filter_value = "ENGLAND" - - # When - id1 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field=filter_field, - filter_value=filter_value, - ) - id2 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field=filter_field, - filter_value=filter_value, - ) - - # Then - assert id1 == id2 - - def test_given_different_filter_field_then_different_id(self): - """Given different filter_field, then a different ID is returned.""" - # Given - dataset_id = uuid4() - model_version_id = uuid4() - policy_id = None - dynamic_id = None - - # When - id1 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field="country", - filter_value="ENGLAND", - ) - id2 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field="state_code", - filter_value="ENGLAND", - ) - - # Then - assert id1 != id2 - - def test_given_different_filter_value_then_different_id(self): - """Given different filter_value, then a different ID is returned.""" - # Given - dataset_id = uuid4() - model_version_id = uuid4() - policy_id = None - dynamic_id = None - - # When - id1 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field="country", - filter_value="ENGLAND", - ) - id2 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field="country", - filter_value="SCOTLAND", - ) - - # Then - assert id1 != id2 - - def test_given_filter_none_vs_filter_set_then_different_id(self): - """Given None filter vs set filter, then different IDs are returned.""" - # Given - dataset_id = uuid4() - model_version_id = uuid4() - policy_id = None - dynamic_id = None - - # When - id_no_filter = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field=None, - filter_value=None, - ) - id_with_filter = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=dataset_id, - filter_field="country", - filter_value="ENGLAND", - ) - - # Then - assert id_no_filter != id_with_filter - - def test_given_different_dataset_then_different_id(self): - """Given different dataset_id, then a different ID is returned.""" - # Given - model_version_id = uuid4() - policy_id = None - dynamic_id = None - filter_field = "country" - filter_value = "ENGLAND" - - # When - id1 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=uuid4(), - filter_field=filter_field, - filter_value=filter_value, - ) - id2 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - policy_id, - dynamic_id, - dataset_id=uuid4(), - filter_field=filter_field, - filter_value=filter_value, - ) - - # Then - assert id1 != id2 - - def test_given_null_optional_params_then_consistent_id(self): - """Given null optional parameters, then consistent ID is generated.""" - # Given - dataset_id = uuid4() - model_version_id = uuid4() - - # When - id1 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - None, - None, - dataset_id=dataset_id, - filter_field=None, - filter_value=None, - ) - id2 = _get_deterministic_simulation_id( - SimulationType.ECONOMY, - model_version_id, - None, - None, - dataset_id=dataset_id, - filter_field=None, - filter_value=None, - ) - - # Then - assert id1 == id2 diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 90dbe7c..ebcb7c2 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -1,26 +1,674 @@ -"""Tests for economic impact analysis endpoint. +"""Tests for economic impact analysis (analysis.py). -These tests require a running database with seeded data. -Run with: make integration-test +Unit tests for internal functions (_resolve_dataset_and_region, +_get_deterministic_simulation_id, _get_or_create_simulation) and +integration tests for the /analysis/economic-impact endpoint. """ -import pytest +from uuid import uuid4 -pytestmark = pytest.mark.integration +import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session, select +from policyengine_api.api.analysis import ( + EconomicImpactRequest, + _get_deterministic_simulation_id, + _get_or_create_simulation, + _resolve_dataset_and_region, +) from policyengine_api.main import app -from policyengine_api.models import Dataset, Simulation, TaxBenefitModel +from policyengine_api.models import ( + Dataset, + Simulation, + SimulationStatus, + SimulationType, + TaxBenefitModel, +) +from test_fixtures.fixtures_regions import ( + create_dataset, + create_region, + create_tax_benefit_model, + create_tax_benefit_model_version, +) client = TestClient(app) +# --------------------------------------------------------------------------- +# _resolve_dataset_and_region +# --------------------------------------------------------------------------- + + +class TestResolveDatasetAndRegion: + """Tests for _resolve_dataset_and_region.""" + + # -- dataset_id path -- + + def test__given_dataset_id__then_region_is_none(self, session: Session): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset.id, + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_region is None + + def test__given_dataset_id__then_dataset_is_returned(self, session: Session): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset.id, + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" + + def test__given_dataset_id_and_region__then_region_takes_precedence( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset1 = create_dataset(session, model, name="dataset_from_id") + dataset2 = create_dataset(session, model, name="dataset_from_region") + create_region( + session, + model=model, + dataset=dataset2, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=dataset1.id, + region="uk", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_dataset.id == dataset2.id + assert resolved_region is not None + assert resolved_region.code == "uk" + + # -- region with filter -- + + def test__given_region_requires_filter__then_returns_filter_fields( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + create_region( + session, + model=model, + dataset=dataset, + code="country/england", + label="England", + region_type="country", + requires_filter=True, + filter_field="country", + filter_value="ENGLAND", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="country/england", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_region is not None + assert resolved_region.filter_field == "country" + assert resolved_region.filter_value == "ENGLAND" + assert resolved_region.requires_filter is True + + def test__given_us_state_region__then_returns_state_filter( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-us") + dataset = create_dataset(session, model, name="us_cps") + create_region( + session, + model=model, + dataset=dataset, + code="state/ca", + label="California", + region_type="state", + requires_filter=True, + filter_field="state_code", + filter_value="CA", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="state/ca", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_region is not None + assert resolved_region.filter_field == "state_code" + assert resolved_region.filter_value == "CA" + + def test__given_region_with_filter__then_dataset_is_resolved( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + create_region( + session, + model=model, + dataset=dataset, + code="country/england", + label="England", + region_type="country", + requires_filter=True, + filter_field="country", + filter_value="ENGLAND", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="country/england", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" + + # -- region without filter -- + + def test__given_national_uk_region__then_filter_params_none( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + create_region( + session, + model=model, + dataset=dataset, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="uk", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_region is not None + assert resolved_region.requires_filter is False + assert resolved_region.filter_field is None + assert resolved_region.filter_value is None + + def test__given_national_us_region__then_filter_params_none( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-us") + dataset = create_dataset(session, model, name="us_cps") + create_region( + session, + model=model, + dataset=dataset, + code="us", + label="United States", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="us", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_region is not None + assert resolved_region.requires_filter is False + assert resolved_region.filter_field is None + assert resolved_region.filter_value is None + + def test__given_national_region__then_dataset_still_resolved( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + dataset = create_dataset(session, model, name="uk_enhanced_frs") + create_region( + session, + model=model, + dataset=dataset, + code="uk", + label="United Kingdom", + region_type="national", + requires_filter=False, + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="uk", + ) + + resolved_dataset, resolved_region = _resolve_dataset_and_region( + request, session + ) + + assert resolved_dataset.id == dataset.id + assert resolved_dataset.name == "uk_enhanced_frs" + + # -- error cases -- + + def test__given_nonexistent_region_code__then_raises_404(self, session: Session): + model = create_tax_benefit_model(session, name="policyengine-uk") + create_dataset(session, model, name="uk_enhanced_frs") + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + region="nonexistent/region", + ) + + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + assert "not found" in exc_info.value.detail.lower() + + def test__given_region_for_wrong_model__then_raises_404(self, session: Session): + uk_model = create_tax_benefit_model(session, name="policyengine-uk") + uk_dataset = create_dataset(session, uk_model, name="uk_enhanced_frs") + create_region( + session, + model=uk_model, + dataset=uk_dataset, + code="uk", + label="United Kingdom", + region_type="national", + ) + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_us", + region="uk", + ) + + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + + def test__given_neither_dataset_nor_region__then_raises_400(self, session: Session): + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + ) + + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 400 + assert "either dataset_id or region" in exc_info.value.detail.lower() + + def test__given_nonexistent_dataset_id__then_raises_404(self, session: Session): + nonexistent_id = uuid4() + request = EconomicImpactRequest( + tax_benefit_model_name="policyengine_uk", + dataset_id=nonexistent_id, + ) + + with pytest.raises(HTTPException) as exc_info: + _resolve_dataset_and_region(request, session) + + assert exc_info.value.status_code == 404 + assert "not found" in exc_info.value.detail.lower() + + +# --------------------------------------------------------------------------- +# _get_deterministic_simulation_id +# --------------------------------------------------------------------------- + + +class TestGetDeterministicSimulationId: + """Tests for _get_deterministic_simulation_id.""" + + def test__given_same_params__then_same_id_returned(self): + dataset_id = uuid4() + model_version_id = uuid4() + policy_id = uuid4() + dynamic_id = uuid4() + + id1 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + policy_id, + dynamic_id, + dataset_id=dataset_id, + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + policy_id, + dynamic_id, + dataset_id=dataset_id, + filter_field="country", + filter_value="ENGLAND", + ) + + assert id1 == id2 + + def test__given_different_filter_field__then_different_id(self): + dataset_id = uuid4() + model_version_id = uuid4() + + id1 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field="state_code", + filter_value="ENGLAND", + ) + + assert id1 != id2 + + def test__given_different_filter_value__then_different_id(self): + dataset_id = uuid4() + model_version_id = uuid4() + + id1 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field="country", + filter_value="SCOTLAND", + ) + + assert id1 != id2 + + def test__given_filter_none_vs_filter_set__then_different_id(self): + dataset_id = uuid4() + model_version_id = uuid4() + + id_no_filter = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field=None, + filter_value=None, + ) + id_with_filter = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field="country", + filter_value="ENGLAND", + ) + + assert id_no_filter != id_with_filter + + def test__given_different_dataset__then_different_id(self): + model_version_id = uuid4() + + id1 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=uuid4(), + filter_field="country", + filter_value="ENGLAND", + ) + id2 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=uuid4(), + filter_field="country", + filter_value="ENGLAND", + ) + + assert id1 != id2 + + def test__given_null_optional_params__then_consistent_id(self): + dataset_id = uuid4() + model_version_id = uuid4() + + id1 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field=None, + filter_value=None, + ) + id2 = _get_deterministic_simulation_id( + SimulationType.ECONOMY, + model_version_id, + None, + None, + dataset_id=dataset_id, + filter_field=None, + filter_value=None, + ) + + assert id1 == id2 + + +# --------------------------------------------------------------------------- +# _get_or_create_simulation +# --------------------------------------------------------------------------- + + +class TestGetOrCreateSimulation: + """Tests for _get_or_create_simulation.""" + + def test__given_existing_simulation_with_filter__then_reuses( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + first_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + second_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + assert first_sim.id == second_sim.id + + def test__given_different_filter__then_creates_new_simulation( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + england_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + scotland_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="SCOTLAND", + ) + + assert england_sim.id != scotland_sim.id + assert england_sim.filter_value == "ENGLAND" + assert scotland_sim.filter_value == "SCOTLAND" + + def test__given_no_filter_vs_filter__then_creates_separate_simulations( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + national_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field=None, + filter_value=None, + ) + filtered_sim = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + assert national_sim.id != filtered_sim.id + assert national_sim.filter_field is None + assert filtered_sim.filter_field == "country" + + def test__given_new_simulation__then_status_is_pending(self, session: Session): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + assert simulation.status == SimulationStatus.PENDING + + def test__given_filter_params__then_simulation_has_filter_fields( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + filter_field="country", + filter_value="ENGLAND", + ) + + assert simulation.filter_field == "country" + assert simulation.filter_value == "ENGLAND" + + def test__given_no_filter_params__then_simulation_has_null_filter_fields( + self, session: Session + ): + model = create_tax_benefit_model(session, name="policyengine-uk") + model_version = create_tax_benefit_model_version(session, model) + dataset = create_dataset(session, model, name="uk_enhanced_frs") + + simulation = _get_or_create_simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=dataset.id, + model_version_id=model_version.id, + policy_id=None, + dynamic_id=None, + session=session, + ) + + assert simulation.filter_field is None + assert simulation.filter_value is None + + +# --------------------------------------------------------------------------- +# HTTP endpoint validation (no database required) +# --------------------------------------------------------------------------- + + class TestEconomicImpactValidation: """Tests for request validation (no database required).""" def test_invalid_model_name(self): - """Test that invalid model name returns 422.""" response = client.post( "/analysis/economic-impact", json={ @@ -31,17 +679,15 @@ def test_invalid_model_name(self): assert response.status_code == 422 def test_missing_dataset_id(self): - """Test that missing dataset_id returns 422.""" response = client.post( "/analysis/economic-impact", json={ "tax_benefit_model_name": "policyengine_uk", }, ) - assert response.status_code == 422 + assert response.status_code == 400 def test_invalid_uuid(self): - """Test that invalid UUID returns 422.""" response = client.post( "/analysis/economic-impact", json={ @@ -56,7 +702,6 @@ class TestEconomicImpactNotFound: """Tests for 404 responses.""" def test_dataset_not_found(self): - """Test that non-existent dataset returns 404.""" response = client.post( "/analysis/economic-impact", json={ @@ -68,8 +713,11 @@ def test_dataset_not_found(self): assert "not found" in response.json()["detail"].lower() -# Integration tests that require a running database with seeded data -# These are marked with pytest.mark.integration and skipped by default +# --------------------------------------------------------------------------- +# Integration tests (require running database with seeded data) +# --------------------------------------------------------------------------- + + @pytest.mark.integration class TestEconomicImpactIntegration: """Integration tests for economic impact analysis. @@ -97,7 +745,6 @@ def uk_dataset_id(self, session: Session): return dataset.id def test_uk_economic_impact_baseline_only(self, uk_dataset_id): - """Test UK economic impact with no reform policy.""" response = client.post( "/analysis/economic-impact", json={ @@ -113,10 +760,8 @@ def test_uk_economic_impact_baseline_only(self, uk_dataset_id): assert "decile_impacts" in data assert "programme_statistics" in data - # Should have 10 deciles assert len(data["decile_impacts"]) == 10 - # Check decile structure for di in data["decile_impacts"]: assert "decile" in di assert "baseline_mean" in di @@ -124,7 +769,6 @@ def test_uk_economic_impact_baseline_only(self, uk_dataset_id): assert "absolute_change" in di def test_simulations_created(self, uk_dataset_id, session: Session): - """Test that simulations are created in the database.""" response = client.post( "/analysis/economic-impact", json={ @@ -135,7 +779,6 @@ def test_simulations_created(self, uk_dataset_id, session: Session): assert response.status_code == 200 data = response.json() - # Check simulations exist in database baseline_sim = session.get(Simulation, data["baseline_simulation_id"]) assert baseline_sim is not None assert baseline_sim.status == "completed" From 73b0db2f97e60fb77d6279b2e24ea1fa15c415ac Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 17 Feb 2026 01:06:40 +0100 Subject: [PATCH 29/87] fix: Mark TestEconomicImpactNotFound as integration test This test hits the real database (valid request passes validation), so it needs a running Supabase instance like the other integration tests. Co-Authored-By: Claude Opus 4.6 --- tests/test_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index ebcb7c2..bfdd4a2 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -698,6 +698,7 @@ def test_invalid_uuid(self): assert response.status_code == 422 +@pytest.mark.integration class TestEconomicImpactNotFound: """Tests for 404 responses.""" From 5cdd5c2fa79dbb701f3e00dfac4b79a1c67f016a Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Mon, 12 Jan 2026 10:28:41 -0800 Subject: [PATCH 30/87] feat: add user-policy associations --- src/policyengine_api/api/__init__.py | 2 + src/policyengine_api/api/user_policies.py | 138 +++++++++ src/policyengine_api/models/__init__.py | 10 + src/policyengine_api/models/user_policy.py | 52 ++++ tests/test_user_policies.py | 317 +++++++++++++++++++++ 5 files changed, 519 insertions(+) create mode 100644 src/policyengine_api/api/user_policies.py create mode 100644 src/policyengine_api/models/user_policy.py create mode 100644 tests/test_user_policies.py diff --git a/src/policyengine_api/api/__init__.py b/src/policyengine_api/api/__init__.py index f135b14..7e94d29 100644 --- a/src/policyengine_api/api/__init__.py +++ b/src/policyengine_api/api/__init__.py @@ -20,6 +20,7 @@ tax_benefit_model_versions, tax_benefit_models, user_household_associations, + user_policies, variables, ) @@ -43,5 +44,6 @@ api_router.include_router(analysis.router) api_router.include_router(agent.router) api_router.include_router(user_household_associations.router) +api_router.include_router(user_policies.router) __all__ = ["api_router"] diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py new file mode 100644 index 0000000..d2326c2 --- /dev/null +++ b/src/policyengine_api/api/user_policies.py @@ -0,0 +1,138 @@ +"""User-policy association endpoints. + +Associates users with policies they've saved/created. This enables users to +maintain a list of their policies across sessions without duplicating the +underlying policy data. +""" + +from datetime import datetime, timezone +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlmodel import Session, select + +from policyengine_api.models import ( + Policy, + User, + UserPolicy, + UserPolicyCreate, + UserPolicyRead, + UserPolicyUpdate, +) +from policyengine_api.services.database import get_session + +router = APIRouter(prefix="/user-policies", tags=["user-policies"]) + + +@router.post("/", response_model=UserPolicyRead) +def create_user_policy( + user_policy: UserPolicyCreate, + session: Session = Depends(get_session), +): + """Create a new user-policy association. + + Associates a user with a policy, allowing them to save it to their list. + A user can only have one association per policy (duplicates are rejected). + """ + # Validate user exists + user = session.get(User, user_policy.user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Validate policy exists + policy = session.get(Policy, user_policy.policy_id) + if not policy: + raise HTTPException(status_code=404, detail="Policy not found") + + # Check for duplicate (same user_id + policy_id) + existing = session.exec( + select(UserPolicy).where( + UserPolicy.user_id == user_policy.user_id, + UserPolicy.policy_id == user_policy.policy_id, + ) + ).first() + if existing: + raise HTTPException( + status_code=409, + detail="User already has an association with this policy", + ) + + # Create the association + db_user_policy = UserPolicy.model_validate(user_policy) + session.add(db_user_policy) + session.commit() + session.refresh(db_user_policy) + return db_user_policy + + +@router.get("/", response_model=list[UserPolicyRead]) +def list_user_policies( + user_id: UUID = Query(..., description="User ID to filter by"), + country_id: str | None = Query(None, description="Country filter (us/uk)"), + session: Session = Depends(get_session), +): + """List all policy associations for a user. + + Returns all policies saved by the specified user. Optionally filter by country. + """ + query = select(UserPolicy).where(UserPolicy.user_id == user_id) + + if country_id: + query = query.where(UserPolicy.country_id == country_id) + + user_policies = session.exec(query).all() + return user_policies + + +@router.get("/{user_policy_id}", response_model=UserPolicyRead) +def get_user_policy( + user_policy_id: UUID, + session: Session = Depends(get_session), +): + """Get a specific user-policy association by ID.""" + user_policy = session.get(UserPolicy, user_policy_id) + if not user_policy: + raise HTTPException(status_code=404, detail="User-policy association not found") + return user_policy + + +@router.patch("/{user_policy_id}", response_model=UserPolicyRead) +def update_user_policy( + user_policy_id: UUID, + updates: UserPolicyUpdate, + session: Session = Depends(get_session), +): + """Update a user-policy association (e.g., rename label).""" + user_policy = session.get(UserPolicy, user_policy_id) + if not user_policy: + raise HTTPException(status_code=404, detail="User-policy association not found") + + # Apply updates + update_data = updates.model_dump(exclude_unset=True) + for key, value in update_data.items(): + setattr(user_policy, key, value) + + # Update timestamp + user_policy.updated_at = datetime.now(timezone.utc) + + session.add(user_policy) + session.commit() + session.refresh(user_policy) + return user_policy + + +@router.delete("/{user_policy_id}", status_code=204) +def delete_user_policy( + user_policy_id: UUID, + session: Session = Depends(get_session), +): + """Delete a user-policy association. + + This only removes the association, not the underlying policy. + """ + user_policy = session.get(UserPolicy, user_policy_id) + if not user_policy: + raise HTTPException(status_code=404, detail="User-policy association not found") + + session.delete(user_policy) + session.commit() diff --git a/src/policyengine_api/models/__init__.py b/src/policyengine_api/models/__init__.py index 7361979..e7b386a 100644 --- a/src/policyengine_api/models/__init__.py +++ b/src/policyengine_api/models/__init__.py @@ -61,6 +61,12 @@ UserHouseholdAssociationRead, UserHouseholdAssociationUpdate, ) +from .user_policy import ( + UserPolicy, + UserPolicyCreate, + UserPolicyRead, + UserPolicyUpdate, +) from .variable import Variable, VariableCreate, VariableRead __all__ = [ @@ -136,6 +142,10 @@ "UserHouseholdAssociationRead", "UserHouseholdAssociationUpdate", "UserRead", + "UserPolicy", + "UserPolicyCreate", + "UserPolicyRead", + "UserPolicyUpdate", "Variable", "VariableCreate", "VariableRead", diff --git a/src/policyengine_api/models/user_policy.py b/src/policyengine_api/models/user_policy.py new file mode 100644 index 0000000..7eb7353 --- /dev/null +++ b/src/policyengine_api/models/user_policy.py @@ -0,0 +1,52 @@ +from datetime import datetime, timezone +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlmodel import Field, Relationship, SQLModel + +if TYPE_CHECKING: + from .policy import Policy + from .user import User + + +class UserPolicyBase(SQLModel): + """Base user-policy association fields.""" + + user_id: UUID = Field(foreign_key="users.id", index=True) + policy_id: UUID = Field(foreign_key="policies.id", index=True) + country_id: str # "us" or "uk" + label: str | None = None + + +class UserPolicy(UserPolicyBase, table=True): + """User-policy association database model.""" + + __tablename__ = "user_policies" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + # Relationships + user: "User" = Relationship() + policy: "Policy" = Relationship() + + +class UserPolicyCreate(UserPolicyBase): + """Schema for creating user-policy associations.""" + + pass + + +class UserPolicyRead(UserPolicyBase): + """Schema for reading user-policy associations.""" + + id: UUID + created_at: datetime + updated_at: datetime + + +class UserPolicyUpdate(SQLModel): + """Schema for updating user-policy associations.""" + + label: str | None = None diff --git a/tests/test_user_policies.py b/tests/test_user_policies.py new file mode 100644 index 0000000..5c4e327 --- /dev/null +++ b/tests/test_user_policies.py @@ -0,0 +1,317 @@ +"""Tests for user-policy association endpoints.""" + +from uuid import uuid4 + +from policyengine_api.models import Policy, User, UserPolicy + + +def test_list_user_policies_empty(client, session): + """List user policies returns empty list when user has no associations.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + session.add(user) + session.commit() + + response = client.get(f"/user-policies?user_id={user.id}") + assert response.status_code == 200 + assert response.json() == [] + + +def test_create_user_policy(client, session): + """Create a new user-policy association.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + response = client.post( + "/user-policies", + json={ + "user_id": str(user.id), + "policy_id": str(policy.id), + "country_id": "us", + "label": "My test policy", + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["user_id"] == str(user.id) + assert data["policy_id"] == str(policy.id) + assert data["country_id"] == "us" + assert data["label"] == "My test policy" + assert "id" in data + assert "created_at" in data + assert "updated_at" in data + + +def test_create_user_policy_without_label(client, session): + """Create a user-policy association without a label.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + response = client.post( + "/user-policies", + json={ + "user_id": str(user.id), + "policy_id": str(policy.id), + "country_id": "uk", + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["label"] is None + + +def test_create_user_policy_user_not_found(client): + """Create user-policy association with non-existent user returns 404.""" + fake_user_id = uuid4() + fake_policy_id = uuid4() + + response = client.post( + "/user-policies", + json={ + "user_id": str(fake_user_id), + "policy_id": str(fake_policy_id), + "country_id": "us", + }, + ) + assert response.status_code == 404 + assert response.json()["detail"] == "User not found" + + +def test_create_user_policy_policy_not_found(client, session): + """Create user-policy association with non-existent policy returns 404.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + session.add(user) + session.commit() + session.refresh(user) + + fake_policy_id = uuid4() + response = client.post( + "/user-policies", + json={ + "user_id": str(user.id), + "policy_id": str(fake_policy_id), + "country_id": "us", + }, + ) + assert response.status_code == 404 + assert response.json()["detail"] == "Policy not found" + + +def test_create_user_policy_duplicate(client, session): + """Creating duplicate user-policy association returns 409.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + # Create first association + user_policy = UserPolicy( + user_id=user.id, + policy_id=policy.id, + country_id="us", + ) + session.add(user_policy) + session.commit() + + # Try to create duplicate + response = client.post( + "/user-policies", + json={ + "user_id": str(user.id), + "policy_id": str(policy.id), + "country_id": "us", + }, + ) + assert response.status_code == 409 + assert "already has an association" in response.json()["detail"] + + +def test_list_user_policies_with_data(client, session): + """List user policies returns all associations for a user.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy1 = Policy(name="Policy 1", description="First policy") + policy2 = Policy(name="Policy 2", description="Second policy") + session.add(user) + session.add(policy1) + session.add(policy2) + session.commit() + session.refresh(user) + session.refresh(policy1) + session.refresh(policy2) + + user_policy1 = UserPolicy( + user_id=user.id, + policy_id=policy1.id, + country_id="us", + label="US policy", + ) + user_policy2 = UserPolicy( + user_id=user.id, + policy_id=policy2.id, + country_id="uk", + label="UK policy", + ) + session.add(user_policy1) + session.add(user_policy2) + session.commit() + + response = client.get(f"/user-policies?user_id={user.id}") + assert response.status_code == 200 + data = response.json() + assert len(data) == 2 + + +def test_list_user_policies_filter_by_country(client, session): + """List user policies with country filter.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy1 = Policy(name="Policy 1", description="First policy") + policy2 = Policy(name="Policy 2", description="Second policy") + session.add(user) + session.add(policy1) + session.add(policy2) + session.commit() + session.refresh(user) + session.refresh(policy1) + session.refresh(policy2) + + user_policy1 = UserPolicy( + user_id=user.id, + policy_id=policy1.id, + country_id="us", + ) + user_policy2 = UserPolicy( + user_id=user.id, + policy_id=policy2.id, + country_id="uk", + ) + session.add(user_policy1) + session.add(user_policy2) + session.commit() + + response = client.get(f"/user-policies?user_id={user.id}&country_id=us") + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["country_id"] == "us" + + +def test_get_user_policy(client, session): + """Get a specific user-policy association by ID.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + user_policy = UserPolicy( + user_id=user.id, + policy_id=policy.id, + country_id="us", + label="My policy", + ) + session.add(user_policy) + session.commit() + session.refresh(user_policy) + + response = client.get(f"/user-policies/{user_policy.id}") + assert response.status_code == 200 + data = response.json() + assert data["id"] == str(user_policy.id) + assert data["label"] == "My policy" + + +def test_get_user_policy_not_found(client): + """Get a non-existent user-policy association returns 404.""" + fake_id = uuid4() + response = client.get(f"/user-policies/{fake_id}") + assert response.status_code == 404 + assert response.json()["detail"] == "User-policy association not found" + + +def test_update_user_policy(client, session): + """Update a user-policy association label.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + user_policy = UserPolicy( + user_id=user.id, + policy_id=policy.id, + country_id="us", + label="Old label", + ) + session.add(user_policy) + session.commit() + session.refresh(user_policy) + + response = client.patch( + f"/user-policies/{user_policy.id}", + json={"label": "New label"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["label"] == "New label" + + +def test_update_user_policy_not_found(client): + """Update a non-existent user-policy association returns 404.""" + fake_id = uuid4() + response = client.patch( + f"/user-policies/{fake_id}", + json={"label": "New label"}, + ) + assert response.status_code == 404 + assert response.json()["detail"] == "User-policy association not found" + + +def test_delete_user_policy(client, session): + """Delete a user-policy association.""" + user = User(first_name="Test", last_name="User", email="test@example.com") + policy = Policy(name="Test policy", description="A test policy") + session.add(user) + session.add(policy) + session.commit() + session.refresh(user) + session.refresh(policy) + + user_policy = UserPolicy( + user_id=user.id, + policy_id=policy.id, + country_id="us", + ) + session.add(user_policy) + session.commit() + session.refresh(user_policy) + + response = client.delete(f"/user-policies/{user_policy.id}") + assert response.status_code == 204 + + # Verify it's deleted + response = client.get(f"/user-policies/{user_policy.id}") + assert response.status_code == 404 + + +def test_delete_user_policy_not_found(client): + """Delete a non-existent user-policy association returns 404.""" + fake_id = uuid4() + response = client.delete(f"/user-policies/{fake_id}") + assert response.status_code == 404 + assert response.json()["detail"] == "User-policy association not found" From ef0683ddd7deeb5855242996f33041ddabbf252a Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Fri, 23 Jan 2026 00:51:19 -0800 Subject: [PATCH 31/87] feat: Add tax_benefit_model_id to Policy, remove country_id from UserPolicy --- src/policyengine_api/api/policies.py | 27 ++++-- src/policyengine_api/api/user_policies.py | 12 ++- src/policyengine_api/models/policy.py | 3 + src/policyengine_api/models/user_policy.py | 1 - tests/conftest.py | 20 +++++ tests/test_policies.py | 63 +++++++++++-- tests/test_user_policies.py | 100 +++++++++++++-------- 7 files changed, 174 insertions(+), 52 deletions(-) diff --git a/src/policyengine_api/api/policies.py b/src/policyengine_api/api/policies.py index d0e2ca5..1f4c744 100644 --- a/src/policyengine_api/api/policies.py +++ b/src/policyengine_api/api/policies.py @@ -31,7 +31,7 @@ from typing import List from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from sqlmodel import Session, select from policyengine_api.models import ( @@ -40,6 +40,7 @@ Policy, PolicyCreate, PolicyRead, + TaxBenefitModel, ) from policyengine_api.services.database import get_session @@ -67,8 +68,17 @@ def create_policy(policy: PolicyCreate, session: Session = Depends(get_session)) ] } """ + # Validate tax_benefit_model exists + tax_model = session.get(TaxBenefitModel, policy.tax_benefit_model_id) + if not tax_model: + raise HTTPException(status_code=404, detail="Tax benefit model not found") + # Create the policy - db_policy = Policy(name=policy.name, description=policy.description) + db_policy = Policy( + name=policy.name, + description=policy.description, + tax_benefit_model_id=policy.tax_benefit_model_id, + ) session.add(db_policy) session.flush() # Get the policy ID before adding parameter values @@ -112,10 +122,15 @@ def create_policy(policy: PolicyCreate, session: Session = Depends(get_session)) @router.get("/", response_model=List[PolicyRead]) -def list_policies(session: Session = Depends(get_session)): - """List all policies.""" - policies = session.exec(select(Policy)).all() - return policies +def list_policies( + tax_benefit_model_id: UUID | None = Query(None, description="Filter by tax benefit model"), + session: Session = Depends(get_session), +): + """List all policies, optionally filtered by tax benefit model.""" + query = select(Policy) + if tax_benefit_model_id: + query = query.where(Policy.tax_benefit_model_id == tax_benefit_model_id) + return session.exec(query).all() @router.get("/{policy_id}", response_model=PolicyRead) diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py index d2326c2..7898f15 100644 --- a/src/policyengine_api/api/user_policies.py +++ b/src/policyengine_api/api/user_policies.py @@ -68,17 +68,21 @@ def create_user_policy( @router.get("/", response_model=list[UserPolicyRead]) def list_user_policies( user_id: UUID = Query(..., description="User ID to filter by"), - country_id: str | None = Query(None, description="Country filter (us/uk)"), + tax_benefit_model_id: UUID | None = Query(None, description="Filter by tax benefit model"), session: Session = Depends(get_session), ): """List all policy associations for a user. - Returns all policies saved by the specified user. Optionally filter by country. + Returns all policies saved by the specified user. Optionally filter by tax benefit model. """ query = select(UserPolicy).where(UserPolicy.user_id == user_id) - if country_id: - query = query.where(UserPolicy.country_id == country_id) + if tax_benefit_model_id: + query = ( + query + .join(Policy, UserPolicy.policy_id == Policy.id) + .where(Policy.tax_benefit_model_id == tax_benefit_model_id) + ) user_policies = session.exec(query).all() return user_policies diff --git a/src/policyengine_api/models/policy.py b/src/policyengine_api/models/policy.py index 570320b..69eeecf 100644 --- a/src/policyengine_api/models/policy.py +++ b/src/policyengine_api/models/policy.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from .parameter_value import ParameterValue + from .tax_benefit_model import TaxBenefitModel class PolicyBase(SQLModel): @@ -13,6 +14,7 @@ class PolicyBase(SQLModel): name: str description: str | None = None + tax_benefit_model_id: UUID = Field(foreign_key="tax_benefit_models.id", index=True) class Policy(PolicyBase, table=True): @@ -26,6 +28,7 @@ class Policy(PolicyBase, table=True): # Relationships parameter_values: list["ParameterValue"] = Relationship(back_populates="policy") + tax_benefit_model: "TaxBenefitModel" = Relationship() class PolicyCreate(PolicyBase): diff --git a/src/policyengine_api/models/user_policy.py b/src/policyengine_api/models/user_policy.py index 7eb7353..6bdf9ff 100644 --- a/src/policyengine_api/models/user_policy.py +++ b/src/policyengine_api/models/user_policy.py @@ -14,7 +14,6 @@ class UserPolicyBase(SQLModel): user_id: UUID = Field(foreign_key="users.id", index=True) policy_id: UUID = Field(foreign_key="policies.id", index=True) - country_id: str # "us" or "uk" label: str | None = None diff --git a/tests/conftest.py b/tests/conftest.py index 8be9b3f..fe5c04a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,6 +48,26 @@ def get_session_override(): app.dependency_overrides.clear() +@pytest.fixture(name="tax_benefit_model") +def tax_benefit_model_fixture(session: Session): + """Create a TaxBenefitModel for tests.""" + model = TaxBenefitModel(name="policyengine-us", description="US model") + session.add(model) + session.commit() + session.refresh(model) + return model + + +@pytest.fixture(name="uk_tax_benefit_model") +def uk_tax_benefit_model_fixture(session: Session): + """Create a UK TaxBenefitModel for tests.""" + model = TaxBenefitModel(name="policyengine-uk", description="UK model") + session.add(model) + session.commit() + session.refresh(model) + return model + + @pytest.fixture(name="simulation_id") def simulation_fixture(session: Session): """Create a test simulation with required dependencies.""" diff --git a/tests/test_policies.py b/tests/test_policies.py index f48730b..9c85f96 100644 --- a/tests/test_policies.py +++ b/tests/test_policies.py @@ -2,7 +2,7 @@ from uuid import uuid4 -from policyengine_api.models import Policy +from policyengine_api.models import Policy, TaxBenefitModel def test_list_policies_empty(client): @@ -12,25 +12,46 @@ def test_list_policies_empty(client): assert response.json() == [] -def test_create_policy(client): +def test_create_policy(client, tax_benefit_model): """Create a new policy.""" response = client.post( "/policies", json={ "name": "Test policy", "description": "A test policy", + "tax_benefit_model_id": str(tax_benefit_model.id), }, ) assert response.status_code == 200 data = response.json() assert data["name"] == "Test policy" assert data["description"] == "A test policy" + assert data["tax_benefit_model_id"] == str(tax_benefit_model.id) assert "id" in data -def test_list_policies_with_data(client, session): +def test_create_policy_invalid_tax_benefit_model(client): + """Create policy with non-existent tax_benefit_model returns 404.""" + fake_id = uuid4() + response = client.post( + "/policies", + json={ + "name": "Test policy", + "description": "A test policy", + "tax_benefit_model_id": str(fake_id), + }, + ) + assert response.status_code == 404 + assert response.json()["detail"] == "Tax benefit model not found" + + +def test_list_policies_with_data(client, session, tax_benefit_model): """List policies returns all policies.""" - policy = Policy(name="test-policy", description="Test") + policy = Policy( + name="test-policy", + description="Test", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(policy) session.commit() @@ -41,9 +62,39 @@ def test_list_policies_with_data(client, session): assert data[0]["name"] == "test-policy" -def test_get_policy(client, session): +def test_list_policies_filter_by_tax_benefit_model( + client, session, tax_benefit_model, uk_tax_benefit_model +): + """List policies with tax_benefit_model_id filter.""" + policy1 = Policy( + name="US policy", + description="US", + tax_benefit_model_id=tax_benefit_model.id, + ) + policy2 = Policy( + name="UK policy", + description="UK", + tax_benefit_model_id=uk_tax_benefit_model.id, + ) + session.add(policy1) + session.add(policy2) + session.commit() + + # Filter by US model + response = client.get(f"/policies?tax_benefit_model_id={tax_benefit_model.id}") + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["name"] == "US policy" + + +def test_get_policy(client, session, tax_benefit_model): """Get a specific policy by ID.""" - policy = Policy(name="test-policy", description="Test") + policy = Policy( + name="test-policy", + description="Test", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(policy) session.commit() session.refresh(policy) diff --git a/tests/test_user_policies.py b/tests/test_user_policies.py index 5c4e327..0db9367 100644 --- a/tests/test_user_policies.py +++ b/tests/test_user_policies.py @@ -16,10 +16,14 @@ def test_list_user_policies_empty(client, session): assert response.json() == [] -def test_create_user_policy(client, session): +def test_create_user_policy(client, session, tax_benefit_model): """Create a new user-policy association.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -31,7 +35,6 @@ def test_create_user_policy(client, session): json={ "user_id": str(user.id), "policy_id": str(policy.id), - "country_id": "us", "label": "My test policy", }, ) @@ -39,17 +42,20 @@ def test_create_user_policy(client, session): data = response.json() assert data["user_id"] == str(user.id) assert data["policy_id"] == str(policy.id) - assert data["country_id"] == "us" assert data["label"] == "My test policy" assert "id" in data assert "created_at" in data assert "updated_at" in data -def test_create_user_policy_without_label(client, session): +def test_create_user_policy_without_label(client, session, tax_benefit_model): """Create a user-policy association without a label.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -61,7 +67,6 @@ def test_create_user_policy_without_label(client, session): json={ "user_id": str(user.id), "policy_id": str(policy.id), - "country_id": "uk", }, ) assert response.status_code == 200 @@ -79,7 +84,6 @@ def test_create_user_policy_user_not_found(client): json={ "user_id": str(fake_user_id), "policy_id": str(fake_policy_id), - "country_id": "us", }, ) assert response.status_code == 404 @@ -99,17 +103,20 @@ def test_create_user_policy_policy_not_found(client, session): json={ "user_id": str(user.id), "policy_id": str(fake_policy_id), - "country_id": "us", }, ) assert response.status_code == 404 assert response.json()["detail"] == "Policy not found" -def test_create_user_policy_duplicate(client, session): +def test_create_user_policy_duplicate(client, session, tax_benefit_model): """Creating duplicate user-policy association returns 409.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -120,7 +127,6 @@ def test_create_user_policy_duplicate(client, session): user_policy = UserPolicy( user_id=user.id, policy_id=policy.id, - country_id="us", ) session.add(user_policy) session.commit() @@ -131,18 +137,25 @@ def test_create_user_policy_duplicate(client, session): json={ "user_id": str(user.id), "policy_id": str(policy.id), - "country_id": "us", }, ) assert response.status_code == 409 assert "already has an association" in response.json()["detail"] -def test_list_user_policies_with_data(client, session): +def test_list_user_policies_with_data(client, session, tax_benefit_model, uk_tax_benefit_model): """List user policies returns all associations for a user.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy1 = Policy(name="Policy 1", description="First policy") - policy2 = Policy(name="Policy 2", description="Second policy") + policy1 = Policy( + name="Policy 1", + description="First policy", + tax_benefit_model_id=tax_benefit_model.id, + ) + policy2 = Policy( + name="Policy 2", + description="Second policy", + tax_benefit_model_id=uk_tax_benefit_model.id, + ) session.add(user) session.add(policy1) session.add(policy2) @@ -154,13 +167,11 @@ def test_list_user_policies_with_data(client, session): user_policy1 = UserPolicy( user_id=user.id, policy_id=policy1.id, - country_id="us", label="US policy", ) user_policy2 = UserPolicy( user_id=user.id, policy_id=policy2.id, - country_id="uk", label="UK policy", ) session.add(user_policy1) @@ -173,11 +184,21 @@ def test_list_user_policies_with_data(client, session): assert len(data) == 2 -def test_list_user_policies_filter_by_country(client, session): - """List user policies with country filter.""" +def test_list_user_policies_filter_by_tax_benefit_model( + client, session, tax_benefit_model, uk_tax_benefit_model +): + """List user policies filtered by tax_benefit_model_id via Policy join.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy1 = Policy(name="Policy 1", description="First policy") - policy2 = Policy(name="Policy 2", description="Second policy") + policy1 = Policy( + name="Policy 1", + description="First policy", + tax_benefit_model_id=tax_benefit_model.id, + ) + policy2 = Policy( + name="Policy 2", + description="Second policy", + tax_benefit_model_id=uk_tax_benefit_model.id, + ) session.add(user) session.add(policy1) session.add(policy2) @@ -189,28 +210,32 @@ def test_list_user_policies_filter_by_country(client, session): user_policy1 = UserPolicy( user_id=user.id, policy_id=policy1.id, - country_id="us", ) user_policy2 = UserPolicy( user_id=user.id, policy_id=policy2.id, - country_id="uk", ) session.add(user_policy1) session.add(user_policy2) session.commit() - response = client.get(f"/user-policies?user_id={user.id}&country_id=us") + response = client.get( + f"/user-policies?user_id={user.id}&tax_benefit_model_id={tax_benefit_model.id}" + ) assert response.status_code == 200 data = response.json() assert len(data) == 1 - assert data[0]["country_id"] == "us" + assert data[0]["policy_id"] == str(policy1.id) -def test_get_user_policy(client, session): +def test_get_user_policy(client, session, tax_benefit_model): """Get a specific user-policy association by ID.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -220,7 +245,6 @@ def test_get_user_policy(client, session): user_policy = UserPolicy( user_id=user.id, policy_id=policy.id, - country_id="us", label="My policy", ) session.add(user_policy) @@ -242,10 +266,14 @@ def test_get_user_policy_not_found(client): assert response.json()["detail"] == "User-policy association not found" -def test_update_user_policy(client, session): +def test_update_user_policy(client, session, tax_benefit_model): """Update a user-policy association label.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -255,7 +283,6 @@ def test_update_user_policy(client, session): user_policy = UserPolicy( user_id=user.id, policy_id=policy.id, - country_id="us", label="Old label", ) session.add(user_policy) @@ -282,10 +309,14 @@ def test_update_user_policy_not_found(client): assert response.json()["detail"] == "User-policy association not found" -def test_delete_user_policy(client, session): +def test_delete_user_policy(client, session, tax_benefit_model): """Delete a user-policy association.""" user = User(first_name="Test", last_name="User", email="test@example.com") - policy = Policy(name="Test policy", description="A test policy") + policy = Policy( + name="Test policy", + description="A test policy", + tax_benefit_model_id=tax_benefit_model.id, + ) session.add(user) session.add(policy) session.commit() @@ -295,7 +326,6 @@ def test_delete_user_policy(client, session): user_policy = UserPolicy( user_id=user.id, policy_id=policy.id, - country_id="us", ) session.add(user_policy) session.commit() From 3aae41e8bccdf527ad413b7a70ef854be0eed3fe Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Tue, 3 Feb 2026 00:37:15 -0800 Subject: [PATCH 32/87] feat: Add base schema migration and allow duplicate user-policy saves --- src/policyengine_api/api/user_policies.py | 18 +- .../migrations/20241115000000_base_schema.sql | 273 ++++++++++++++++++ 2 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 supabase/migrations/20241115000000_base_schema.sql diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py index 7898f15..3ac2786 100644 --- a/src/policyengine_api/api/user_policies.py +++ b/src/policyengine_api/api/user_policies.py @@ -32,7 +32,8 @@ def create_user_policy( """Create a new user-policy association. Associates a user with a policy, allowing them to save it to their list. - A user can only have one association per policy (duplicates are rejected). + Duplicates are allowed - users can save the same policy multiple times + with different labels (matching FE localStorage behavior). """ # Validate user exists user = session.get(User, user_policy.user_id) @@ -44,20 +45,7 @@ def create_user_policy( if not policy: raise HTTPException(status_code=404, detail="Policy not found") - # Check for duplicate (same user_id + policy_id) - existing = session.exec( - select(UserPolicy).where( - UserPolicy.user_id == user_policy.user_id, - UserPolicy.policy_id == user_policy.policy_id, - ) - ).first() - if existing: - raise HTTPException( - status_code=409, - detail="User already has an association with this policy", - ) - - # Create the association + # Create the association (duplicates allowed) db_user_policy = UserPolicy.model_validate(user_policy) session.add(db_user_policy) session.commit() diff --git a/supabase/migrations/20241115000000_base_schema.sql b/supabase/migrations/20241115000000_base_schema.sql new file mode 100644 index 0000000..1b1a0ce --- /dev/null +++ b/supabase/migrations/20241115000000_base_schema.sql @@ -0,0 +1,273 @@ +-- Base schema for PolicyEngine API v2 +-- This migration creates all tables required by SQLModel definitions + +-- ============================================================================ +-- ENUM TYPES +-- ============================================================================ + +CREATE TYPE householdjobstatus AS ENUM ('pending', 'running', 'completed', 'failed'); +CREATE TYPE simulationstatus AS ENUM ('pending', 'running', 'completed', 'failed'); +CREATE TYPE reportstatus AS ENUM ('pending', 'running', 'completed', 'failed'); +CREATE TYPE aggregatetype AS ENUM ('sum', 'mean', 'count'); +CREATE TYPE aggregatestatus AS ENUM ('pending', 'running', 'completed', 'failed'); +CREATE TYPE changeaggregatetype AS ENUM ('sum', 'mean', 'count'); +CREATE TYPE changeaggregatestatus AS ENUM ('pending', 'running', 'completed', 'failed'); + +-- ============================================================================ +-- TABLES (in dependency order - parents before children) +-- ============================================================================ + +-- Independent tables (no foreign keys) + +CREATE TABLE dynamics ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + description VARCHAR, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE tax_benefit_models ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + description VARCHAR, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + first_name VARCHAR NOT NULL, + last_name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE UNIQUE INDEX ix_users_email ON users (email); + +-- Tables with single foreign key dependency + +CREATE TABLE datasets ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + description VARCHAR, + filepath VARCHAR NOT NULL, + year INTEGER NOT NULL, + is_output_dataset BOOLEAN NOT NULL, + tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE policies ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + description VARCHAR, + tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE INDEX ix_policies_tax_benefit_model_id ON policies (tax_benefit_model_id); + +CREATE TABLE tax_benefit_model_versions ( + id UUID PRIMARY KEY, + model_id UUID NOT NULL REFERENCES tax_benefit_models(id), + version VARCHAR NOT NULL, + description VARCHAR, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +-- Tables with multiple foreign key dependencies + +CREATE TABLE dataset_versions ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + description VARCHAR NOT NULL, + dataset_id UUID NOT NULL REFERENCES datasets(id), + tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE household_jobs ( + id UUID PRIMARY KEY, + tax_benefit_model_name VARCHAR NOT NULL, + request_data JSON, + policy_id UUID REFERENCES policies(id), + dynamic_id UUID REFERENCES dynamics(id), + status householdjobstatus NOT NULL, + error_message VARCHAR, + result JSON, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + started_at TIMESTAMP WITHOUT TIME ZONE, + completed_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE TABLE parameters ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + label VARCHAR, + description VARCHAR, + data_type VARCHAR, + unit VARCHAR, + tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE simulations ( + id UUID PRIMARY KEY, + dataset_id UUID NOT NULL REFERENCES datasets(id), + policy_id UUID REFERENCES policies(id), + dynamic_id UUID REFERENCES dynamics(id), + tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), + output_dataset_id UUID REFERENCES datasets(id), + status simulationstatus NOT NULL, + error_message VARCHAR, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + started_at TIMESTAMP WITHOUT TIME ZONE, + completed_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE TABLE user_policies ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id), + policy_id UUID NOT NULL REFERENCES policies(id), + label VARCHAR, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE INDEX ix_user_policies_user_id ON user_policies (user_id); +CREATE INDEX ix_user_policies_policy_id ON user_policies (policy_id); + +CREATE TABLE variables ( + id UUID PRIMARY KEY, + name VARCHAR NOT NULL, + entity VARCHAR NOT NULL, + description VARCHAR, + data_type VARCHAR, + possible_values JSON, + tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE parameter_values ( + id UUID PRIMARY KEY, + parameter_id UUID NOT NULL REFERENCES parameters(id), + value_json JSON, + start_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + end_date TIMESTAMP WITHOUT TIME ZONE, + policy_id UUID REFERENCES policies(id), + dynamic_id UUID REFERENCES dynamics(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE reports ( + id UUID PRIMARY KEY, + label VARCHAR NOT NULL, + description VARCHAR, + user_id UUID REFERENCES users(id), + markdown TEXT, + parent_report_id UUID REFERENCES reports(id), + status reportstatus NOT NULL, + error_message VARCHAR, + baseline_simulation_id UUID REFERENCES simulations(id), + reform_simulation_id UUID REFERENCES simulations(id), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE aggregates ( + id UUID PRIMARY KEY, + simulation_id UUID NOT NULL REFERENCES simulations(id), + user_id UUID REFERENCES users(id), + report_id UUID REFERENCES reports(id), + variable VARCHAR NOT NULL, + aggregate_type aggregatetype NOT NULL, + entity VARCHAR, + filter_config JSON, + status aggregatestatus NOT NULL, + error_message VARCHAR, + result FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE change_aggregates ( + id UUID PRIMARY KEY, + baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), + reform_simulation_id UUID NOT NULL REFERENCES simulations(id), + user_id UUID REFERENCES users(id), + report_id UUID REFERENCES reports(id), + variable VARCHAR NOT NULL, + aggregate_type changeaggregatetype NOT NULL, + entity VARCHAR, + filter_config JSON, + change_geq FLOAT, + change_leq FLOAT, + status changeaggregatestatus NOT NULL, + error_message VARCHAR, + result FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE decile_impacts ( + id UUID PRIMARY KEY, + baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), + reform_simulation_id UUID NOT NULL REFERENCES simulations(id), + report_id UUID REFERENCES reports(id), + income_variable VARCHAR NOT NULL, + entity VARCHAR, + decile INTEGER NOT NULL, + quantiles INTEGER NOT NULL, + baseline_mean FLOAT, + reform_mean FLOAT, + absolute_change FLOAT, + relative_change FLOAT, + count_better_off FLOAT, + count_worse_off FLOAT, + count_no_change FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE inequality ( + id UUID PRIMARY KEY, + simulation_id UUID NOT NULL REFERENCES simulations(id), + report_id UUID REFERENCES reports(id), + income_variable VARCHAR NOT NULL, + entity VARCHAR NOT NULL, + gini FLOAT, + top_10_share FLOAT, + top_1_share FLOAT, + bottom_50_share FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE poverty ( + id UUID PRIMARY KEY, + simulation_id UUID NOT NULL REFERENCES simulations(id), + report_id UUID REFERENCES reports(id), + poverty_type VARCHAR NOT NULL, + entity VARCHAR NOT NULL, + filter_variable VARCHAR, + headcount FLOAT, + total_population FLOAT, + rate FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE TABLE program_statistics ( + id UUID PRIMARY KEY, + baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), + reform_simulation_id UUID NOT NULL REFERENCES simulations(id), + report_id UUID REFERENCES reports(id), + program_name VARCHAR NOT NULL, + entity VARCHAR NOT NULL, + is_tax BOOLEAN NOT NULL, + baseline_total FLOAT, + reform_total FLOAT, + change FLOAT, + baseline_count FLOAT, + reform_count FLOAT, + winners FLOAT, + losers FLOAT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL +); From 8c3f4bfe5c8522fd23675ed6a34964e58cac3b3b Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Tue, 3 Feb 2026 11:10:46 -0800 Subject: [PATCH 33/87] fix: Update tests for tax_benefit_model_id and duplicate policy behavior --- test_fixtures/fixtures_parameters.py | 10 ++++++++-- tests/conftest.py | 2 -- tests/test_parameters.py | 4 ++-- tests/test_policies.py | 2 +- tests/test_user_policies.py | 17 +++++++++++------ 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/test_fixtures/fixtures_parameters.py b/test_fixtures/fixtures_parameters.py index ff69b0e..0df134c 100644 --- a/test_fixtures/fixtures_parameters.py +++ b/test_fixtures/fixtures_parameters.py @@ -54,9 +54,15 @@ def create_parameter(session, model_version, name: str, label: str) -> Parameter return param -def create_policy(session, name: str, description: str = "A test policy") -> Policy: +def create_policy( + session, name: str, model_version, description: str = "A test policy" +) -> Policy: """Create and persist a Policy.""" - policy = Policy(name=name, description=description) + policy = Policy( + name=name, + description=description, + tax_benefit_model_id=model_version.model_id, + ) session.add(policy) session.commit() session.refresh(policy) diff --git a/tests/conftest.py b/tests/conftest.py index fe5c04a..77c29ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,5 @@ """Pytest fixtures for tests.""" -from uuid import uuid4 - import pytest from fastapi.testclient import TestClient from fastapi_cache import FastAPICache diff --git a/tests/test_parameters.py b/tests/test_parameters.py index f95016b..50bb213 100644 --- a/tests/test_parameters.py +++ b/tests/test_parameters.py @@ -107,7 +107,7 @@ def test__given_policy_id_filter__then_returns_only_matching_values( """GET /parameter-values?policy_id=X returns only values for that policy.""" # Given param = create_parameter(session, model_version, "test.param", "Test Param") - policy = create_policy(session, "Test Policy") + policy = create_policy(session, "Test Policy", model_version) create_parameter_value(session, param.id, 100, policy_id=None) # baseline create_parameter_value(session, param.id, 150, policy_id=policy.id) # reform @@ -135,7 +135,7 @@ def test__given_both_parameter_and_policy_filters__then_returns_matching_interse param2 = create_parameter( session, model_version, "test.both.param2", "Test Both Param 2" ) - policy = create_policy(session, "Test Both Policy") + policy = create_policy(session, "Test Both Policy", model_version) create_parameter_value(session, param1.id, 100, policy_id=None) # baseline create_parameter_value(session, param1.id, 150, policy_id=policy.id) # target diff --git a/tests/test_policies.py b/tests/test_policies.py index 9c85f96..b4ac25f 100644 --- a/tests/test_policies.py +++ b/tests/test_policies.py @@ -2,7 +2,7 @@ from uuid import uuid4 -from policyengine_api.models import Policy, TaxBenefitModel +from policyengine_api.models import Policy def test_list_policies_empty(client): diff --git a/tests/test_user_policies.py b/tests/test_user_policies.py index 0db9367..f4d1910 100644 --- a/tests/test_user_policies.py +++ b/tests/test_user_policies.py @@ -109,8 +109,8 @@ def test_create_user_policy_policy_not_found(client, session): assert response.json()["detail"] == "Policy not found" -def test_create_user_policy_duplicate(client, session, tax_benefit_model): - """Creating duplicate user-policy association returns 409.""" +def test_create_user_policy_duplicate_allowed(client, session, tax_benefit_model): + """Creating duplicate user-policy association is allowed (matches FE localStorage behavior).""" user = User(first_name="Test", last_name="User", email="test@example.com") policy = Policy( name="Test policy", @@ -131,7 +131,7 @@ def test_create_user_policy_duplicate(client, session, tax_benefit_model): session.add(user_policy) session.commit() - # Try to create duplicate + # Create duplicate - should succeed with a new ID response = client.post( "/user-policies", json={ @@ -139,11 +139,16 @@ def test_create_user_policy_duplicate(client, session, tax_benefit_model): "policy_id": str(policy.id), }, ) - assert response.status_code == 409 - assert "already has an association" in response.json()["detail"] + assert response.status_code == 200 + data = response.json() + assert data["id"] != str(user_policy.id) # New association created + assert data["user_id"] == str(user.id) + assert data["policy_id"] == str(policy.id) -def test_list_user_policies_with_data(client, session, tax_benefit_model, uk_tax_benefit_model): +def test_list_user_policies_with_data( + client, session, tax_benefit_model, uk_tax_benefit_model +): """List user policies returns all associations for a user.""" user = User(first_name="Test", last_name="User", email="test@example.com") policy1 = Policy( From 6f707235f95d791ad8d1c76c1e9419ed49494dc8 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Thu, 5 Feb 2026 23:47:38 -0800 Subject: [PATCH 34/87] feat: Add Alembic migration infrastructure --- .../versions/20260204_0001_initial_schema.py | 538 ++++++++++++++++++ ...20260205_c7254ea1c129_add_user_policies.py | 82 +++ .../migrations/20241115000000_base_schema.sql | 273 --------- uv.lock | 2 +- 4 files changed, 621 insertions(+), 274 deletions(-) create mode 100644 alembic/versions/20260204_0001_initial_schema.py create mode 100644 alembic/versions/20260205_c7254ea1c129_add_user_policies.py delete mode 100644 supabase/migrations/20241115000000_base_schema.sql diff --git a/alembic/versions/20260204_0001_initial_schema.py b/alembic/versions/20260204_0001_initial_schema.py new file mode 100644 index 0000000..f488734 --- /dev/null +++ b/alembic/versions/20260204_0001_initial_schema.py @@ -0,0 +1,538 @@ +"""Initial schema (main branch state) + +Revision ID: 0001_initial +Revises: +Create Date: 2026-02-04 + +This migration creates all base tables for the PolicyEngine API as they +exist on the main branch, BEFORE the household CRUD changes. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0001_initial" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create all tables as they exist on main branch.""" + # ======================================================================== + # TIER 1: Tables with no foreign key dependencies + # ======================================================================== + + # Tax benefit models (e.g., "uk", "us") + op.create_table( + "tax_benefit_models", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # Users + op.create_table( + "users", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("first_name", sa.String(), nullable=False), + sa.Column("last_name", sa.String(), nullable=False), + sa.Column("email", sa.String(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + ) + op.create_index("ix_users_email", "users", ["email"]) + + # Policies (reform definitions) + op.create_table( + "policies", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # Dynamics (behavioral response definitions) + op.create_table( + "dynamics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + # ======================================================================== + # TIER 2: Tables depending on tier 1 + # ======================================================================== + + # Tax benefit model versions + op.create_table( + "tax_benefit_model_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("model_id", sa.Uuid(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["model_id"], ["tax_benefit_models.id"]), + ) + + # Datasets (h5 files in storage) + op.create_table( + "datasets", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("filepath", sa.String(), nullable=False), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("is_output_dataset", sa.Boolean(), nullable=False, default=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), + ) + + # ======================================================================== + # TIER 3: Tables depending on tier 2 + # ======================================================================== + + # Parameters (tax-benefit system parameters) + op.create_table( + "parameters", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("label", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("data_type", sa.String(), nullable=True), + sa.Column("unit", sa.String(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + ) + + # Variables (tax-benefit system variables) + op.create_table( + "variables", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("data_type", sa.String(), nullable=True), + sa.Column("possible_values", sa.JSON(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + ) + + # Dataset versions + op.create_table( + "dataset_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=False), + sa.Column("dataset_id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), + sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), + ) + + # ======================================================================== + # TIER 4: Tables depending on tier 3 + # ======================================================================== + + # Parameter values (policy/dynamic parameter modifications) + op.create_table( + "parameter_values", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("parameter_id", sa.Uuid(), nullable=False), + sa.Column("value_json", sa.JSON(), nullable=True), + sa.Column("start_date", sa.DateTime(timezone=True), nullable=False), + sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["parameter_id"], ["parameters.id"]), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + ) + + # Simulations (economy calculations) - NOTE: No household support yet + op.create_table( + "simulations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("dataset_id", sa.Uuid(), nullable=False), # Required in main + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column("output_dataset_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] + ), + sa.ForeignKeyConstraint(["output_dataset_id"], ["datasets.id"]), + ) + + # Household jobs (async household calculations) - legacy approach + op.create_table( + "household_jobs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_name", sa.String(), nullable=False), + sa.Column("request_data", sa.JSON(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), + ) + + # ======================================================================== + # TIER 5: Tables depending on simulations + # ======================================================================== + + # Reports (analysis reports) - NOTE: No report_type yet + op.create_table( + "reports", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("label", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("markdown", sa.Text(), nullable=True), + sa.Column("parent_report_id", sa.Uuid(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=True), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["parent_report_id"], ["reports.id"]), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + ) + + # Aggregates (single-simulation aggregate outputs) + op.create_table( + "aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("variable", sa.String(), nullable=False), + sa.Column("aggregate_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("filter_config", sa.JSON(), nullable=False, default={}), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Change aggregates (baseline vs reform comparison) + op.create_table( + "change_aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("variable", sa.String(), nullable=False), + sa.Column("aggregate_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("filter_config", sa.JSON(), nullable=False, default={}), + sa.Column("change_geq", sa.Float(), nullable=True), + sa.Column("change_leq", sa.Float(), nullable=True), + sa.Column("status", sa.String(), nullable=False, default="pending"), + sa.Column("error_message", sa.String(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Decile impacts + op.create_table( + "decile_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("income_variable", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=True), + sa.Column("decile", sa.Integer(), nullable=False), + sa.Column("quantiles", sa.Integer(), nullable=False, default=10), + sa.Column("baseline_mean", sa.Float(), nullable=True), + sa.Column("reform_mean", sa.Float(), nullable=True), + sa.Column("absolute_change", sa.Float(), nullable=True), + sa.Column("relative_change", sa.Float(), nullable=True), + sa.Column("count_better_off", sa.Float(), nullable=True), + sa.Column("count_worse_off", sa.Float(), nullable=True), + sa.Column("count_no_change", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Program statistics + op.create_table( + "program_statistics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("program_name", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False), + sa.Column("is_tax", sa.Boolean(), nullable=False, default=False), + sa.Column("baseline_total", sa.Float(), nullable=True), + sa.Column("reform_total", sa.Float(), nullable=True), + sa.Column("change", sa.Float(), nullable=True), + sa.Column("baseline_count", sa.Float(), nullable=True), + sa.Column("reform_count", sa.Float(), nullable=True), + sa.Column("winners", sa.Float(), nullable=True), + sa.Column("losers", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), + ) + + # Poverty + op.create_table( + "poverty", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("poverty_type", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False, default="person"), + sa.Column("filter_variable", sa.String(), nullable=True), + sa.Column("headcount", sa.Float(), nullable=True), + sa.Column("total_population", sa.Float(), nullable=True), + sa.Column("rate", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["simulation_id"], ["simulations.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), + ) + op.create_index("idx_poverty_simulation_id", "poverty", ["simulation_id"]) + op.create_index("idx_poverty_report_id", "poverty", ["report_id"]) + + # Inequality + op.create_table( + "inequality", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=True), + sa.Column("income_variable", sa.String(), nullable=False), + sa.Column("entity", sa.String(), nullable=False, default="household"), + sa.Column("gini", sa.Float(), nullable=True), + sa.Column("top_10_share", sa.Float(), nullable=True), + sa.Column("top_1_share", sa.Float(), nullable=True), + sa.Column("bottom_50_share", sa.Float(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["simulation_id"], ["simulations.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), + ) + op.create_index("idx_inequality_simulation_id", "inequality", ["simulation_id"]) + op.create_index("idx_inequality_report_id", "inequality", ["report_id"]) + + +def downgrade() -> None: + """Drop all tables in reverse order.""" + # Tier 5 + op.drop_index("idx_inequality_report_id", "inequality") + op.drop_index("idx_inequality_simulation_id", "inequality") + op.drop_table("inequality") + op.drop_index("idx_poverty_report_id", "poverty") + op.drop_index("idx_poverty_simulation_id", "poverty") + op.drop_table("poverty") + op.drop_table("program_statistics") + op.drop_table("decile_impacts") + op.drop_table("change_aggregates") + op.drop_table("aggregates") + op.drop_table("reports") + + # Tier 4 + op.drop_table("household_jobs") + op.drop_table("simulations") + op.drop_table("parameter_values") + + # Tier 3 + op.drop_table("dataset_versions") + op.drop_table("variables") + op.drop_table("parameters") + + # Tier 2 + op.drop_table("datasets") + op.drop_table("tax_benefit_model_versions") + + # Tier 1 + op.drop_table("dynamics") + op.drop_table("policies") + op.drop_index("ix_users_email", "users") + op.drop_table("users") + op.drop_table("tax_benefit_models") diff --git a/alembic/versions/20260205_c7254ea1c129_add_user_policies.py b/alembic/versions/20260205_c7254ea1c129_add_user_policies.py new file mode 100644 index 0000000..3e1470d --- /dev/null +++ b/alembic/versions/20260205_c7254ea1c129_add_user_policies.py @@ -0,0 +1,82 @@ +"""add_user_policies + +Revision ID: c7254ea1c129 +Revises: 0001_initial +Create Date: 2026-02-05 23:28:58.822168 + +This migration adds: +1. tax_benefit_model_id foreign key to policies table +2. user_policies table for user-policy associations +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c7254ea1c129" +down_revision: Union[str, Sequence[str], None] = "0001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add user_policies table and policy.tax_benefit_model_id.""" + # Add tax_benefit_model_id to policies table + op.add_column( + "policies", sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False) + ) + op.create_index( + op.f("ix_policies_tax_benefit_model_id"), + "policies", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_foreign_key( + "fk_policies_tax_benefit_model_id", + "policies", + "tax_benefit_models", + ["tax_benefit_model_id"], + ["id"], + ) + + # Create user_policies table + op.create_table( + "user_policies", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_user_policies_policy_id"), + "user_policies", + ["policy_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_policies_user_id"), "user_policies", ["user_id"], unique=False + ) + + +def downgrade() -> None: + """Remove user_policies table and policy.tax_benefit_model_id.""" + # Drop user_policies table + op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") + op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") + op.drop_table("user_policies") + + # Remove tax_benefit_model_id from policies + op.drop_constraint( + "fk_policies_tax_benefit_model_id", "policies", type_="foreignkey" + ) + op.drop_index(op.f("ix_policies_tax_benefit_model_id"), table_name="policies") + op.drop_column("policies", "tax_benefit_model_id") diff --git a/supabase/migrations/20241115000000_base_schema.sql b/supabase/migrations/20241115000000_base_schema.sql deleted file mode 100644 index 1b1a0ce..0000000 --- a/supabase/migrations/20241115000000_base_schema.sql +++ /dev/null @@ -1,273 +0,0 @@ --- Base schema for PolicyEngine API v2 --- This migration creates all tables required by SQLModel definitions - --- ============================================================================ --- ENUM TYPES --- ============================================================================ - -CREATE TYPE householdjobstatus AS ENUM ('pending', 'running', 'completed', 'failed'); -CREATE TYPE simulationstatus AS ENUM ('pending', 'running', 'completed', 'failed'); -CREATE TYPE reportstatus AS ENUM ('pending', 'running', 'completed', 'failed'); -CREATE TYPE aggregatetype AS ENUM ('sum', 'mean', 'count'); -CREATE TYPE aggregatestatus AS ENUM ('pending', 'running', 'completed', 'failed'); -CREATE TYPE changeaggregatetype AS ENUM ('sum', 'mean', 'count'); -CREATE TYPE changeaggregatestatus AS ENUM ('pending', 'running', 'completed', 'failed'); - --- ============================================================================ --- TABLES (in dependency order - parents before children) --- ============================================================================ - --- Independent tables (no foreign keys) - -CREATE TABLE dynamics ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE tax_benefit_models ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE users ( - id UUID PRIMARY KEY, - first_name VARCHAR NOT NULL, - last_name VARCHAR NOT NULL, - email VARCHAR NOT NULL, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE UNIQUE INDEX ix_users_email ON users (email); - --- Tables with single foreign key dependency - -CREATE TABLE datasets ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR, - filepath VARCHAR NOT NULL, - year INTEGER NOT NULL, - is_output_dataset BOOLEAN NOT NULL, - tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE policies ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR, - tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE INDEX ix_policies_tax_benefit_model_id ON policies (tax_benefit_model_id); - -CREATE TABLE tax_benefit_model_versions ( - id UUID PRIMARY KEY, - model_id UUID NOT NULL REFERENCES tax_benefit_models(id), - version VARCHAR NOT NULL, - description VARCHAR, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - --- Tables with multiple foreign key dependencies - -CREATE TABLE dataset_versions ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR NOT NULL, - dataset_id UUID NOT NULL REFERENCES datasets(id), - tax_benefit_model_id UUID NOT NULL REFERENCES tax_benefit_models(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE household_jobs ( - id UUID PRIMARY KEY, - tax_benefit_model_name VARCHAR NOT NULL, - request_data JSON, - policy_id UUID REFERENCES policies(id), - dynamic_id UUID REFERENCES dynamics(id), - status householdjobstatus NOT NULL, - error_message VARCHAR, - result JSON, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - started_at TIMESTAMP WITHOUT TIME ZONE, - completed_at TIMESTAMP WITHOUT TIME ZONE -); - -CREATE TABLE parameters ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - label VARCHAR, - description VARCHAR, - data_type VARCHAR, - unit VARCHAR, - tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE simulations ( - id UUID PRIMARY KEY, - dataset_id UUID NOT NULL REFERENCES datasets(id), - policy_id UUID REFERENCES policies(id), - dynamic_id UUID REFERENCES dynamics(id), - tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), - output_dataset_id UUID REFERENCES datasets(id), - status simulationstatus NOT NULL, - error_message VARCHAR, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - started_at TIMESTAMP WITHOUT TIME ZONE, - completed_at TIMESTAMP WITHOUT TIME ZONE -); - -CREATE TABLE user_policies ( - id UUID PRIMARY KEY, - user_id UUID NOT NULL REFERENCES users(id), - policy_id UUID NOT NULL REFERENCES policies(id), - label VARCHAR, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE INDEX ix_user_policies_user_id ON user_policies (user_id); -CREATE INDEX ix_user_policies_policy_id ON user_policies (policy_id); - -CREATE TABLE variables ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - entity VARCHAR NOT NULL, - description VARCHAR, - data_type VARCHAR, - possible_values JSON, - tax_benefit_model_version_id UUID NOT NULL REFERENCES tax_benefit_model_versions(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE parameter_values ( - id UUID PRIMARY KEY, - parameter_id UUID NOT NULL REFERENCES parameters(id), - value_json JSON, - start_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, - end_date TIMESTAMP WITHOUT TIME ZONE, - policy_id UUID REFERENCES policies(id), - dynamic_id UUID REFERENCES dynamics(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE reports ( - id UUID PRIMARY KEY, - label VARCHAR NOT NULL, - description VARCHAR, - user_id UUID REFERENCES users(id), - markdown TEXT, - parent_report_id UUID REFERENCES reports(id), - status reportstatus NOT NULL, - error_message VARCHAR, - baseline_simulation_id UUID REFERENCES simulations(id), - reform_simulation_id UUID REFERENCES simulations(id), - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE aggregates ( - id UUID PRIMARY KEY, - simulation_id UUID NOT NULL REFERENCES simulations(id), - user_id UUID REFERENCES users(id), - report_id UUID REFERENCES reports(id), - variable VARCHAR NOT NULL, - aggregate_type aggregatetype NOT NULL, - entity VARCHAR, - filter_config JSON, - status aggregatestatus NOT NULL, - error_message VARCHAR, - result FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE change_aggregates ( - id UUID PRIMARY KEY, - baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), - reform_simulation_id UUID NOT NULL REFERENCES simulations(id), - user_id UUID REFERENCES users(id), - report_id UUID REFERENCES reports(id), - variable VARCHAR NOT NULL, - aggregate_type changeaggregatetype NOT NULL, - entity VARCHAR, - filter_config JSON, - change_geq FLOAT, - change_leq FLOAT, - status changeaggregatestatus NOT NULL, - error_message VARCHAR, - result FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE decile_impacts ( - id UUID PRIMARY KEY, - baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), - reform_simulation_id UUID NOT NULL REFERENCES simulations(id), - report_id UUID REFERENCES reports(id), - income_variable VARCHAR NOT NULL, - entity VARCHAR, - decile INTEGER NOT NULL, - quantiles INTEGER NOT NULL, - baseline_mean FLOAT, - reform_mean FLOAT, - absolute_change FLOAT, - relative_change FLOAT, - count_better_off FLOAT, - count_worse_off FLOAT, - count_no_change FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE inequality ( - id UUID PRIMARY KEY, - simulation_id UUID NOT NULL REFERENCES simulations(id), - report_id UUID REFERENCES reports(id), - income_variable VARCHAR NOT NULL, - entity VARCHAR NOT NULL, - gini FLOAT, - top_10_share FLOAT, - top_1_share FLOAT, - bottom_50_share FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE poverty ( - id UUID PRIMARY KEY, - simulation_id UUID NOT NULL REFERENCES simulations(id), - report_id UUID REFERENCES reports(id), - poverty_type VARCHAR NOT NULL, - entity VARCHAR NOT NULL, - filter_variable VARCHAR, - headcount FLOAT, - total_population FLOAT, - rate FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); - -CREATE TABLE program_statistics ( - id UUID PRIMARY KEY, - baseline_simulation_id UUID NOT NULL REFERENCES simulations(id), - reform_simulation_id UUID NOT NULL REFERENCES simulations(id), - report_id UUID REFERENCES reports(id), - program_name VARCHAR NOT NULL, - entity VARCHAR NOT NULL, - is_tax BOOLEAN NOT NULL, - baseline_total FLOAT, - reform_total FLOAT, - change FLOAT, - baseline_count FLOAT, - reform_count FLOAT, - winners FLOAT, - losers FLOAT, - created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL -); diff --git a/uv.lock b/uv.lock index 466caf4..f66f5d0 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]] From 5d7388c6b1abde24f483374656951207c10f6298 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Thu, 5 Feb 2026 23:56:31 -0800 Subject: [PATCH 35/87] chore: Rename migration to 0002 for readability --- .DS_Store | Bin 0 -> 6148 bytes ....py => 20260205_0002_add_user_policies.py} | 6 +- docs/api-v2-alpha-architecture.md | 264 +++++++++++++++++ docs/policy-ingredient-status.md | 271 ++++++++++++++++++ docs/supabase-migration-notes.md | 174 +++++++++++ src/policyengine_api/agent_sandbox.py | 45 +-- src/policyengine_api/api/agent.py | 22 +- src/policyengine_api/api/household.py | 29 +- src/policyengine_api/api/policies.py | 4 +- src/policyengine_api/api/user_policies.py | 10 +- src/policyengine_api/api/variables.py | 4 +- src/policyengine_api/modal_app.py | 8 +- tests/test_agent.py | 1 + tests/test_agent_policy_questions.py | 8 +- tests/test_integration.py | 1 + 15 files changed, 796 insertions(+), 51 deletions(-) create mode 100644 .DS_Store rename alembic/versions/{20260205_c7254ea1c129_add_user_policies.py => 20260205_0002_add_user_policies.py} (96%) create mode 100644 docs/api-v2-alpha-architecture.md create mode 100644 docs/policy-ingredient-status.md create mode 100644 docs/supabase-migration-notes.md diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4bf6461c4c4dcb4ac126e58927c5525107ec899b GIT binary patch literal 6148 zcmeHKK~BRk5FA4hEr^0RaY4!lsKgJ_QVvLPq2dfQEvS$-C~Xnka_0|xf%kEOS=)ld z1&I?vXjj@D$KLg3C$(JzFrE2eAJ_s=rwZ2AXuc2`7hRAg-?K?Hc8)F%(ZU#mqjbSr zJCp%s;IAW)Yh97wt#I6kRib>hRCtUR>oC-M>}Ot z3y*wCkJ8Ehmcf|@L!s>I0)m5^WFxQ9~ zX9#M9@K^R;rgx4cU_uxxd)4VR=X{swQ~7lQe_^id-(Y6>ra8T@SZ9uL5Fs3uBQ|Wr zl=hqvBRsG*XWTHXz%o>hT=|F$Yb7(LSz%_5dorK8UrhClgqeyrUr^--49JS0)Yl!V zRR)v+W#EGW*&iaRVC=DQXtxd;dj%lY*lmSv`NJSFk;m9$;gBN~<3fop)c7lgapCNb zykG3GaOlEe{N=;=%*NkPjLpvZBkK+mJ5;L-C<9dnmfho)od1X4-~X#Y`lJjf1OJKv zQ|tD+9hT(J)`jBatc|D_R1xtj9NG{Xe;nI}9K}0SD;$fYL5w{X4%tJ|KLVBpHOj!R GGVl%&a?hs# literal 0 HcmV?d00001 diff --git a/alembic/versions/20260205_c7254ea1c129_add_user_policies.py b/alembic/versions/20260205_0002_add_user_policies.py similarity index 96% rename from alembic/versions/20260205_c7254ea1c129_add_user_policies.py rename to alembic/versions/20260205_0002_add_user_policies.py index 3e1470d..b19a6cf 100644 --- a/alembic/versions/20260205_c7254ea1c129_add_user_policies.py +++ b/alembic/versions/20260205_0002_add_user_policies.py @@ -1,8 +1,8 @@ """add_user_policies -Revision ID: c7254ea1c129 +Revision ID: 0002_user_policies Revises: 0001_initial -Create Date: 2026-02-05 23:28:58.822168 +Create Date: 2026-02-05 This migration adds: 1. tax_benefit_model_id foreign key to policies table @@ -17,7 +17,7 @@ from alembic import op # revision identifiers, used by Alembic. -revision: str = "c7254ea1c129" +revision: str = "0002_user_policies" down_revision: Union[str, Sequence[str], None] = "0001_initial" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/docs/api-v2-alpha-architecture.md b/docs/api-v2-alpha-architecture.md new file mode 100644 index 0000000..10924db --- /dev/null +++ b/docs/api-v2-alpha-architecture.md @@ -0,0 +1,264 @@ +# policyengine-api-v2-alpha Deep Dive + +## Overview + +A **FastAPI backend** for tax-benefit policy microsimulations using PolicyEngine's UK and US models. It's designed as a thin orchestration layer that offloads heavy calculations to **Modal.com** serverless functions. + +--- + +## Architecture + +### Three-Level Hierarchy +``` +Level 2: Reports AI-generated analysis documents +Level 1: Analyses Comparisons (economy_comparison_*) +Level 0: Simulations Single calculations (simulate_household_*, simulate_economy_*) +``` + +### Request Flow +1. Client → FastAPI (Cloud Run) +2. API creates job record in **Supabase** and triggers **Modal.com** function +3. Modal runs calculation with pre-loaded PolicyEngine models (sub-1s cold start via memory snapshotting) +4. Modal writes results directly to Supabase +5. Client polls API until `status = "completed"` + +--- + +## Tech Stack + +| Component | Technology | +|-----------|------------| +| Framework | FastAPI + async | +| Database | Supabase (PostgreSQL) via SQLModel | +| Compute | Modal.com serverless | +| Storage | Supabase Storage (S3-compatible for .h5 datasets) | +| Package mgr | UV | +| Formatting | Ruff | +| Testing | Pytest + pytest-asyncio | +| Deployment | Terraform → GCP Cloud Run | +| Observability | Logfire (OpenTelemetry) | + +--- + +## Database Models (SQLModel) + +### Core Entities + +| Model | Purpose | Key Fields | +|-------|---------|------------| +| `TaxBenefitModel` | Country identifier | `name` ("uk"/"us") | +| `TaxBenefitModelVersion` | Version of model | `version`, FK to model | +| `Dataset` | Population microdata (.h5) | `filepath` (S3), `year` | +| `DatasetVersion` | Dataset versions | FK to dataset + model | +| `User` | Basic user info | `email`, `first_name`, `last_name` | + +### Policy System (Normalized) + +| Model | Purpose | Key Fields | +|-------|---------|------------| +| `Policy` | Policy reform definition | `name`, `description`, `simulation_modifier` (Python code) | +| `Parameter` | Tax/benefit parameter metadata | `name` (path), `label`, `data_type`, FK to model version | +| `ParameterValue` | Parameter values (baseline OR reform) | `value_json`, `start_date`, `policy_id` (NULL=baseline) | +| `Variable` | Model variables metadata | `name`, `entity`, `data_type`, `possible_values` | +| `Dynamic` | Behavioral response config | Similar to Policy | + +### Simulation & Results + +| Model | Purpose | Key Fields | +|-------|---------|------------| +| `Simulation` | Economy simulation instance | FK to dataset, policy, dynamic; `status` | +| `HouseholdJob` | Single household calculation | `request_data` (JSON), `result` (JSON), `status` | +| `Report` | Analysis report | FK to baseline_sim, reform_sim; `status`, `markdown` | +| `DecileImpact` | Income impacts by decile | FK to baseline/reform sims; `absolute_change`, `relative_change` | +| `ProgramStatistics` | Budget impacts by program | `baseline_total`, `reform_total`, `change` | +| `Poverty` | Poverty rate outputs | `poverty_type`, `headcount`, `rate` | +| `Inequality` | Inequality metrics | `gini`, `top_10_share`, `bottom_50_share` | +| `AggregateOutput` | Custom aggregations | `variable`, `aggregate_type`, `filter_config` | +| `ChangeAggregate` | Change aggregations | baseline vs reform comparison | + +--- + +## API Endpoints + +### Registered Routers (`api/__init__.py`) +``` +/datasets - Dataset CRUD + listing +/policies - Policy CRUD (POST, GET, LIST - no PATCH/DELETE) +/simulations - Simulation management +/parameters - Parameter metadata +/parameter-values - Parameter value CRUD +/variables - Variable metadata +/dynamics - Behavioral response configs +/tax-benefit-models - Model listing +/tax-benefit-model-versions - Version listing +/outputs - Result outputs +/change-aggregates - Comparison aggregates +/household/* - Household calculations (async) +/analysis/* - Economy-wide analysis (async) +/agent/* - AI agent endpoint (Claude Code) +``` + +### Key Endpoint Patterns + +**Async Pattern** (household, analysis): +``` +POST /household/calculate → {"job_id": "...", "status": "pending"} +GET /household/calculate/{job_id} → poll until status="completed" +``` + +**CRUD Pattern** (policies, datasets): +``` +POST /policies - Create +GET /policies - List all +GET /policies/{id} - Get by ID +(PATCH, DELETE - NOT implemented) +``` + +--- + +## Modal.com Functions (`modal_app.py`) + +| Function | Purpose | Resources | +|----------|---------|-----------| +| `simulate_household_uk` | UK household calc | 4GB RAM, 4 CPU | +| `simulate_household_us` | US household calc | 4GB RAM, 4 CPU | +| `simulate_economy_uk` | UK economy sim | 8GB RAM, 8 CPU | +| `simulate_economy_us` | US economy sim | 8GB RAM, 8 CPU | +| `economy_comparison_uk` | UK decile/budget analysis | 8GB RAM, 8 CPU, 30min timeout | +| `economy_comparison_us` | US decile/budget analysis | 8GB RAM, 8 CPU, 30min timeout | + +### Key Feature: Memory Snapshotting +```python +# UK image - uses run_function to snapshot imported modules in memory +uk_image = base_image.run_commands( + "uv pip install --system policyengine-uk>=2.0.0" +).run_function(_import_uk) # ← pre-loads uk_latest at build time +``` +This enables **sub-1s cold starts** - PolicyEngine models are already loaded in memory. + +--- + +## Database Migrations (`supabase/migrations/`) + +| Migration | Purpose | +|-----------|---------| +| `20241119000000_storage_bucket.sql` | Create datasets S3 bucket | +| `20241121000000_storage_policies.sql` | Storage access policies | +| `20251229000000_add_parameter_values_indexes.sql` | Index optimization | +| `20260103000000_add_poverty_inequality.sql` | Poverty/inequality tables | +| `20260108000000_add_simulation_modifier.sql` | Add `simulation_modifier` to policies | + +**Note**: Tables are auto-created by SQLModel on app startup, not via migrations. Migrations are only for: +- Storage bucket setup +- Adding indexes +- Schema alterations + +--- + +## Agent Endpoint (`/agent/*`) + +An AI-powered policy analysis feature using Claude Code: +- `POST /agent/run` - Start agent with a question +- `GET /agent/logs/{call_id}` - Poll for logs and result +- `GET /agent/status/{call_id}` - Quick status check +- `POST /agent/log/{call_id}` - Modal calls this to stream logs +- `POST /agent/complete/{call_id}` - Modal calls this when done + +Runs in: +- **Production**: Modal sandbox +- **Development**: Local background thread + +--- + +## Configuration (`config/settings.py`) + +Key settings loaded from environment: +- `DATABASE_URL` - Supabase Postgres connection +- `SUPABASE_URL` / `SUPABASE_KEY` - Supabase client +- `POLICYENGINE_API_URL` - Self-reference URL +- `AGENT_USE_MODAL` - True for production, False for local + +--- + +## Key Design Decisions + +1. **Normalized parameter storage** - `parameter_values` table with FKs instead of JSON blobs +2. **Async job pattern** - All heavy calculations return immediately with job_id +3. **Deterministic UUIDs** - Simulations/reports use uuid5 for deduplication +4. **Memory snapshotting** - PolicyEngine models loaded at Modal image build time +5. **SQLModel** - Single source of truth for DB schema and Pydantic validation +6. **simulation_modifier** - Python code injection for custom variable formulas + +--- + +## What's Missing (Gaps) + +| Feature | Status | +|---------|--------| +| `PATCH /policies/{id}` | Not implemented | +| `DELETE /policies/{id}` | Not implemented | +| User-policy associations | Not implemented | +| User-simulation associations | Not implemented | +| Authentication | No endpoints exposed | + +--- + +## File Structure + +``` +src/policyengine_api/ +├── api/ # FastAPI routers +│ ├── __init__.py # Router registration +│ ├── agent.py # AI agent endpoint +│ ├── analysis.py # Economy analysis +│ ├── household.py # Household calculations +│ ├── policies.py # Policy CRUD +│ ├── parameters.py # Parameter metadata +│ ├── parameter_values.py # Parameter values +│ └── ... +├── config/ +│ └── settings.py # Environment config +├── models/ # SQLModel definitions +│ ├── __init__.py # Model exports +│ ├── policy.py +│ ├── parameter.py +│ ├── parameter_value.py +│ ├── simulation.py +│ ├── report.py +│ └── ... +├── services/ +│ ├── database.py # DB session management +│ └── storage.py # Supabase storage client +├── main.py # FastAPI app entry point +└── modal_app.py # Modal serverless functions + +supabase/ +└── migrations/ # SQL migrations (storage, indexes) + +terraform/ # GCP Cloud Run infrastructure + +tests/ # Pytest tests +``` + +--- + +## Development Commands + +```bash +make install # Install dependencies with uv +make dev # Start supabase + api via docker compose +make test # Run unit tests +make integration-test # Full integration tests +make format # Ruff formatting +make lint # Ruff linting with auto-fix +make modal-deploy # Deploy Modal.com serverless functions +make init # Reset tables and storage +make seed # Populate UK/US models with variables, parameters, datasets +``` + +--- + +## Contributors + +- **Nikhil Woodruff** - Initial implementation (Nov 2025) +- **Anthony Volk** - Parameter values filtering and indexing (Dec 2025) diff --git a/docs/policy-ingredient-status.md b/docs/policy-ingredient-status.md new file mode 100644 index 0000000..e9c79a4 --- /dev/null +++ b/docs/policy-ingredient-status.md @@ -0,0 +1,271 @@ +# Policy Ingredient Status in API v2 Alpha + +This document summarizes the current state of the policy ingredient implementation in `policyengine-api-v2-alpha`, compares it to API v1, and outlines gaps and recommendations for migration. + +## Current Implementation + +### Database Tables + +The policy system uses a normalized relational model with two main tables: + +#### `policies` table +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key, auto-generated | +| `name` | str | Policy name (required) | +| `description` | str \| None | Optional description | +| `simulation_modifier` | str \| None | Python code for custom variable formulas | +| `created_at` | datetime | Auto-generated UTC timestamp | +| `updated_at` | datetime | Auto-generated UTC timestamp | + +#### `parameter_values` table +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key | +| `parameter_id` | UUID | FK to `parameters` table | +| `policy_id` | UUID \| None | FK to `policies` (NULL = baseline/current law) | +| `dynamic_id` | UUID \| None | FK to `dynamics` table | +| `value_json` | Any | The actual parameter value (JSON) | +| `start_date` | datetime | Effective start date | +| `end_date` | datetime \| None | Effective end date | +| `created_at` | datetime | Auto-generated | + +**Key design:** `parameter_values` stores BOTH baseline values (`policy_id IS NULL`) AND reform values (`policy_id = `). Each policy is self-contained with its own set of parameter value overrides - this is intentional, not a normalization issue. When running simulations, baseline (NULL) vs reform (policy_id = X) values are compared to compute impacts. + +### Supporting Tables + +#### `parameters` table (metadata) +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key | +| `name` | str | Parameter path (e.g., `gov.irs.ctc.max`) | +| `label` | str \| None | Human-readable label | +| `description` | str \| None | Documentation | +| `data_type` | str \| None | Value type | +| `unit` | str \| None | Unit of measurement | +| `tax_benefit_model_version_id` | UUID | FK to model version | +| `created_at` | datetime | Auto-generated | + +#### `tax_benefit_models` table +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key | +| `name` | str | Country identifier ("uk" or "us") | +| `description` | str \| None | Optional | +| `created_at` | datetime | Auto-generated | + +### API Endpoints + +| Method | Path | Status | Description | +|--------|------|--------|-------------| +| `POST` | `/policies` | ✅ Implemented | Create policy with parameter values | +| `GET` | `/policies` | ✅ Implemented | List all policies | +| `GET` | `/policies/{id}` | ✅ Implemented | Get policy by ID | +| `PATCH` | `/policies/{id}` | ❌ Missing | Update policy | +| `DELETE` | `/policies/{id}` | ❌ Missing | Delete policy | + +### Files + +| File | Purpose | +|------|---------| +| `src/policyengine_api/models/policy.py` | SQLModel definitions | +| `src/policyengine_api/models/parameter_value.py` | Parameter value model | +| `src/policyengine_api/api/policies.py` | Policy router/endpoints | +| `src/policyengine_api/api/parameter_values.py` | Parameter values router | +| `tests/test_policies.py` | Basic CRUD tests | + +### Contributors + +- **Nikhil Woodruff** - Initial implementation (Nov 2025) +- **Anthony Volk** - Added parameter_values filtering and indexing (Dec 2025) + +--- + +## Comparison: API v1 vs API v2 Alpha + +### URL Structure + +| Aspect | API v1 | API v2 Alpha | +|--------|--------|--------------| +| Create | `POST /{country_id}/policy` | `POST /policies` | +| Get | `GET /{country_id}/policy/{id}` | `GET /policies/{id}` | +| List | `GET /{country_id}/policies` | `GET /policies` | + +**Key difference:** v1 has country in URL path; v2 does not. + +### Country Handling + +| API v1 | API v2 Alpha | +|--------|--------------| +| Country in URL path (`/us/policy/123`) | Country determined by `tax_benefit_model_name` in request body | +| `@validate_country` decorator | No explicit country validation on policy endpoints | +| Policy implicitly scoped to country | Policy is country-agnostic; country context from analysis request | + +### Policy Storage + +| API v1 | API v2 Alpha | +|--------|--------------| +| `policy_hash` - hash of JSON blob | `parameter_values` - relational FK to parameters | +| Policy content stored as JSON | Policy content normalized in separate table | +| Embedded values in metadata response | Separate queries for parameter values | + +### Metadata Response (Parameters) + +**API v1 - Embedded values:** +```json +{ + "gov.irs.ctc.max": { + "type": "parameter", + "label": "CTC maximum", + "values": { + "2024-01-01": 2000, + "2023-01-01": 2000, + "2022-01-01": 2000 + } + } +} +``` + +**API v2 Alpha - Normalized:** +``` +-- parameters table +id | name | label +1 | gov.irs.ctc.max | CTC maximum + +-- parameter_values table +id | parameter_id | start_date | value_json | policy_id +1 | 1 | 2024-01-01 | 2000 | NULL (baseline) +2 | 1 | 2024-01-01 | 3000 | +``` + +### User Associations + +| API v1 | API v2 Alpha | +|--------|--------------| +| `user_policies` table exists in code | Not implemented | +| Endpoints NOT exposed (dead code) | No user association tables/endpoints | +| Fields: `user_id`, `reform_id`, `baseline_id`, `year`, `geography`, etc. | N/A | + +--- + +## Pros and Cons + +### API v2 Alpha Advantages + +| Pro | Explanation | +|-----|-------------| +| **Normalized schema** | Easier to query specific parameter changes without parsing JSON | +| **Relational integrity** | FK constraints ensure valid parameter references | +| **Better indexing** | Can index on `parameter_id`, `policy_id`, `start_date` | +| **Audit trail** | Each parameter value has its own `created_at` | +| **Cleaner reform diffs** | Query `WHERE policy_id = X` to see all reform changes | + +### API v2 Alpha Disadvantages + +| Con | Explanation | +|-----|-------------| +| **No country on policy** | Can't filter policies by country at DB level | +| **No user associations** | Must be built from scratch | +| **Missing PATCH/DELETE** | Incomplete CRUD | +| **No label field** | Only `name` + `description`, no user-friendly `label` | +| **More complex queries** | JOIN required to get policy with values | + +### API v1 Advantages + +| Pro | Explanation | +|-----|-------------| +| **Country in URL** | Clear API contract, easy filtering | +| **Simple storage** | Policy is a single JSON blob | +| **User associations designed** | Schema exists (though not exposed) | + +### API v1 Disadvantages + +| Con | Explanation | +|-----|-------------| +| **JSON blob parsing** | Must parse to query specific parameters | +| **No referential integrity** | Policy JSON could reference invalid parameters | +| **Harder to diff** | Must compare two JSON blobs to see changes | + +--- + +## Gaps for Migration + +### Must Have + +1. **User associations table and endpoints** + - `user_policies` table with `user_id`, `policy_id`, `label`, `created_at` + - `POST /user-policies` - Create association + - `GET /user-policies?user_id=X` - List user's policies + - `DELETE /user-policies/{id}` - Remove association + +2. **PATCH endpoint for policies** + - Update `name`, `description` + - Update parameter values + +3. **DELETE endpoint for policies** + - Cascade delete parameter values + +### Should Have + +4. **Country validation** + - Either add `country_id` to policy model OR + - Validate at creation that all parameter_ids belong to same country + +5. **Label field on policy** + - User-friendly display name separate from `name` + +### Nice to Have + +6. **Soft delete** + - `deleted_at` field instead of hard delete + +7. **Policy versioning** + - Track changes over time + +--- + +## Suggested Schema Changes + +### Option A: Add country_id to Policy + +```python +class Policy(PolicyBase, table=True): + # ... existing fields ... + country_id: str # "us" or "uk" +``` + +**Pros:** Simple filtering, matches v1 pattern +**Cons:** Redundant with parameter → tax_benefit_model → name + +### Option B: Derive country from parameters (current approach) + +Keep as-is, derive country from first parameter's tax_benefit_model. + +**Pros:** No schema change, DRY +**Cons:** Requires JOIN to filter by country + +### Recommendation: Option A + +Add explicit `country_id` for simpler queries and clearer data model. + +--- + +## Next Steps + +1. [ ] Add `country_id` to Policy model +2. [ ] Add `label` field to Policy model +3. [ ] Create UserPolicy model and table +4. [ ] Implement `PATCH /policies/{id}` endpoint +5. [ ] Implement `DELETE /policies/{id}` endpoint +6. [ ] Implement user policy association endpoints +7. [ ] Add database migration for schema changes +8. [ ] Update tests + +--- + +## References + +- Policy model: `src/policyengine_api/models/policy.py` +- Policy router: `src/policyengine_api/api/policies.py` +- Parameter value model: `src/policyengine_api/models/parameter_value.py` +- Tests: `tests/test_policies.py` diff --git a/docs/supabase-migration-notes.md b/docs/supabase-migration-notes.md new file mode 100644 index 0000000..b3c09d7 --- /dev/null +++ b/docs/supabase-migration-notes.md @@ -0,0 +1,174 @@ +# Supabase & Database Migration Notes + +## Overview + +This document summarizes the database setup, migration issues, and workarounds discovered during local development of the PolicyEngine API v2. + +--- + +## What is Supabase? + +Supabase is a backend-as-a-service platform that provides: +- **PostgreSQL database** - The actual database +- **PostgREST** - Auto-generated REST API from your database schema +- **GoTrue** - Authentication service +- **Kong** - API gateway +- **Studio** - Web UI for managing your database +- **Realtime** - WebSocket subscriptions for live updates +- **Storage** - File storage with CDN + +### Local Development with Supabase + +Running `supabase start` spins up Docker containers for all these services locally: +- PostgreSQL on port **54322** +- Studio UI on port **54323** +- API on port **54321** + +--- + +## The Migration Bug + +### What Happened + +When running `supabase start` on a fresh machine, it failed with: + +``` +ERROR: relation "parameter_values" does not exist (SQLSTATE 42P01) +``` + +### Root Cause + +The migration file `20251229000000_add_parameter_values_indexes.sql` tries to create an index on the `parameter_values` table: + +```sql +CREATE INDEX idx_parameter_values_parameter_id ON parameter_values(parameter_id); +``` + +But there's **no base schema migration** that creates the tables first. The migrations run in timestamp order, so this index migration runs before any tables exist. + +### Why This Wasn't Caught Before + +Different developers may have had working setups because: + +1. **SQLite for local dev** - Using `SUPABASE_DB_URL="sqlite:///./test.db"` bypasses Supabase entirely. SQLModel's `create_all()` auto-creates tables. + +2. **Existing PostgreSQL** - If someone had previously created tables manually or through the API startup, the migration would succeed. + +3. **Never ran `supabase start`** - The migrations only run when Supabase starts fresh. + +--- + +## Two Ways Tables Get Created + +### 1. SQLModel's `create_all()` (Automatic) + +When the API starts, this code runs in `database.py`: + +```python +def init_db(): + SQLModel.metadata.create_all(engine) +``` + +This automatically creates all tables defined in SQLModel classes. It's idempotent (safe to run multiple times) but: +- Doesn't track schema changes over time +- Can't roll back changes +- Doesn't work for Supabase migrations (they run BEFORE the API starts) + +### 2. Migration Scripts (Manual/Versioned) + +SQL files in `supabase/migrations/` that run in timestamp order: +- `20241119000000_storage_bucket.sql` +- `20241121000000_storage_policies.sql` +- `20251229000000_add_parameter_values_indexes.sql` ← Problem file +- `20260103000000_add_poverty_inequality.sql` +- `20260111000000_add_aggregate_status.sql` + +**Why migrations exist:** +- Version-controlled schema changes +- Can roll back to previous versions +- Team members get same schema automatically +- Production deployments are reproducible + +--- + +## The Fix (Not Yet Implemented) + +Add a base schema migration with an earlier timestamp that creates all tables: + +``` +supabase/migrations/20241101000000_base_schema.sql +``` + +This file should contain `CREATE TABLE` statements for all tables, generated from the SQLModel definitions. + +--- + +## Workaround: Use SQLite for Local Testing + +For quick local development without Docker/Supabase: + +```bash +cd /Users/sakshikekre/Work/PolicyEngine/Repos/policyengine-api-v2-alpha + +# Start API with SQLite instead of PostgreSQL +SUPABASE_DB_URL="sqlite:///./test.db" .venv/bin/uvicorn policyengine_api.main:app --host 0.0.0.0 --port 8000 --reload +``` + +This: +- Creates a local `test.db` file +- SQLModel auto-creates all tables on startup +- No Docker required +- Fast iteration for development + +--- + +## Docker/Colima Setup (For Full Supabase) + +Supabase local requires Docker. Options: + +### Option A: Docker Desktop (~2-3GB) +```bash +brew install --cask docker +# Then open Docker Desktop app +``` + +### Option B: Colima (Lightweight, ~500MB) +```bash +brew install colima docker docker-compose +colima start +``` + +**Note:** Colima installation failed due to disk space. Need ~2GB free for the build process. + +--- + +## Current .env Configuration + +```env +# For Supabase (when running) +SUPABASE_URL=http://127.0.0.1:54321 +SUPABASE_DB_URL=postgresql://postgres:postgres@127.0.0.1:54322/postgres + +# For SQLite (workaround) +SUPABASE_DB_URL=sqlite:///./test.db +``` + +--- + +## Summary + +| Approach | Pros | Cons | +|----------|------|------| +| **SQLite** | Fast, no Docker, instant tables | Not production-like, some SQL differences | +| **Supabase** | Production-like, full stack | Needs Docker, migration bug to fix | + +**Recommended for now:** Use SQLite for development until the base schema migration is added. + +--- + +## TODO + +- [ ] Create `20241101000000_base_schema.sql` with all table definitions +- [ ] Test `supabase start` with the new migration +- [ ] Free up disk space and install Colima/Docker +- [ ] Document the full Supabase setup process diff --git a/src/policyengine_api/agent_sandbox.py b/src/policyengine_api/agent_sandbox.py index 9d0436c..bcf1ab0 100644 --- a/src/policyengine_api/agent_sandbox.py +++ b/src/policyengine_api/agent_sandbox.py @@ -235,8 +235,7 @@ def openapi_to_claude_tools(spec: dict) -> list[dict]: prop = schema_to_json_schema(spec, param_schema) prop["description"] = ( - param.get("description", "") - + f" (in: {param_in})" + param.get("description", "") + f" (in: {param_in})" ) properties[param_name] = prop @@ -268,16 +267,18 @@ def openapi_to_claude_tools(spec: dict) -> list[dict]: if required: input_schema["required"] = list(set(required)) - tools.append({ - "name": tool_name, - "description": full_desc[:1024], # Claude has limits - "input_schema": input_schema, - "_meta": { - "path": path, - "method": method, - "parameters": operation.get("parameters", []), - }, - }) + tools.append( + { + "name": tool_name, + "description": full_desc[:1024], # Claude has limits + "input_schema": input_schema, + "_meta": { + "path": path, + "method": method, + "parameters": operation.get("parameters", []), + }, + } + ) return tools @@ -347,7 +348,9 @@ def execute_api_tool( url, params=query_params, json=body_data, headers=headers, timeout=60 ) elif method == "delete": - resp = requests.delete(url, params=query_params, headers=headers, timeout=60) + resp = requests.delete( + url, params=query_params, headers=headers, timeout=60 + ) else: return f"Unsupported method: {method}" @@ -415,9 +418,7 @@ def log(msg: str) -> None: tool_lookup = {t["name"]: t for t in tools} # Strip _meta from tools before sending to Claude (it doesn't need it) - claude_tools = [ - {k: v for k, v in t.items() if k != "_meta"} for t in tools - ] + claude_tools = [{k: v for k, v in t.items() if k != "_meta"} for t in tools] # Add the sleep tool claude_tools.append(SLEEP_TOOL) @@ -477,11 +478,13 @@ def log(msg: str) -> None: log(f"[TOOL_RESULT] {result[:300]}") - tool_results.append({ - "type": "tool_result", - "tool_use_id": block.id, - "content": result, - }) + tool_results.append( + { + "type": "tool_result", + "tool_use_id": block.id, + "content": result, + } + ) messages.append({"role": "assistant", "content": assistant_content}) diff --git a/src/policyengine_api/api/agent.py b/src/policyengine_api/api/agent.py index 7b7d108..6c26e80 100644 --- a/src/policyengine_api/api/agent.py +++ b/src/policyengine_api/api/agent.py @@ -24,6 +24,7 @@ def get_traceparent() -> str | None: TraceContextTextMapPropagator().inject(carrier) return carrier.get("traceparent") + router = APIRouter(prefix="/agent", tags=["agent"]) @@ -93,7 +94,9 @@ def _run_local_agent( from policyengine_api.agent_sandbox import _run_agent_impl try: - history_dicts = [{"role": m.role, "content": m.content} for m in (history or [])] + history_dicts = [ + {"role": m.role, "content": m.content} for m in (history or []) + ] result = _run_agent_impl(question, api_base_url, call_id, history_dicts) _calls[call_id]["status"] = result.get("status", "completed") _calls[call_id]["result"] = result @@ -136,9 +139,15 @@ async def run_agent(request: RunRequest) -> RunResponse: traceparent = get_traceparent() run_fn = modal.Function.from_name("policyengine-sandbox", "run_agent") - history_dicts = [{"role": m.role, "content": m.content} for m in request.history] + history_dicts = [ + {"role": m.role, "content": m.content} for m in request.history + ] call = run_fn.spawn( - request.question, api_base_url, call_id, history_dicts, traceparent=traceparent + request.question, + api_base_url, + call_id, + history_dicts, + traceparent=traceparent, ) _calls[call_id] = { @@ -166,7 +175,12 @@ async def run_agent(request: RunRequest) -> RunResponse: # Run in background using asyncio loop = asyncio.get_event_loop() loop.run_in_executor( - None, _run_local_agent, call_id, request.question, api_base_url, request.history + None, + _run_local_agent, + call_id, + request.question, + api_base_url, + request.history, ) return RunResponse(call_id=call_id, status="running") diff --git a/src/policyengine_api/api/household.py b/src/policyengine_api/api/household.py index 0e89b5e..eba986d 100644 --- a/src/policyengine_api/api/household.py +++ b/src/policyengine_api/api/household.py @@ -300,11 +300,13 @@ def _calculate_household_uk( from pathlib import Path import pandas as pd - from policyengine.core import Simulation from microdf import MicroDataFrame + from policyengine.core import Simulation from policyengine.tax_benefit_models.uk import uk_latest - from policyengine.tax_benefit_models.uk.datasets import PolicyEngineUKDataset - from policyengine.tax_benefit_models.uk.datasets import UKYearData + from policyengine.tax_benefit_models.uk.datasets import ( + PolicyEngineUKDataset, + UKYearData, + ) n_people = len(people) n_benunits = max(1, len(benunit)) @@ -466,7 +468,14 @@ def _run_local_household_us( try: result = _calculate_household_us( - people, marital_unit, family, spm_unit, tax_unit, household, year, policy_data + people, + marital_unit, + family, + spm_unit, + tax_unit, + household, + year, + policy_data, ) # Update job with result @@ -512,11 +521,13 @@ def _calculate_household_us( from pathlib import Path import pandas as pd - from policyengine.core import Simulation from microdf import MicroDataFrame + from policyengine.core import Simulation from policyengine.tax_benefit_models.us import us_latest - from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset - from policyengine.tax_benefit_models.us.datasets import USYearData + from policyengine.tax_benefit_models.us.datasets import ( + PolicyEngineUSDataset, + USYearData, + ) n_people = len(people) n_households = max(1, len(household)) @@ -672,7 +683,9 @@ def safe_convert(value): except (ValueError, TypeError): return str(value) - def extract_entity_outputs(entity_name: str, entity_data, n_rows: int) -> list[dict]: + def extract_entity_outputs( + entity_name: str, entity_data, n_rows: int + ) -> list[dict]: outputs = [] for i in range(n_rows): row_dict = {} diff --git a/src/policyengine_api/api/policies.py b/src/policyengine_api/api/policies.py index 1f4c744..ad5397d 100644 --- a/src/policyengine_api/api/policies.py +++ b/src/policyengine_api/api/policies.py @@ -123,7 +123,9 @@ def create_policy(policy: PolicyCreate, session: Session = Depends(get_session)) @router.get("/", response_model=List[PolicyRead]) def list_policies( - tax_benefit_model_id: UUID | None = Query(None, description="Filter by tax benefit model"), + tax_benefit_model_id: UUID | None = Query( + None, description="Filter by tax benefit model" + ), session: Session = Depends(get_session), ): """List all policies, optionally filtered by tax benefit model.""" diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py index 3ac2786..4025cc9 100644 --- a/src/policyengine_api/api/user_policies.py +++ b/src/policyengine_api/api/user_policies.py @@ -56,7 +56,9 @@ def create_user_policy( @router.get("/", response_model=list[UserPolicyRead]) def list_user_policies( user_id: UUID = Query(..., description="User ID to filter by"), - tax_benefit_model_id: UUID | None = Query(None, description="Filter by tax benefit model"), + tax_benefit_model_id: UUID | None = Query( + None, description="Filter by tax benefit model" + ), session: Session = Depends(get_session), ): """List all policy associations for a user. @@ -66,10 +68,8 @@ def list_user_policies( query = select(UserPolicy).where(UserPolicy.user_id == user_id) if tax_benefit_model_id: - query = ( - query - .join(Policy, UserPolicy.policy_id == Policy.id) - .where(Policy.tax_benefit_model_id == tax_benefit_model_id) + query = query.join(Policy, UserPolicy.policy_id == Policy.id).where( + Policy.tax_benefit_model_id == tax_benefit_model_id ) user_policies = session.exec(query).all() diff --git a/src/policyengine_api/api/variables.py b/src/policyengine_api/api/variables.py index d660b1b..3c24f3d 100644 --- a/src/policyengine_api/api/variables.py +++ b/src/policyengine_api/api/variables.py @@ -56,9 +56,9 @@ def list_variables( # Case-insensitive search using ILIKE # Note: Variables don't have a label field, only name and description search_pattern = f"%{search}%" - search_filter = Variable.name.ilike(search_pattern) | Variable.description.ilike( + search_filter = Variable.name.ilike( search_pattern - ) + ) | Variable.description.ilike(search_pattern) query = query.where(search_filter) variables = session.exec( diff --git a/src/policyengine_api/modal_app.py b/src/policyengine_api/modal_app.py index 332c349..84f8e89 100644 --- a/src/policyengine_api/modal_app.py +++ b/src/policyengine_api/modal_app.py @@ -242,13 +242,13 @@ def simulate_household_uk( engine = create_engine(database_url) try: - from policyengine.core import Simulation from microdf import MicroDataFrame + from policyengine.core import Simulation from policyengine.tax_benefit_models.uk import uk_latest from policyengine.tax_benefit_models.uk.datasets import ( PolicyEngineUKDataset, + UKYearData, ) - from policyengine.tax_benefit_models.uk.datasets import UKYearData n_people = len(people) n_benunits = max(1, len(benunit)) @@ -487,13 +487,13 @@ def simulate_household_us( engine = create_engine(database_url) try: - from policyengine.core import Simulation from microdf import MicroDataFrame + from policyengine.core import Simulation from policyengine.tax_benefit_models.us import us_latest from policyengine.tax_benefit_models.us.datasets import ( PolicyEngineUSDataset, + USYearData, ) - from policyengine.tax_benefit_models.us.datasets import USYearData n_people = len(people) n_households = max(1, len(household)) diff --git a/tests/test_agent.py b/tests/test_agent.py index 2c591f5..55bb2c2 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -9,6 +9,7 @@ import json from unittest.mock import AsyncMock, MagicMock, patch + from fastapi.testclient import TestClient from policyengine_api.main import app diff --git a/tests/test_agent_policy_questions.py b/tests/test_agent_policy_questions.py index 1550f89..289d73c 100644 --- a/tests/test_agent_policy_questions.py +++ b/tests/test_agent_policy_questions.py @@ -11,10 +11,10 @@ pytestmark = pytest.mark.integration -from policyengine_api.agent_sandbox import _run_agent_impl - import os +from policyengine_api.agent_sandbox import _run_agent_impl + # Use local API by default, override with POLICYENGINE_API_URL env var API_BASE = os.environ.get("POLICYENGINE_API_URL", "http://localhost:8000") @@ -218,4 +218,6 @@ def test_turn_efficiency(self, question, max_expected_turns): print(f"Result: {result['result'][:300]}") if result["turns"] > max_expected_turns: - print(f"WARNING: Took {result['turns']} turns, expected <= {max_expected_turns}") + print( + f"WARNING: Took {result['turns']} turns, expected <= {max_expected_turns}" + ) diff --git a/tests/test_integration.py b/tests/test_integration.py index e044cab..e055423 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -9,6 +9,7 @@ pytestmark = pytest.mark.integration from datetime import datetime, timezone + from rich.console import Console from sqlmodel import Session, create_engine, select From bd3ae473bf6868dc6cc8309b879f7519a421c5e7 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Thu, 12 Feb 2026 20:04:27 +0530 Subject: [PATCH 36/87] refactor: Switch to auto-generated initial migration from PR #80 --- .../versions/20260204_0001_initial_schema.py | 538 ------------------ .../20260205_0002_add_user_policies.py | 4 +- 2 files changed, 2 insertions(+), 540 deletions(-) delete mode 100644 alembic/versions/20260204_0001_initial_schema.py diff --git a/alembic/versions/20260204_0001_initial_schema.py b/alembic/versions/20260204_0001_initial_schema.py deleted file mode 100644 index f488734..0000000 --- a/alembic/versions/20260204_0001_initial_schema.py +++ /dev/null @@ -1,538 +0,0 @@ -"""Initial schema (main branch state) - -Revision ID: 0001_initial -Revises: -Create Date: 2026-02-04 - -This migration creates all base tables for the PolicyEngine API as they -exist on the main branch, BEFORE the household CRUD changes. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0001_initial" -down_revision: Union[str, Sequence[str], None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Create all tables as they exist on main branch.""" - # ======================================================================== - # TIER 1: Tables with no foreign key dependencies - # ======================================================================== - - # Tax benefit models (e.g., "uk", "us") - op.create_table( - "tax_benefit_models", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # Users - op.create_table( - "users", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("first_name", sa.String(), nullable=False), - sa.Column("last_name", sa.String(), nullable=False), - sa.Column("email", sa.String(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("email"), - ) - op.create_index("ix_users_email", "users", ["email"]) - - # Policies (reform definitions) - op.create_table( - "policies", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # Dynamics (behavioral response definitions) - op.create_table( - "dynamics", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - # ======================================================================== - # TIER 2: Tables depending on tier 1 - # ======================================================================== - - # Tax benefit model versions - op.create_table( - "tax_benefit_model_versions", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("model_id", sa.Uuid(), nullable=False), - sa.Column("version", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["model_id"], ["tax_benefit_models.id"]), - ) - - # Datasets (h5 files in storage) - op.create_table( - "datasets", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("filepath", sa.String(), nullable=False), - sa.Column("year", sa.Integer(), nullable=False), - sa.Column("is_output_dataset", sa.Boolean(), nullable=False, default=False), - sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), - ) - - # ======================================================================== - # TIER 3: Tables depending on tier 2 - # ======================================================================== - - # Parameters (tax-benefit system parameters) - op.create_table( - "parameters", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("label", sa.String(), nullable=True), - sa.Column("description", sa.String(), nullable=True), - sa.Column("data_type", sa.String(), nullable=True), - sa.Column("unit", sa.String(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - ) - - # Variables (tax-benefit system variables) - op.create_table( - "variables", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("data_type", sa.String(), nullable=True), - sa.Column("possible_values", sa.JSON(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - ) - - # Dataset versions - op.create_table( - "dataset_versions", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=False), - sa.Column("dataset_id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), - sa.ForeignKeyConstraint(["tax_benefit_model_id"], ["tax_benefit_models.id"]), - ) - - # ======================================================================== - # TIER 4: Tables depending on tier 3 - # ======================================================================== - - # Parameter values (policy/dynamic parameter modifications) - op.create_table( - "parameter_values", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("parameter_id", sa.Uuid(), nullable=False), - sa.Column("value_json", sa.JSON(), nullable=True), - sa.Column("start_date", sa.DateTime(timezone=True), nullable=False), - sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["parameter_id"], ["parameters.id"]), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - ) - - # Simulations (economy calculations) - NOTE: No household support yet - op.create_table( - "simulations", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("dataset_id", sa.Uuid(), nullable=False), # Required in main - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column("output_dataset_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"]), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], ["tax_benefit_model_versions.id"] - ), - sa.ForeignKeyConstraint(["output_dataset_id"], ["datasets.id"]), - ) - - # Household jobs (async household calculations) - legacy approach - op.create_table( - "household_jobs", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("tax_benefit_model_name", sa.String(), nullable=False), - sa.Column("request_data", sa.JSON(), nullable=False), - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.JSON(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["dynamic_id"], ["dynamics.id"]), - ) - - # ======================================================================== - # TIER 5: Tables depending on simulations - # ======================================================================== - - # Reports (analysis reports) - NOTE: No report_type yet - op.create_table( - "reports", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("label", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=True), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("markdown", sa.Text(), nullable=True), - sa.Column("parent_report_id", sa.Uuid(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=True), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["parent_report_id"], ["reports.id"]), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - ) - - # Aggregates (single-simulation aggregate outputs) - op.create_table( - "aggregates", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("variable", sa.String(), nullable=False), - sa.Column("aggregate_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("filter_config", sa.JSON(), nullable=False, default={}), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Change aggregates (baseline vs reform comparison) - op.create_table( - "change_aggregates", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("user_id", sa.Uuid(), nullable=True), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("variable", sa.String(), nullable=False), - sa.Column("aggregate_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("filter_config", sa.JSON(), nullable=False, default={}), - sa.Column("change_geq", sa.Float(), nullable=True), - sa.Column("change_leq", sa.Float(), nullable=True), - sa.Column("status", sa.String(), nullable=False, default="pending"), - sa.Column("error_message", sa.String(), nullable=True), - sa.Column("result", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Decile impacts - op.create_table( - "decile_impacts", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("income_variable", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=True), - sa.Column("decile", sa.Integer(), nullable=False), - sa.Column("quantiles", sa.Integer(), nullable=False, default=10), - sa.Column("baseline_mean", sa.Float(), nullable=True), - sa.Column("reform_mean", sa.Float(), nullable=True), - sa.Column("absolute_change", sa.Float(), nullable=True), - sa.Column("relative_change", sa.Float(), nullable=True), - sa.Column("count_better_off", sa.Float(), nullable=True), - sa.Column("count_worse_off", sa.Float(), nullable=True), - sa.Column("count_no_change", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Program statistics - op.create_table( - "program_statistics", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), - sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("program_name", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False), - sa.Column("is_tax", sa.Boolean(), nullable=False, default=False), - sa.Column("baseline_total", sa.Float(), nullable=True), - sa.Column("reform_total", sa.Float(), nullable=True), - sa.Column("change", sa.Float(), nullable=True), - sa.Column("baseline_count", sa.Float(), nullable=True), - sa.Column("reform_count", sa.Float(), nullable=True), - sa.Column("winners", sa.Float(), nullable=True), - sa.Column("losers", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["baseline_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["reform_simulation_id"], ["simulations.id"]), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"]), - ) - - # Poverty - op.create_table( - "poverty", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("poverty_type", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False, default="person"), - sa.Column("filter_variable", sa.String(), nullable=True), - sa.Column("headcount", sa.Float(), nullable=True), - sa.Column("total_population", sa.Float(), nullable=True), - sa.Column("rate", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["simulation_id"], ["simulations.id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), - ) - op.create_index("idx_poverty_simulation_id", "poverty", ["simulation_id"]) - op.create_index("idx_poverty_report_id", "poverty", ["report_id"]) - - # Inequality - op.create_table( - "inequality", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("simulation_id", sa.Uuid(), nullable=False), - sa.Column("report_id", sa.Uuid(), nullable=True), - sa.Column("income_variable", sa.String(), nullable=False), - sa.Column("entity", sa.String(), nullable=False, default="household"), - sa.Column("gini", sa.Float(), nullable=True), - sa.Column("top_10_share", sa.Float(), nullable=True), - sa.Column("top_1_share", sa.Float(), nullable=True), - sa.Column("bottom_50_share", sa.Float(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["simulation_id"], ["simulations.id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint(["report_id"], ["reports.id"], ondelete="CASCADE"), - ) - op.create_index("idx_inequality_simulation_id", "inequality", ["simulation_id"]) - op.create_index("idx_inequality_report_id", "inequality", ["report_id"]) - - -def downgrade() -> None: - """Drop all tables in reverse order.""" - # Tier 5 - op.drop_index("idx_inequality_report_id", "inequality") - op.drop_index("idx_inequality_simulation_id", "inequality") - op.drop_table("inequality") - op.drop_index("idx_poverty_report_id", "poverty") - op.drop_index("idx_poverty_simulation_id", "poverty") - op.drop_table("poverty") - op.drop_table("program_statistics") - op.drop_table("decile_impacts") - op.drop_table("change_aggregates") - op.drop_table("aggregates") - op.drop_table("reports") - - # Tier 4 - op.drop_table("household_jobs") - op.drop_table("simulations") - op.drop_table("parameter_values") - - # Tier 3 - op.drop_table("dataset_versions") - op.drop_table("variables") - op.drop_table("parameters") - - # Tier 2 - op.drop_table("datasets") - op.drop_table("tax_benefit_model_versions") - - # Tier 1 - op.drop_table("dynamics") - op.drop_table("policies") - op.drop_index("ix_users_email", "users") - op.drop_table("users") - op.drop_table("tax_benefit_models") diff --git a/alembic/versions/20260205_0002_add_user_policies.py b/alembic/versions/20260205_0002_add_user_policies.py index b19a6cf..3af5e7f 100644 --- a/alembic/versions/20260205_0002_add_user_policies.py +++ b/alembic/versions/20260205_0002_add_user_policies.py @@ -1,7 +1,7 @@ """add_user_policies Revision ID: 0002_user_policies -Revises: 0001_initial +Revises: 36f9d434e95b Create Date: 2026-02-05 This migration adds: @@ -18,7 +18,7 @@ # revision identifiers, used by Alembic. revision: str = "0002_user_policies" -down_revision: Union[str, Sequence[str], None] = "0001_initial" +down_revision: Union[str, Sequence[str], None] = "36f9d434e95b" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 8bc80f6e378be16e115117f134340d7d73871916 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Thu, 12 Feb 2026 20:56:55 +0530 Subject: [PATCH 37/87] feat: Remove user FK constraint for client-generated UUIDs --- .../20260205_0002_add_user_policies.py | 6 +- src/policyengine_api/api/user_policies.py | 12 +- src/policyengine_api/models/user_policy.py | 8 +- tests/test_user_policies.py | 106 ++++++------------ 4 files changed, 52 insertions(+), 80 deletions(-) diff --git a/alembic/versions/20260205_0002_add_user_policies.py b/alembic/versions/20260205_0002_add_user_policies.py index 3af5e7f..df3b707 100644 --- a/alembic/versions/20260205_0002_add_user_policies.py +++ b/alembic/versions/20260205_0002_add_user_policies.py @@ -7,6 +7,10 @@ This migration adds: 1. tax_benefit_model_id foreign key to policies table 2. user_policies table for user-policy associations + +Note: user_id in user_policies is NOT a foreign key to users table. +It's a client-generated UUID stored in localStorage, allowing anonymous +users to save policies without authentication. """ from typing import Sequence, Union @@ -44,6 +48,7 @@ def upgrade() -> None: ) # Create user_policies table + # Note: user_id is NOT a foreign key - it's a client-generated UUID from localStorage op.create_table( "user_policies", sa.Column("user_id", sa.Uuid(), nullable=False), @@ -53,7 +58,6 @@ def upgrade() -> None: sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("updated_at", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(["policy_id"], ["policies.id"]), - sa.ForeignKeyConstraint(["user_id"], ["users.id"]), sa.PrimaryKeyConstraint("id"), ) op.create_index( diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py index 4025cc9..b01ca15 100644 --- a/src/policyengine_api/api/user_policies.py +++ b/src/policyengine_api/api/user_policies.py @@ -3,6 +3,10 @@ Associates users with policies they've saved/created. This enables users to maintain a list of their policies across sessions without duplicating the underlying policy data. + +Note: user_id is a client-generated UUID (via crypto.randomUUID()) stored in +the browser's localStorage. It is NOT validated against a users table, allowing +anonymous users to save policies without authentication. """ from datetime import datetime, timezone @@ -13,7 +17,6 @@ from policyengine_api.models import ( Policy, - User, UserPolicy, UserPolicyCreate, UserPolicyRead, @@ -34,12 +37,9 @@ def create_user_policy( Associates a user with a policy, allowing them to save it to their list. Duplicates are allowed - users can save the same policy multiple times with different labels (matching FE localStorage behavior). - """ - # Validate user exists - user = session.get(User, user_policy.user_id) - if not user: - raise HTTPException(status_code=404, detail="User not found") + Note: user_id is not validated - it's a client-generated UUID from localStorage. + """ # Validate policy exists policy = session.get(Policy, user_policy.policy_id) if not policy: diff --git a/src/policyengine_api/models/user_policy.py b/src/policyengine_api/models/user_policy.py index 6bdf9ff..b85505f 100644 --- a/src/policyengine_api/models/user_policy.py +++ b/src/policyengine_api/models/user_policy.py @@ -6,13 +6,16 @@ if TYPE_CHECKING: from .policy import Policy - from .user import User class UserPolicyBase(SQLModel): """Base user-policy association fields.""" - user_id: UUID = Field(foreign_key="users.id", index=True) + # user_id is a client-generated UUID stored in localStorage, not a foreign key. + # This allows anonymous users to save policies without requiring authentication. + # The UUID is generated once per browser via crypto.randomUUID() and persisted + # in localStorage for stable identity across sessions. + user_id: UUID = Field(index=True) policy_id: UUID = Field(foreign_key="policies.id", index=True) label: str | None = None @@ -27,7 +30,6 @@ class UserPolicy(UserPolicyBase, table=True): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) # Relationships - user: "User" = Relationship() policy: "Policy" = Relationship() diff --git a/tests/test_user_policies.py b/tests/test_user_policies.py index f4d1910..94725f1 100644 --- a/tests/test_user_policies.py +++ b/tests/test_user_policies.py @@ -1,46 +1,45 @@ -"""Tests for user-policy association endpoints.""" +"""Tests for user-policy association endpoints. + +Note: user_id is a client-generated UUID (not validated against users table), +so tests use uuid4() directly rather than creating User records. +""" from uuid import uuid4 -from policyengine_api.models import Policy, User, UserPolicy +from policyengine_api.models import Policy, UserPolicy -def test_list_user_policies_empty(client, session): +def test_list_user_policies_empty(client): """List user policies returns empty list when user has no associations.""" - user = User(first_name="Test", last_name="User", email="test@example.com") - session.add(user) - session.commit() - - response = client.get(f"/user-policies?user_id={user.id}") + user_id = uuid4() + response = client.get(f"/user-policies?user_id={user_id}") assert response.status_code == 200 assert response.json() == [] def test_create_user_policy(client, session, tax_benefit_model): """Create a new user-policy association.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) response = client.post( "/user-policies", json={ - "user_id": str(user.id), + "user_id": str(user_id), "policy_id": str(policy.id), "label": "My test policy", }, ) assert response.status_code == 200 data = response.json() - assert data["user_id"] == str(user.id) + assert data["user_id"] == str(user_id) assert data["policy_id"] == str(policy.id) assert data["label"] == "My test policy" assert "id" in data @@ -50,22 +49,20 @@ def test_create_user_policy(client, session, tax_benefit_model): def test_create_user_policy_without_label(client, session, tax_benefit_model): """Create a user-policy association without a label.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) response = client.post( "/user-policies", json={ - "user_id": str(user.id), + "user_id": str(user_id), "policy_id": str(policy.id), }, ) @@ -74,34 +71,15 @@ def test_create_user_policy_without_label(client, session, tax_benefit_model): assert data["label"] is None -def test_create_user_policy_user_not_found(client): - """Create user-policy association with non-existent user returns 404.""" - fake_user_id = uuid4() - fake_policy_id = uuid4() - - response = client.post( - "/user-policies", - json={ - "user_id": str(fake_user_id), - "policy_id": str(fake_policy_id), - }, - ) - assert response.status_code == 404 - assert response.json()["detail"] == "User not found" - - -def test_create_user_policy_policy_not_found(client, session): +def test_create_user_policy_policy_not_found(client): """Create user-policy association with non-existent policy returns 404.""" - user = User(first_name="Test", last_name="User", email="test@example.com") - session.add(user) - session.commit() - session.refresh(user) - + user_id = uuid4() fake_policy_id = uuid4() + response = client.post( "/user-policies", json={ - "user_id": str(user.id), + "user_id": str(user_id), "policy_id": str(fake_policy_id), }, ) @@ -111,21 +89,19 @@ def test_create_user_policy_policy_not_found(client, session): def test_create_user_policy_duplicate_allowed(client, session, tax_benefit_model): """Creating duplicate user-policy association is allowed (matches FE localStorage behavior).""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) # Create first association user_policy = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy.id, ) session.add(user_policy) @@ -135,14 +111,14 @@ def test_create_user_policy_duplicate_allowed(client, session, tax_benefit_model response = client.post( "/user-policies", json={ - "user_id": str(user.id), + "user_id": str(user_id), "policy_id": str(policy.id), }, ) assert response.status_code == 200 data = response.json() assert data["id"] != str(user_policy.id) # New association created - assert data["user_id"] == str(user.id) + assert data["user_id"] == str(user_id) assert data["policy_id"] == str(policy.id) @@ -150,7 +126,7 @@ def test_list_user_policies_with_data( client, session, tax_benefit_model, uk_tax_benefit_model ): """List user policies returns all associations for a user.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy1 = Policy( name="Policy 1", description="First policy", @@ -161,21 +137,19 @@ def test_list_user_policies_with_data( description="Second policy", tax_benefit_model_id=uk_tax_benefit_model.id, ) - session.add(user) session.add(policy1) session.add(policy2) session.commit() - session.refresh(user) session.refresh(policy1) session.refresh(policy2) user_policy1 = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy1.id, label="US policy", ) user_policy2 = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy2.id, label="UK policy", ) @@ -183,7 +157,7 @@ def test_list_user_policies_with_data( session.add(user_policy2) session.commit() - response = client.get(f"/user-policies?user_id={user.id}") + response = client.get(f"/user-policies?user_id={user_id}") assert response.status_code == 200 data = response.json() assert len(data) == 2 @@ -193,7 +167,7 @@ def test_list_user_policies_filter_by_tax_benefit_model( client, session, tax_benefit_model, uk_tax_benefit_model ): """List user policies filtered by tax_benefit_model_id via Policy join.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy1 = Policy( name="Policy 1", description="First policy", @@ -204,20 +178,18 @@ def test_list_user_policies_filter_by_tax_benefit_model( description="Second policy", tax_benefit_model_id=uk_tax_benefit_model.id, ) - session.add(user) session.add(policy1) session.add(policy2) session.commit() - session.refresh(user) session.refresh(policy1) session.refresh(policy2) user_policy1 = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy1.id, ) user_policy2 = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy2.id, ) session.add(user_policy1) @@ -225,7 +197,7 @@ def test_list_user_policies_filter_by_tax_benefit_model( session.commit() response = client.get( - f"/user-policies?user_id={user.id}&tax_benefit_model_id={tax_benefit_model.id}" + f"/user-policies?user_id={user_id}&tax_benefit_model_id={tax_benefit_model.id}" ) assert response.status_code == 200 data = response.json() @@ -235,20 +207,18 @@ def test_list_user_policies_filter_by_tax_benefit_model( def test_get_user_policy(client, session, tax_benefit_model): """Get a specific user-policy association by ID.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) user_policy = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy.id, label="My policy", ) @@ -273,20 +243,18 @@ def test_get_user_policy_not_found(client): def test_update_user_policy(client, session, tax_benefit_model): """Update a user-policy association label.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) user_policy = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy.id, label="Old label", ) @@ -316,20 +284,18 @@ def test_update_user_policy_not_found(client): def test_delete_user_policy(client, session, tax_benefit_model): """Delete a user-policy association.""" - user = User(first_name="Test", last_name="User", email="test@example.com") + user_id = uuid4() policy = Policy( name="Test policy", description="A test policy", tax_benefit_model_id=tax_benefit_model.id, ) - session.add(user) session.add(policy) session.commit() - session.refresh(user) session.refresh(policy) user_policy = UserPolicy( - user_id=user.id, + user_id=user_id, policy_id=policy.id, ) session.add(user_policy) From 3ee5e68ca028fcc0ecd8a9cb36355f1074be73b4 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Mon, 16 Feb 2026 23:26:04 +0530 Subject: [PATCH 38/87] feat: Use country_id for user-policy filtering --- .../20260205_0002_add_user_policies.py | 8 ++++++ src/policyengine_api/api/user_policies.py | 27 ++++++++++++++----- src/policyengine_api/models/user_policy.py | 1 + 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/alembic/versions/20260205_0002_add_user_policies.py b/alembic/versions/20260205_0002_add_user_policies.py index df3b707..5f061de 100644 --- a/alembic/versions/20260205_0002_add_user_policies.py +++ b/alembic/versions/20260205_0002_add_user_policies.py @@ -53,6 +53,7 @@ def upgrade() -> None: "user_policies", sa.Column("user_id", sa.Uuid(), nullable=False), sa.Column("policy_id", sa.Uuid(), nullable=False), + sa.Column("country_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), sa.Column("label", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column("id", sa.Uuid(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False), @@ -69,11 +70,18 @@ def upgrade() -> None: op.create_index( op.f("ix_user_policies_user_id"), "user_policies", ["user_id"], unique=False ) + op.create_index( + op.f("ix_user_policies_country_id"), + "user_policies", + ["country_id"], + unique=False, + ) def downgrade() -> None: """Remove user_policies table and policy.tax_benefit_model_id.""" # Drop user_policies table + op.drop_index(op.f("ix_user_policies_country_id"), table_name="user_policies") op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") op.drop_table("user_policies") diff --git a/src/policyengine_api/api/user_policies.py b/src/policyengine_api/api/user_policies.py index b01ca15..e1b6958 100644 --- a/src/policyengine_api/api/user_policies.py +++ b/src/policyengine_api/api/user_policies.py @@ -24,6 +24,9 @@ ) from policyengine_api.services.database import get_session +# Valid country IDs +VALID_COUNTRY_IDS = {"us", "uk"} + router = APIRouter(prefix="/user-policies", tags=["user-policies"]) @@ -40,6 +43,13 @@ def create_user_policy( Note: user_id is not validated - it's a client-generated UUID from localStorage. """ + # Validate country_id + if user_policy.country_id not in VALID_COUNTRY_IDS: + raise HTTPException( + status_code=400, + detail=f"Invalid country_id: {user_policy.country_id}. Must be one of: {list(VALID_COUNTRY_IDS)}", + ) + # Validate policy exists policy = session.get(Policy, user_policy.policy_id) if not policy: @@ -56,21 +66,24 @@ def create_user_policy( @router.get("/", response_model=list[UserPolicyRead]) def list_user_policies( user_id: UUID = Query(..., description="User ID to filter by"), - tax_benefit_model_id: UUID | None = Query( - None, description="Filter by tax benefit model" + country_id: str | None = Query( + None, description="Filter by country (e.g., 'us', 'uk')" ), session: Session = Depends(get_session), ): """List all policy associations for a user. - Returns all policies saved by the specified user. Optionally filter by tax benefit model. + Returns all policies saved by the specified user. Optionally filter by country. """ query = select(UserPolicy).where(UserPolicy.user_id == user_id) - if tax_benefit_model_id: - query = query.join(Policy, UserPolicy.policy_id == Policy.id).where( - Policy.tax_benefit_model_id == tax_benefit_model_id - ) + if country_id: + if country_id not in VALID_COUNTRY_IDS: + raise HTTPException( + status_code=400, + detail=f"Invalid country_id: {country_id}. Must be one of: {list(VALID_COUNTRY_IDS)}", + ) + query = query.where(UserPolicy.country_id == country_id) user_policies = session.exec(query).all() return user_policies diff --git a/src/policyengine_api/models/user_policy.py b/src/policyengine_api/models/user_policy.py index b85505f..7b3917c 100644 --- a/src/policyengine_api/models/user_policy.py +++ b/src/policyengine_api/models/user_policy.py @@ -17,6 +17,7 @@ class UserPolicyBase(SQLModel): # in localStorage for stable identity across sessions. user_id: UUID = Field(index=True) policy_id: UUID = Field(foreign_key="policies.id", index=True) + country_id: str # e.g., "us", "uk" - denormalized for efficient filtering label: str | None = None From 8193c514c297d04ffd8478e4a51eb1d8c0243584 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Tue, 17 Feb 2026 02:27:46 +0530 Subject: [PATCH 39/87] chore: Remove docs, temp files, and fix user policy tests for country_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove .DS_Store, docs/, and supabase/.temp/ from git tracking - Add these patterns to .gitignore - Update test_user_policies.py to include country_id in all tests - Rename test_list_user_policies_filter_by_tax_benefit_model to test_list_user_policies_filter_by_country 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 9 + docs/.env.example | 7 - docs/.gitignore | 42 - docs/AGENT_TESTING.md | 192 --- docs/DESIGN.md | 331 ----- docs/README.md | 30 - docs/api-v2-alpha-architecture.md | 264 ---- docs/bun.lock | 1088 ----------------- docs/eslint.config.mjs | 18 - docs/next.config.ts | 12 - docs/package.json | 29 - docs/policy-ingredient-status.md | 271 ---- docs/postcss.config.mjs | 7 - docs/public/file.svg | 1 - docs/public/globe.svg | 1 - docs/public/next.svg | 1 - docs/public/vercel.svg | 1 - docs/public/window.svg | 1 - docs/src/app/agent/page.tsx | 45 - docs/src/app/architecture/page.tsx | 138 --- docs/src/app/design/page.tsx | 151 --- docs/src/app/endpoints/aggregates/page.tsx | 127 -- .../app/endpoints/change-aggregates/page.tsx | 139 --- docs/src/app/endpoints/datasets/page.tsx | 143 --- docs/src/app/endpoints/dynamics/page.tsx | 138 --- .../app/endpoints/economic-impact/page.tsx | 272 ----- .../app/endpoints/household-impact/page.tsx | 131 -- docs/src/app/endpoints/household/page.tsx | 98 -- docs/src/app/endpoints/parameters/page.tsx | 103 -- docs/src/app/endpoints/policies/page.tsx | 160 --- docs/src/app/endpoints/simulations/page.tsx | 158 --- .../app/endpoints/tax-benefit-models/page.tsx | 96 -- docs/src/app/endpoints/variables/page.tsx | 105 -- docs/src/app/favicon.ico | Bin 25931 -> 0 bytes docs/src/app/globals.css | 234 ---- docs/src/app/layout.tsx | 45 - docs/src/app/mcp/page.tsx | 93 -- docs/src/app/modal/page.tsx | 205 ---- docs/src/app/page.tsx | 103 -- docs/src/app/quickstart/page.tsx | 149 --- docs/src/app/reference/models/page.tsx | 172 --- docs/src/app/reference/status-codes/page.tsx | 173 --- docs/src/app/setup/page.tsx | 198 --- docs/src/components/api-context.tsx | 29 - docs/src/components/api-playground.tsx | 184 --- docs/src/components/header.tsx | 46 - docs/src/components/json-viewer.tsx | 87 -- docs/src/components/policy-chat.tsx | 631 ---------- docs/src/components/sidebar.tsx | 107 -- docs/supabase-migration-notes.md | 174 --- docs/tsconfig.json | 42 - supabase/.temp/cli-latest | 1 - supabase/.temp/gotrue-version | 1 - supabase/.temp/pooler-url | 1 - supabase/.temp/postgres-version | 1 - supabase/.temp/project-ref | 1 - supabase/.temp/rest-version | 1 - supabase/.temp/storage-version | 1 - tests/test_user_policies.py | 23 +- 60 files changed, 29 insertions(+), 6982 deletions(-) delete mode 100644 .DS_Store delete mode 100644 docs/.env.example delete mode 100644 docs/.gitignore delete mode 100644 docs/AGENT_TESTING.md delete mode 100644 docs/DESIGN.md delete mode 100644 docs/README.md delete mode 100644 docs/api-v2-alpha-architecture.md delete mode 100644 docs/bun.lock delete mode 100644 docs/eslint.config.mjs delete mode 100644 docs/next.config.ts delete mode 100644 docs/package.json delete mode 100644 docs/policy-ingredient-status.md delete mode 100644 docs/postcss.config.mjs delete mode 100644 docs/public/file.svg delete mode 100644 docs/public/globe.svg delete mode 100644 docs/public/next.svg delete mode 100644 docs/public/vercel.svg delete mode 100644 docs/public/window.svg delete mode 100644 docs/src/app/agent/page.tsx delete mode 100644 docs/src/app/architecture/page.tsx delete mode 100644 docs/src/app/design/page.tsx delete mode 100644 docs/src/app/endpoints/aggregates/page.tsx delete mode 100644 docs/src/app/endpoints/change-aggregates/page.tsx delete mode 100644 docs/src/app/endpoints/datasets/page.tsx delete mode 100644 docs/src/app/endpoints/dynamics/page.tsx delete mode 100644 docs/src/app/endpoints/economic-impact/page.tsx delete mode 100644 docs/src/app/endpoints/household-impact/page.tsx delete mode 100644 docs/src/app/endpoints/household/page.tsx delete mode 100644 docs/src/app/endpoints/parameters/page.tsx delete mode 100644 docs/src/app/endpoints/policies/page.tsx delete mode 100644 docs/src/app/endpoints/simulations/page.tsx delete mode 100644 docs/src/app/endpoints/tax-benefit-models/page.tsx delete mode 100644 docs/src/app/endpoints/variables/page.tsx delete mode 100644 docs/src/app/favicon.ico delete mode 100644 docs/src/app/globals.css delete mode 100644 docs/src/app/layout.tsx delete mode 100644 docs/src/app/mcp/page.tsx delete mode 100644 docs/src/app/modal/page.tsx delete mode 100644 docs/src/app/page.tsx delete mode 100644 docs/src/app/quickstart/page.tsx delete mode 100644 docs/src/app/reference/models/page.tsx delete mode 100644 docs/src/app/reference/status-codes/page.tsx delete mode 100644 docs/src/app/setup/page.tsx delete mode 100644 docs/src/components/api-context.tsx delete mode 100644 docs/src/components/api-playground.tsx delete mode 100644 docs/src/components/header.tsx delete mode 100644 docs/src/components/json-viewer.tsx delete mode 100644 docs/src/components/policy-chat.tsx delete mode 100644 docs/src/components/sidebar.tsx delete mode 100644 docs/supabase-migration-notes.md delete mode 100644 docs/tsconfig.json delete mode 100644 supabase/.temp/cli-latest delete mode 100644 supabase/.temp/gotrue-version delete mode 100644 supabase/.temp/pooler-url delete mode 100644 supabase/.temp/postgres-version delete mode 100644 supabase/.temp/project-ref delete mode 100644 supabase/.temp/rest-version delete mode 100644 supabase/.temp/storage-version diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4bf6461c4c4dcb4ac126e58927c5525107ec899b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKK~BRk5FA4hEr^0RaY4!lsKgJ_QVvLPq2dfQEvS$-C~Xnka_0|xf%kEOS=)ld z1&I?vXjj@D$KLg3C$(JzFrE2eAJ_s=rwZ2AXuc2`7hRAg-?K?Hc8)F%(ZU#mqjbSr zJCp%s;IAW)Yh97wt#I6kRib>hRCtUR>oC-M>}Ot z3y*wCkJ8Ehmcf|@L!s>I0)m5^WFxQ9~ zX9#M9@K^R;rgx4cU_uxxd)4VR=X{swQ~7lQe_^id-(Y6>ra8T@SZ9uL5Fs3uBQ|Wr zl=hqvBRsG*XWTHXz%o>hT=|F$Yb7(LSz%_5dorK8UrhClgqeyrUr^--49JS0)Yl!V zRR)v+W#EGW*&iaRVC=DQXtxd;dj%lY*lmSv`NJSFk;m9$;gBN~<3fop)c7lgapCNb zykG3GaOlEe{N=;=%*NkPjLpvZBkK+mJ5;L-C<9dnmfho)od1X4-~X#Y`lJjf1OJKv zQ|tD+9hT(J)`jBatc|D_R1xtj9NG{Xe;nI}9K}0SD;$fYL5w{X4%tJ|KLVBpHOj!R GGVl%&a?hs# diff --git a/.gitignore b/.gitignore index 2f8c3e6..4b856db 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,12 @@ docs/.env.local data/ *.h5 *.db + +# macOS +.DS_Store + +# Docs (Next.js site) +docs/ + +# Supabase temp files +supabase/.temp/ diff --git a/docs/.env.example b/docs/.env.example deleted file mode 100644 index bc8a7d1..0000000 --- a/docs/.env.example +++ /dev/null @@ -1,7 +0,0 @@ -# Docs site environment variables -# Copy to .env.local for local development - -# API URL for the docs site to connect to -# Local development: http://localhost:8000 -# Production: https://v2.api.policyengine.org -NEXT_PUBLIC_API_URL=http://localhost:8000 diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 7b8da95..0000000 --- a/docs/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* -!.env.example - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts diff --git a/docs/AGENT_TESTING.md b/docs/AGENT_TESTING.md deleted file mode 100644 index c512ce1..0000000 --- a/docs/AGENT_TESTING.md +++ /dev/null @@ -1,192 +0,0 @@ -# Agent testing and optimisation - -This document tracks ongoing work to test and improve the PolicyEngine agent's ability to answer policy questions efficiently. - -## Goal - -Minimise the number of turns the agent needs to answer policy questions by improving API metadata, documentation, and structure - not by hacking for specific test cases. - -## Test categories - -We want comprehensive coverage across: -- **Country**: UK and US -- **Scope**: Household (single family) and Economy (population-wide) -- **Complexity**: Simple (single variable lookup) to Complex (multi-step reforms) - -## Example questions to test - -### UK Household (simple) -- "What is my income tax if I earn £50,000?" -- "How much child benefit would a family with 2 children receive?" - -### UK Household (complex) -- "Compare my net income under current law vs if the basic rate was 25%" -- "What's the marginal tax rate for someone earning £100,000?" - -### UK Economy (simple) -- "What's the total cost of child benefit?" -- "How many people pay higher rate tax?" - -### UK Economy (complex) -- "What would be the budgetary impact of raising the personal allowance to £15,000?" -- "How would a £500 UBI affect poverty rates?" - -### US Household (simple) -- "What is my federal income tax if I earn $75,000?" -- "How much SNAP would a family of 4 with $30,000 income receive?" - -### US Household (complex) -- "Compare my benefits under current law vs doubling the EITC" -- "What's my marginal tax rate including state taxes in California?" - -### US Economy (simple) -- "What's the total cost of SNAP?" -- "How many households receive the EITC?" - -### US Economy (complex) -- "What would be the budgetary impact of expanding the Child Tax Credit to $3,600?" -- "How would eliminating the SALT cap affect different income deciles?" - -## Current agent architecture - -The agent uses Claude Code in a Modal sandbox with: -- System prompt containing API documentation (see `src/policyengine_api/prompts/`) -- Direct HTTP calls via curl to the PolicyEngine API -- No MCP (it was causing issues in Modal containers) - -## Optimisation strategies - -1. **Improve system prompt** - Make API usage clearer, provide more examples -2. **Add API response examples** - Show what successful responses look like -3. **Parameter documentation** - Ensure all parameters are well-documented with valid values -4. **Error messages** - Make error messages actionable so agent can self-correct -5. **Endpoint discoverability** - Help agent find the right endpoint quickly - -## Test file location - -Tests are in `tests/test_agent_policy_questions.py` (integration tests requiring Modal). - -## How to continue this work - -1. Run existing tests: `pytest tests/test_agent_policy_questions.py -v -s` -2. Check agent logs in Logfire for turn counts and errors -3. Identify common failure patterns -4. Improve prompts/metadata to address failures -5. Add new test cases as coverage expands - -## Observed issues - -### Issue 1: Parameter search doesn't filter by country (9 turns for personal allowance) - -**Problem**: When searching for "personal allowance", the agent gets US results (Illinois AABD) mixed with UK results. It took 9 turns to find the UK personal allowance. - -**Agent's failed searches**: -1. "personal allowance" → Illinois AABD (US) -2. "income tax personal allowance" → empty -3. "income_tax" → US CBO parameters -4. "basic rate" → UK CGT (closer!) -5. "allowance" → California SSI (US) -6. "hmrc income_tax allowances personal" → empty -7. "hmrc.income_tax.allowances" → found it! - -**Solution implemented**: -- Added `tax_benefit_model_name` filter to `/parameters/` endpoint -- Updated system prompt to instruct agent to use country filter - -**NOT acceptable solutions** (test hacking): -- Adding specific parameter name examples to system prompt -- Telling agent exactly what to search for - -### Issue 2: Duplicate parameters in database - -**Problem**: Same parameter name exists with multiple IDs. One has values, one doesn't. Agent picks wrong one first. - -**Example**: `gov.hmrc.income_tax.allowances.personal_allowance.amount` has two entries with different IDs. - -**Solution implemented**: Deduplicate parameters by name in seed script (`seen_names` set). - -### Issue 6: Case-sensitive search - -**Problem**: Search for "personal allowance" didn't find "Personal allowance" (capital P). - -**Solution implemented**: Changed search to use `ILIKE` instead of `contains` for case-insensitive matching. - -### Issue 7: Model name mismatch - -**Problem**: System prompt said `policyengine_uk` but database has `policyengine-uk` (hyphen vs underscore). - -**Solution implemented**: Updated system prompt and API docstrings to use correct model names with hyphens. - -### Issue 3: Variables endpoint lacks search - -**Problem**: `/variables/` had no search or country filter. Agent can't discover variable names. - -**Solution implemented**: Added `search` and `tax_benefit_model_name` filters to `/variables/`. - -### Issue 4: Datasets endpoint lacks country filter - -**Problem**: `/datasets/` returned all datasets, mixing UK and US. - -**Solution implemented**: Added `tax_benefit_model_name` filter to `/datasets/`. - -### Issue 5: Parameter values lack "current" filter - -**Problem**: Agent had to parse through all historical values to find current one. - -**Solution implemented**: Added `current=true` filter to `/parameter-values/` endpoint. - -## API improvements summary - -| Endpoint | Improvement | -|----------|-------------| -| `/parameters/` | Added `tax_benefit_model_name` filter, case-insensitive search | -| `/variables/` | Added `search` and `tax_benefit_model_name` filters, case-insensitive search | -| `/datasets/` | Added `tax_benefit_model_name` filter | -| `/parameter-values/` | Added `current` filter | -| Seed script | Deduplicate parameters by name | -| System prompt | Fixed model names (hyphen not underscore) | - -### Issue 8: Agent not using country filter in economy-wide analysis - -**Problem**: When answering economic impact questions, agent didn't use `tax_benefit_model_name` filter despite it being in the system prompt. This led to 18 turns for UK budgetary impact question (12 turns just searching for parameter). - -**Root cause**: System prompt mentioned the filter but didn't emphasize it enough; economic impact workflow didn't show the filter in example. - -**Solution implemented**: Restructured system prompt with: -- **CRITICAL** section at the top emphasizing country filter -- Explanation of why filter is needed (mixed results waste turns) -- Added filter to all workflow examples including economic impact - -**Result**: UK budgetary impact question now completes in **6 turns** (down from 18). - -### Issue 9: Key US parameters missing from database - -**Problem**: Core CTC parameters like `gov.irs.credits.ctc.amount.base[0].amount` have `label=None` in policyengine-us package, so they're not seeded (seed script only includes parameters with labels). - -**Impact**: Agent can't find the main CTC amount parameter to double it. Had to use `refundable.individual_max` as a proxy. - -**Solution needed**: Add labels to core parameters in policyengine-us package (upstream fix). - -## Measurements - -| Question type | Baseline | After improvements | Target | -|---------------|----------|-------------------|--------| -| Parameter lookup (UK personal allowance) | 10 turns | **3 turns** | 3-4 | -| Household calculation (UK £50k income) | 6 turns | - | 5-6 | -| Economy-wide (UK budgetary impact) | 18 turns | **6 turns** | 5-8 | -| Economy-wide (US CTC impact) | 20+ turns | - | 8-10 | - -## Progress log - -- 2024-12-30: Initial setup, created test framework and first batch of questions -- 2024-12-30: Tested personal allowance lookup - 9-10 turns (target: 3-4). Root cause: no country filter on parameter search -- 2024-12-30: Added `tax_benefit_model_name` filter to `/parameters/`, `/variables/`, `/datasets/` -- 2024-12-30: Tested household calc - 6 turns (acceptable). Async polling is the overhead -- 2024-12-30: Discovered duplicate parameters in DB causing extra turns -- 2024-12-30: Fixed model name mismatch (policyengine-uk with hyphen, not underscore) -- 2024-12-30: Added case-insensitive search using ILIKE -- 2024-12-30: Tested personal allowance lookup - **3 turns** (target met!) -- 2025-12-30: Tested UK economy-wide (budgetary impact) - 18 turns initially -- 2025-12-30: Restructured system prompt to emphasize country filter at top -- 2025-12-30: UK economy-wide now **6 turns** (3x improvement) -- 2025-12-30: Discovered US CTC parameters missing labels (upstream issue in policyengine-us) diff --git a/docs/DESIGN.md b/docs/DESIGN.md deleted file mode 100644 index fbe2bd8..0000000 --- a/docs/DESIGN.md +++ /dev/null @@ -1,331 +0,0 @@ -# API hierarchy design - -## Levels of analysis - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Level 2: Reports │ -│ AI-generated documents, orchestrating multiple jobs │ -│ /reports/* │ -├─────────────────────────────────────────────────────────────┤ -│ Level 1: Analyses │ -│ Operations on simulations - thin wrappers around │ -│ policyengine package functions │ -│ │ -│ Common (baked-in, trivial to call): │ -│ /analysis/decile-impact/* │ -│ /analysis/budget-impact/* │ -│ /analysis/winners-losers/* │ -│ │ -│ Flexible (configurable): │ -│ /analysis/compare/* │ -├─────────────────────────────────────────────────────────────┤ -│ Level 0: Simulations │ -│ Single world-state calculations │ -│ /simulate/household │ -│ /simulate/economy │ -└─────────────────────────────────────────────────────────────┘ -``` - -All operations are **async** (Modal compute). The API is a thin orchestration layer - all analysis logic lives in the `policyengine` package. - -## Mapping to policyengine package - -| API endpoint | policyengine function | -|--------------|----------------------| -| `/simulate/household` | `calculate_household_impact()` | -| `/simulate/economy` | `Simulation.run()` | -| `/analysis/decile-impact/*` | `calculate_decile_impacts()` | -| `/analysis/budget-impact/*` | `ProgrammeStatistics` | -| `/analysis/winners-losers/*` | `ChangeAggregate` with filters | -| `/analysis/compare/*` | `economic_impact_analysis()` or custom | - -## Level 0: Simulations - -### `/simulate/household` - -Single household calculation. Wraps `policyengine.tax_benefit_models.uk.analysis.calculate_household_impact()`. - -``` -POST /simulate/household -{ - "model": "policyengine_uk", - "household": { - "people": [{"age": 30, "employment_income": 50000}], - "benunit": {}, - "household": {} - }, - "year": 2026, - "policy_id": null -} - -→ Returns job_id, poll for results -``` - -### `/simulate/economy` - -Population simulation. Creates and runs a `policyengine.core.Simulation`. - -``` -POST /simulate/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "policy_id": null, - "dynamic_id": null -} - -→ Returns simulation_id, poll for results -``` - -Economy simulations are **deterministic and cached** by (dataset_id, model_version, policy_id, dynamic_id). - -## Level 1: Analyses - Common (baked-in) - -These are the bread-and-butter analyses. Trivial to call, no configuration needed. - -### `/analysis/decile-impact/economy` - -Income decile breakdown. Wraps `calculate_decile_impacts()`. - -``` -POST /analysis/decile-impact/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "baseline_policy_id": null, - "reform_policy_id": "..." -} - -→ Returns job_id - -GET /analysis/decile-impact/economy/{job_id} -→ Returns: -{ - "status": "completed", - "deciles": [ - {"decile": 1, "baseline_mean": 15000, "reform_mean": 15500, "change": 500, "pct_change": 3.3, ...}, - {"decile": 2, ...}, - ... - {"decile": 10, ...} - ] -} -``` - -### `/analysis/budget-impact/economy` - -Tax and benefit programme totals. Wraps `ProgrammeStatistics`. - -``` -POST /analysis/budget-impact/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "baseline_policy_id": null, - "reform_policy_id": "..." -} - -→ Returns job_id - -GET /analysis/budget-impact/economy/{job_id} -→ Returns: -{ - "status": "completed", - "net_budget_impact": -20000000000, - "programmes": [ - {"name": "income_tax", "is_tax": true, "baseline_total": 200e9, "reform_total": 180e9, "change": -20e9}, - {"name": "universal_credit", "is_tax": false, "baseline_total": 50e9, "reform_total": 52e9, "change": 2e9}, - ... - ] -} -``` - -### `/analysis/winners-losers/economy` - -Who gains and loses. Wraps `ChangeAggregate` with change filters. - -``` -POST /analysis/winners-losers/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "baseline_policy_id": null, - "reform_policy_id": "...", - "threshold": 0 // Change threshold (default: any change) -} - -→ Returns job_id - -GET /analysis/winners-losers/economy/{job_id} -→ Returns: -{ - "status": "completed", - "winners": {"count": 15000000, "mean_gain": 500}, - "losers": {"count": 5000000, "mean_loss": -200}, - "unchanged": {"count": 30000000} -} -``` - -### `/analysis/decile-impact/household` - -Compare household across scenarios by artificial decile assignment. - -``` -POST /analysis/decile-impact/household -{ - "model": "policyengine_uk", - "household": {"people": [{"employment_income": 50000}]}, - "year": 2026, - "baseline_policy_id": null, - "reform_policy_id": "..." -} - -→ Returns which decile this household falls into and their change -``` - -## Level 1: Analyses - Flexible - -### `/analysis/compare/economy` - -Full comparison with all outputs. Wraps `economic_impact_analysis()`. - -``` -POST /analysis/compare/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "scenarios": [ - {"label": "baseline"}, - {"label": "reform", "policy_id": "..."}, - {"label": "reform_dynamic", "policy_id": "...", "dynamic_id": "..."} - ] -} - -→ Returns job_id - -GET /analysis/compare/economy/{job_id} -→ Returns: -{ - "status": "completed", - "scenarios": {...simulation results...}, - "comparisons": { - "reform": { - "relative_to": "baseline", - "decile_impacts": [...], - "budget_impact": {...}, - "winners_losers": {...} - }, - "reform_dynamic": {...} - } -} -``` - -### `/analysis/compare/household` - -Compare multiple scenarios for a household. - -``` -POST /analysis/compare/household -{ - "model": "policyengine_uk", - "household": {...}, - "year": 2026, - "scenarios": [ - {"label": "baseline"}, - {"label": "reform", "policy_id": "..."} - ] -} - -→ Returns all scenario results + computed differences -``` - -### `/analysis/aggregate/economy` (power user) - -Custom aggregation with full filter control. Directly exposes `Aggregate` / `ChangeAggregate`. - -``` -POST /analysis/aggregate/economy -{ - "model": "policyengine_uk", - "dataset_id": "...", - "simulation_id": "...", // or policy_id to create - "variable": "household_net_income", - "aggregate_type": "mean", - "entity": "household", - "filters": { - "quantile": {"variable": "household_net_income", "n": 10, "eq": 1} - } -} - -→ Returns single aggregate value -``` - -## Adding new analysis types - -To add a new common analysis (e.g. marginal tax rates): - -1. **policyengine package**: Add `MarginalTaxRate` output class and `calculate_marginal_rates()` function -2. **API**: Add `/analysis/marginal-rates/*` endpoint that wraps the function -3. **Modal**: Add function to run it - -The API endpoint is ~20 lines - just parameter parsing and calling the policyengine function. - -## URL structure summary - -``` -# Level 0: Simulations -POST /simulate/household -GET /simulate/household/{job_id} -POST /simulate/economy -GET /simulate/economy/{simulation_id} - -# Level 1: Common analyses (baked-in, trivial) -POST /analysis/decile-impact/economy -GET /analysis/decile-impact/economy/{job_id} -POST /analysis/budget-impact/economy -GET /analysis/budget-impact/economy/{job_id} -POST /analysis/winners-losers/economy -GET /analysis/winners-losers/economy/{job_id} - -# Level 1: Flexible analyses -POST /analysis/compare/economy -GET /analysis/compare/economy/{job_id} -POST /analysis/compare/household -GET /analysis/compare/household/{job_id} -POST /analysis/aggregate/economy -GET /analysis/aggregate/economy/{job_id} - -# Level 2: Reports (future) -POST /reports/policy-impact -GET /reports/policy-impact/{report_id} -``` - -## Use cases - -| Use case | Endpoint | -|----------|----------| -| My tax under current law | `/simulate/household` | -| Reform impact on my household | `/analysis/compare/household` with 2 scenarios | -| Revenue impact of reform | `/analysis/budget-impact/economy` | -| Decile breakdown of reform | `/analysis/decile-impact/economy` | -| Who wins and loses | `/analysis/winners-losers/economy` | -| Full reform analysis | `/analysis/compare/economy` | -| Compare 3 reform proposals | `/analysis/compare/economy` with 4 scenarios | -| Static vs dynamic comparison | `/analysis/compare/economy` with 3 scenarios | -| Custom aggregation | `/analysis/aggregate/economy` | - -## Migration - -Deprecate existing endpoints: -- `/household/calculate` → `/simulate/household` -- `/household/impact` → `/analysis/compare/household` -- `/analysis/economic-impact` → `/analysis/compare/economy` - -## Implementation notes - -1. All Modal functions import from `policyengine` package -2. API endpoints do minimal work: parse request, call Modal, store results -3. New analysis types require: - - Add to policyengine package (logic) - - Add API endpoint (orchestration) - - Add Modal function (compute) diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 9d487a6..0000000 --- a/docs/README.md +++ /dev/null @@ -1,30 +0,0 @@ -This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). - -## Getting Started - -First, run the development server: - -```bash -bun dev -``` - -Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. - -You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. - -This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. - -## Learn More - -To learn more about Next.js, take a look at the following resources: - -- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. -- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. - -You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! - -## Deploy on Vercel - -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. - -Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/docs/api-v2-alpha-architecture.md b/docs/api-v2-alpha-architecture.md deleted file mode 100644 index 10924db..0000000 --- a/docs/api-v2-alpha-architecture.md +++ /dev/null @@ -1,264 +0,0 @@ -# policyengine-api-v2-alpha Deep Dive - -## Overview - -A **FastAPI backend** for tax-benefit policy microsimulations using PolicyEngine's UK and US models. It's designed as a thin orchestration layer that offloads heavy calculations to **Modal.com** serverless functions. - ---- - -## Architecture - -### Three-Level Hierarchy -``` -Level 2: Reports AI-generated analysis documents -Level 1: Analyses Comparisons (economy_comparison_*) -Level 0: Simulations Single calculations (simulate_household_*, simulate_economy_*) -``` - -### Request Flow -1. Client → FastAPI (Cloud Run) -2. API creates job record in **Supabase** and triggers **Modal.com** function -3. Modal runs calculation with pre-loaded PolicyEngine models (sub-1s cold start via memory snapshotting) -4. Modal writes results directly to Supabase -5. Client polls API until `status = "completed"` - ---- - -## Tech Stack - -| Component | Technology | -|-----------|------------| -| Framework | FastAPI + async | -| Database | Supabase (PostgreSQL) via SQLModel | -| Compute | Modal.com serverless | -| Storage | Supabase Storage (S3-compatible for .h5 datasets) | -| Package mgr | UV | -| Formatting | Ruff | -| Testing | Pytest + pytest-asyncio | -| Deployment | Terraform → GCP Cloud Run | -| Observability | Logfire (OpenTelemetry) | - ---- - -## Database Models (SQLModel) - -### Core Entities - -| Model | Purpose | Key Fields | -|-------|---------|------------| -| `TaxBenefitModel` | Country identifier | `name` ("uk"/"us") | -| `TaxBenefitModelVersion` | Version of model | `version`, FK to model | -| `Dataset` | Population microdata (.h5) | `filepath` (S3), `year` | -| `DatasetVersion` | Dataset versions | FK to dataset + model | -| `User` | Basic user info | `email`, `first_name`, `last_name` | - -### Policy System (Normalized) - -| Model | Purpose | Key Fields | -|-------|---------|------------| -| `Policy` | Policy reform definition | `name`, `description`, `simulation_modifier` (Python code) | -| `Parameter` | Tax/benefit parameter metadata | `name` (path), `label`, `data_type`, FK to model version | -| `ParameterValue` | Parameter values (baseline OR reform) | `value_json`, `start_date`, `policy_id` (NULL=baseline) | -| `Variable` | Model variables metadata | `name`, `entity`, `data_type`, `possible_values` | -| `Dynamic` | Behavioral response config | Similar to Policy | - -### Simulation & Results - -| Model | Purpose | Key Fields | -|-------|---------|------------| -| `Simulation` | Economy simulation instance | FK to dataset, policy, dynamic; `status` | -| `HouseholdJob` | Single household calculation | `request_data` (JSON), `result` (JSON), `status` | -| `Report` | Analysis report | FK to baseline_sim, reform_sim; `status`, `markdown` | -| `DecileImpact` | Income impacts by decile | FK to baseline/reform sims; `absolute_change`, `relative_change` | -| `ProgramStatistics` | Budget impacts by program | `baseline_total`, `reform_total`, `change` | -| `Poverty` | Poverty rate outputs | `poverty_type`, `headcount`, `rate` | -| `Inequality` | Inequality metrics | `gini`, `top_10_share`, `bottom_50_share` | -| `AggregateOutput` | Custom aggregations | `variable`, `aggregate_type`, `filter_config` | -| `ChangeAggregate` | Change aggregations | baseline vs reform comparison | - ---- - -## API Endpoints - -### Registered Routers (`api/__init__.py`) -``` -/datasets - Dataset CRUD + listing -/policies - Policy CRUD (POST, GET, LIST - no PATCH/DELETE) -/simulations - Simulation management -/parameters - Parameter metadata -/parameter-values - Parameter value CRUD -/variables - Variable metadata -/dynamics - Behavioral response configs -/tax-benefit-models - Model listing -/tax-benefit-model-versions - Version listing -/outputs - Result outputs -/change-aggregates - Comparison aggregates -/household/* - Household calculations (async) -/analysis/* - Economy-wide analysis (async) -/agent/* - AI agent endpoint (Claude Code) -``` - -### Key Endpoint Patterns - -**Async Pattern** (household, analysis): -``` -POST /household/calculate → {"job_id": "...", "status": "pending"} -GET /household/calculate/{job_id} → poll until status="completed" -``` - -**CRUD Pattern** (policies, datasets): -``` -POST /policies - Create -GET /policies - List all -GET /policies/{id} - Get by ID -(PATCH, DELETE - NOT implemented) -``` - ---- - -## Modal.com Functions (`modal_app.py`) - -| Function | Purpose | Resources | -|----------|---------|-----------| -| `simulate_household_uk` | UK household calc | 4GB RAM, 4 CPU | -| `simulate_household_us` | US household calc | 4GB RAM, 4 CPU | -| `simulate_economy_uk` | UK economy sim | 8GB RAM, 8 CPU | -| `simulate_economy_us` | US economy sim | 8GB RAM, 8 CPU | -| `economy_comparison_uk` | UK decile/budget analysis | 8GB RAM, 8 CPU, 30min timeout | -| `economy_comparison_us` | US decile/budget analysis | 8GB RAM, 8 CPU, 30min timeout | - -### Key Feature: Memory Snapshotting -```python -# UK image - uses run_function to snapshot imported modules in memory -uk_image = base_image.run_commands( - "uv pip install --system policyengine-uk>=2.0.0" -).run_function(_import_uk) # ← pre-loads uk_latest at build time -``` -This enables **sub-1s cold starts** - PolicyEngine models are already loaded in memory. - ---- - -## Database Migrations (`supabase/migrations/`) - -| Migration | Purpose | -|-----------|---------| -| `20241119000000_storage_bucket.sql` | Create datasets S3 bucket | -| `20241121000000_storage_policies.sql` | Storage access policies | -| `20251229000000_add_parameter_values_indexes.sql` | Index optimization | -| `20260103000000_add_poverty_inequality.sql` | Poverty/inequality tables | -| `20260108000000_add_simulation_modifier.sql` | Add `simulation_modifier` to policies | - -**Note**: Tables are auto-created by SQLModel on app startup, not via migrations. Migrations are only for: -- Storage bucket setup -- Adding indexes -- Schema alterations - ---- - -## Agent Endpoint (`/agent/*`) - -An AI-powered policy analysis feature using Claude Code: -- `POST /agent/run` - Start agent with a question -- `GET /agent/logs/{call_id}` - Poll for logs and result -- `GET /agent/status/{call_id}` - Quick status check -- `POST /agent/log/{call_id}` - Modal calls this to stream logs -- `POST /agent/complete/{call_id}` - Modal calls this when done - -Runs in: -- **Production**: Modal sandbox -- **Development**: Local background thread - ---- - -## Configuration (`config/settings.py`) - -Key settings loaded from environment: -- `DATABASE_URL` - Supabase Postgres connection -- `SUPABASE_URL` / `SUPABASE_KEY` - Supabase client -- `POLICYENGINE_API_URL` - Self-reference URL -- `AGENT_USE_MODAL` - True for production, False for local - ---- - -## Key Design Decisions - -1. **Normalized parameter storage** - `parameter_values` table with FKs instead of JSON blobs -2. **Async job pattern** - All heavy calculations return immediately with job_id -3. **Deterministic UUIDs** - Simulations/reports use uuid5 for deduplication -4. **Memory snapshotting** - PolicyEngine models loaded at Modal image build time -5. **SQLModel** - Single source of truth for DB schema and Pydantic validation -6. **simulation_modifier** - Python code injection for custom variable formulas - ---- - -## What's Missing (Gaps) - -| Feature | Status | -|---------|--------| -| `PATCH /policies/{id}` | Not implemented | -| `DELETE /policies/{id}` | Not implemented | -| User-policy associations | Not implemented | -| User-simulation associations | Not implemented | -| Authentication | No endpoints exposed | - ---- - -## File Structure - -``` -src/policyengine_api/ -├── api/ # FastAPI routers -│ ├── __init__.py # Router registration -│ ├── agent.py # AI agent endpoint -│ ├── analysis.py # Economy analysis -│ ├── household.py # Household calculations -│ ├── policies.py # Policy CRUD -│ ├── parameters.py # Parameter metadata -│ ├── parameter_values.py # Parameter values -│ └── ... -├── config/ -│ └── settings.py # Environment config -├── models/ # SQLModel definitions -│ ├── __init__.py # Model exports -│ ├── policy.py -│ ├── parameter.py -│ ├── parameter_value.py -│ ├── simulation.py -│ ├── report.py -│ └── ... -├── services/ -│ ├── database.py # DB session management -│ └── storage.py # Supabase storage client -├── main.py # FastAPI app entry point -└── modal_app.py # Modal serverless functions - -supabase/ -└── migrations/ # SQL migrations (storage, indexes) - -terraform/ # GCP Cloud Run infrastructure - -tests/ # Pytest tests -``` - ---- - -## Development Commands - -```bash -make install # Install dependencies with uv -make dev # Start supabase + api via docker compose -make test # Run unit tests -make integration-test # Full integration tests -make format # Ruff formatting -make lint # Ruff linting with auto-fix -make modal-deploy # Deploy Modal.com serverless functions -make init # Reset tables and storage -make seed # Populate UK/US models with variables, parameters, datasets -``` - ---- - -## Contributors - -- **Nikhil Woodruff** - Initial implementation (Nov 2025) -- **Anthony Volk** - Parameter values filtering and indexing (Dec 2025) diff --git a/docs/bun.lock b/docs/bun.lock deleted file mode 100644 index 8b46639..0000000 --- a/docs/bun.lock +++ /dev/null @@ -1,1088 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "docs-site", - "dependencies": { - "next": "16.0.10", - "react": "19.2.1", - "react-dom": "19.2.1", - "react-markdown": "^10.1.0", - "remark-breaks": "^4.0.0", - "remark-gfm": "^4.0.1", - }, - "devDependencies": { - "@tailwindcss/postcss": "^4", - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "eslint": "^9", - "eslint-config-next": "16.0.10", - "tailwindcss": "^4", - "typescript": "^5", - }, - }, - }, - "packages": { - "@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="], - - "@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], - - "@babel/compat-data": ["@babel/compat-data@7.28.5", "", {}, "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA=="], - - "@babel/core": ["@babel/core@7.28.5", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-compilation-targets": "^7.27.2", "@babel/helper-module-transforms": "^7.28.3", "@babel/helpers": "^7.28.4", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/traverse": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw=="], - - "@babel/generator": ["@babel/generator@7.28.5", "", { "dependencies": { "@babel/parser": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ=="], - - "@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.27.2", "", { "dependencies": { "@babel/compat-data": "^7.27.2", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ=="], - - "@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="], - - "@babel/helper-module-imports": ["@babel/helper-module-imports@7.27.1", "", { "dependencies": { "@babel/traverse": "^7.27.1", "@babel/types": "^7.27.1" } }, "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w=="], - - "@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.3", "", { "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", "@babel/traverse": "^7.28.3" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw=="], - - "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], - - "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], - - "@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="], - - "@babel/helpers": ["@babel/helpers@7.28.4", "", { "dependencies": { "@babel/template": "^7.27.2", "@babel/types": "^7.28.4" } }, "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w=="], - - "@babel/parser": ["@babel/parser@7.28.5", "", { "dependencies": { "@babel/types": "^7.28.5" }, "bin": "./bin/babel-parser.js" }, "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ=="], - - "@babel/template": ["@babel/template@7.27.2", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/parser": "^7.27.2", "@babel/types": "^7.27.1" } }, "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw=="], - - "@babel/traverse": ["@babel/traverse@7.28.5", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/types": "^7.28.5", "debug": "^4.3.1" } }, "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ=="], - - "@babel/types": ["@babel/types@7.28.5", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA=="], - - "@emnapi/core": ["@emnapi/core@1.7.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg=="], - - "@emnapi/runtime": ["@emnapi/runtime@1.7.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA=="], - - "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.1.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ=="], - - "@eslint-community/eslint-utils": ["@eslint-community/eslint-utils@4.9.0", "", { "dependencies": { "eslint-visitor-keys": "^3.4.3" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g=="], - - "@eslint-community/regexpp": ["@eslint-community/regexpp@4.12.2", "", {}, "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew=="], - - "@eslint/config-array": ["@eslint/config-array@0.21.1", "", { "dependencies": { "@eslint/object-schema": "^2.1.7", "debug": "^4.3.1", "minimatch": "^3.1.2" } }, "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA=="], - - "@eslint/config-helpers": ["@eslint/config-helpers@0.4.2", "", { "dependencies": { "@eslint/core": "^0.17.0" } }, "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw=="], - - "@eslint/core": ["@eslint/core@0.17.0", "", { "dependencies": { "@types/json-schema": "^7.0.15" } }, "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ=="], - - "@eslint/eslintrc": ["@eslint/eslintrc@3.3.3", "", { "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", "espree": "^10.0.1", "globals": "^14.0.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.1", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" } }, "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ=="], - - "@eslint/js": ["@eslint/js@9.39.2", "", {}, "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA=="], - - "@eslint/object-schema": ["@eslint/object-schema@2.1.7", "", {}, "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA=="], - - "@eslint/plugin-kit": ["@eslint/plugin-kit@0.4.1", "", { "dependencies": { "@eslint/core": "^0.17.0", "levn": "^0.4.1" } }, "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA=="], - - "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], - - "@humanfs/node": ["@humanfs/node@0.16.7", "", { "dependencies": { "@humanfs/core": "^0.19.1", "@humanwhocodes/retry": "^0.4.0" } }, "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ=="], - - "@humanwhocodes/module-importer": ["@humanwhocodes/module-importer@1.0.1", "", {}, "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA=="], - - "@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="], - - "@img/colour": ["@img/colour@1.0.0", "", {}, "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw=="], - - "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], - - "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], - - "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], - - "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], - - "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], - - "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], - - "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], - - "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], - - "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], - - "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], - - "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], - - "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], - - "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], - - "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], - - "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], - - "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], - - "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], - - "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], - - "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], - - "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], - - "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], - - "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], - - "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], - - "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], - - "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], - - "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], - - "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], - - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], - - "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], - - "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@0.2.12", "", { "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ=="], - - "@next/env": ["@next/env@16.0.10", "", {}, "sha512-8tuaQkyDVgeONQ1MeT9Mkk8pQmZapMKFh5B+OrFUlG3rVmYTXcXlBetBgTurKXGaIZvkoqRT9JL5K3phXcgang=="], - - "@next/eslint-plugin-next": ["@next/eslint-plugin-next@16.0.10", "", { "dependencies": { "fast-glob": "3.3.1" } }, "sha512-b2NlWN70bbPLmfyoLvvidPKWENBYYIe017ZGUpElvQjDytCWgxPJx7L9juxHt0xHvNVA08ZHJdOyhGzon/KJuw=="], - - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.0.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4XgdKtdVsaflErz+B5XeG0T5PeXKDdruDf3CRpnhN+8UebNa5N2H58+3GDgpn/9GBurrQ1uWW768FfscwYkJRg=="], - - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.0.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-spbEObMvRKkQ3CkYVOME+ocPDFo5UqHb8EMTS78/0mQ+O1nqE8toHJVioZo4TvebATxgA8XMTHHrScPrn68OGw=="], - - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.0.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-uQtWE3X0iGB8apTIskOMi2w/MKONrPOUCi5yLO+v3O8Mb5c7K4Q5KD1jvTpTF5gJKa3VH/ijKjKUq9O9UhwOYw=="], - - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.0.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-llA+hiDTrYvyWI21Z0L1GiXwjQaanPVQQwru5peOgtooeJ8qx3tlqRV2P7uH2pKQaUfHxI/WVarvI5oYgGxaTw=="], - - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.0.10", "", { "os": "linux", "cpu": "x64" }, "sha512-AK2q5H0+a9nsXbeZ3FZdMtbtu9jxW4R/NgzZ6+lrTm3d6Zb7jYrWcgjcpM1k8uuqlSy4xIyPR2YiuUr+wXsavA=="], - - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.0.10", "", { "os": "linux", "cpu": "x64" }, "sha512-1TDG9PDKivNw5550S111gsO4RGennLVl9cipPhtkXIFVwo31YZ73nEbLjNC8qG3SgTz/QZyYyaFYMeY4BKZR/g=="], - - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.0.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-aEZIS4Hh32xdJQbHz121pyuVZniSNoqDVx1yIr2hy+ZwJGipeqnMZBJHyMxv2tiuAXGx6/xpTcQJ6btIiBjgmg=="], - - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.0.10", "", { "os": "win32", "cpu": "x64" }, "sha512-E+njfCoFLb01RAFEnGZn6ERoOqhK1Gl3Lfz1Kjnj0Ulfu7oJbuMyvBKNj/bw8XZnenHDASlygTjZICQW+rYW1Q=="], - - "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], - - "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], - - "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - - "@nolyfill/is-core-module": ["@nolyfill/is-core-module@1.0.39", "", {}, "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA=="], - - "@rtsao/scc": ["@rtsao/scc@1.1.0", "", {}, "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g=="], - - "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], - - "@tailwindcss/node": ["@tailwindcss/node@4.1.18", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "enhanced-resolve": "^5.18.3", "jiti": "^2.6.1", "lightningcss": "1.30.2", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.1.18" } }, "sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ=="], - - "@tailwindcss/oxide": ["@tailwindcss/oxide@4.1.18", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.1.18", "@tailwindcss/oxide-darwin-arm64": "4.1.18", "@tailwindcss/oxide-darwin-x64": "4.1.18", "@tailwindcss/oxide-freebsd-x64": "4.1.18", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.18", "@tailwindcss/oxide-linux-arm64-gnu": "4.1.18", "@tailwindcss/oxide-linux-arm64-musl": "4.1.18", "@tailwindcss/oxide-linux-x64-gnu": "4.1.18", "@tailwindcss/oxide-linux-x64-musl": "4.1.18", "@tailwindcss/oxide-wasm32-wasi": "4.1.18", "@tailwindcss/oxide-win32-arm64-msvc": "4.1.18", "@tailwindcss/oxide-win32-x64-msvc": "4.1.18" } }, "sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A=="], - - "@tailwindcss/oxide-android-arm64": ["@tailwindcss/oxide-android-arm64@4.1.18", "", { "os": "android", "cpu": "arm64" }, "sha512-dJHz7+Ugr9U/diKJA0W6N/6/cjI+ZTAoxPf9Iz9BFRF2GzEX8IvXxFIi/dZBloVJX/MZGvRuFA9rqwdiIEZQ0Q=="], - - "@tailwindcss/oxide-darwin-arm64": ["@tailwindcss/oxide-darwin-arm64@4.1.18", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Gc2q4Qhs660bhjyBSKgq6BYvwDz4G+BuyJ5H1xfhmDR3D8HnHCmT/BSkvSL0vQLy/nkMLY20PQ2OoYMO15Jd0A=="], - - "@tailwindcss/oxide-darwin-x64": ["@tailwindcss/oxide-darwin-x64@4.1.18", "", { "os": "darwin", "cpu": "x64" }, "sha512-FL5oxr2xQsFrc3X9o1fjHKBYBMD1QZNyc1Xzw/h5Qu4XnEBi3dZn96HcHm41c/euGV+GRiXFfh2hUCyKi/e+yw=="], - - "@tailwindcss/oxide-freebsd-x64": ["@tailwindcss/oxide-freebsd-x64@4.1.18", "", { "os": "freebsd", "cpu": "x64" }, "sha512-Fj+RHgu5bDodmV1dM9yAxlfJwkkWvLiRjbhuO2LEtwtlYlBgiAT4x/j5wQr1tC3SANAgD+0YcmWVrj8R9trVMA=="], - - "@tailwindcss/oxide-linux-arm-gnueabihf": ["@tailwindcss/oxide-linux-arm-gnueabihf@4.1.18", "", { "os": "linux", "cpu": "arm" }, "sha512-Fp+Wzk/Ws4dZn+LV2Nqx3IilnhH51YZoRaYHQsVq3RQvEl+71VGKFpkfHrLM/Li+kt5c0DJe/bHXK1eHgDmdiA=="], - - "@tailwindcss/oxide-linux-arm64-gnu": ["@tailwindcss/oxide-linux-arm64-gnu@4.1.18", "", { "os": "linux", "cpu": "arm64" }, "sha512-S0n3jboLysNbh55Vrt7pk9wgpyTTPD0fdQeh7wQfMqLPM/Hrxi+dVsLsPrycQjGKEQk85Kgbx+6+QnYNiHalnw=="], - - "@tailwindcss/oxide-linux-arm64-musl": ["@tailwindcss/oxide-linux-arm64-musl@4.1.18", "", { "os": "linux", "cpu": "arm64" }, "sha512-1px92582HkPQlaaCkdRcio71p8bc8i/ap5807tPRDK/uw953cauQBT8c5tVGkOwrHMfc2Yh6UuxaH4vtTjGvHg=="], - - "@tailwindcss/oxide-linux-x64-gnu": ["@tailwindcss/oxide-linux-x64-gnu@4.1.18", "", { "os": "linux", "cpu": "x64" }, "sha512-v3gyT0ivkfBLoZGF9LyHmts0Isc8jHZyVcbzio6Wpzifg/+5ZJpDiRiUhDLkcr7f/r38SWNe7ucxmGW3j3Kb/g=="], - - "@tailwindcss/oxide-linux-x64-musl": ["@tailwindcss/oxide-linux-x64-musl@4.1.18", "", { "os": "linux", "cpu": "x64" }, "sha512-bhJ2y2OQNlcRwwgOAGMY0xTFStt4/wyU6pvI6LSuZpRgKQwxTec0/3Scu91O8ir7qCR3AuepQKLU/kX99FouqQ=="], - - "@tailwindcss/oxide-wasm32-wasi": ["@tailwindcss/oxide-wasm32-wasi@4.1.18", "", { "dependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1", "@emnapi/wasi-threads": "^1.1.0", "@napi-rs/wasm-runtime": "^1.1.0", "@tybys/wasm-util": "^0.10.1", "tslib": "^2.4.0" }, "cpu": "none" }, "sha512-LffYTvPjODiP6PT16oNeUQJzNVyJl1cjIebq/rWWBF+3eDst5JGEFSc5cWxyRCJ0Mxl+KyIkqRxk1XPEs9x8TA=="], - - "@tailwindcss/oxide-win32-arm64-msvc": ["@tailwindcss/oxide-win32-arm64-msvc@4.1.18", "", { "os": "win32", "cpu": "arm64" }, "sha512-HjSA7mr9HmC8fu6bdsZvZ+dhjyGCLdotjVOgLA2vEqxEBZaQo9YTX4kwgEvPCpRh8o4uWc4J/wEoFzhEmjvPbA=="], - - "@tailwindcss/oxide-win32-x64-msvc": ["@tailwindcss/oxide-win32-x64-msvc@4.1.18", "", { "os": "win32", "cpu": "x64" }, "sha512-bJWbyYpUlqamC8dpR7pfjA0I7vdF6t5VpUGMWRkXVE3AXgIZjYUYAK7II1GNaxR8J1SSrSrppRar8G++JekE3Q=="], - - "@tailwindcss/postcss": ["@tailwindcss/postcss@4.1.18", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "@tailwindcss/node": "4.1.18", "@tailwindcss/oxide": "4.1.18", "postcss": "^8.4.41", "tailwindcss": "4.1.18" } }, "sha512-Ce0GFnzAOuPyfV5SxjXGn0CubwGcuDB0zcdaPuCSzAa/2vII24JTkH+I6jcbXLb1ctjZMZZI6OjDaLPJQL1S0g=="], - - "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], - - "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], - - "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], - - "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], - - "@types/hast": ["@types/hast@3.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ=="], - - "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], - - "@types/json5": ["@types/json5@0.0.29", "", {}, "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ=="], - - "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], - - "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], - - "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], - - "@types/react": ["@types/react@19.2.7", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg=="], - - "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], - - "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], - - "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.50.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.50.0", "@typescript-eslint/type-utils": "8.50.0", "@typescript-eslint/utils": "8.50.0", "@typescript-eslint/visitor-keys": "8.50.0", "ignore": "^7.0.0", "natural-compare": "^1.4.0", "ts-api-utils": "^2.1.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.50.0", "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-O7QnmOXYKVtPrfYzMolrCTfkezCJS9+ljLdKW/+DCvRsc3UAz+sbH6Xcsv7p30+0OwUbeWfUDAQE0vpabZ3QLg=="], - - "@typescript-eslint/parser": ["@typescript-eslint/parser@8.50.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.50.0", "@typescript-eslint/types": "8.50.0", "@typescript-eslint/typescript-estree": "8.50.0", "@typescript-eslint/visitor-keys": "8.50.0", "debug": "^4.3.4" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-6/cmF2piao+f6wSxUsJLZjck7OQsYyRtcOZS02k7XINSNlz93v6emM8WutDQSXnroG2xwYlEVHJI+cPA7CPM3Q=="], - - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.50.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.50.0", "@typescript-eslint/types": "^8.50.0", "debug": "^4.3.4" }, "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "sha512-Cg/nQcL1BcoTijEWyx4mkVC56r8dj44bFDvBdygifuS20f3OZCHmFbjF34DPSi07kwlFvqfv/xOLnJ5DquxSGQ=="], - - "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.50.0", "", { "dependencies": { "@typescript-eslint/types": "8.50.0", "@typescript-eslint/visitor-keys": "8.50.0" } }, "sha512-xCwfuCZjhIqy7+HKxBLrDVT5q/iq7XBVBXLn57RTIIpelLtEIZHXAF/Upa3+gaCpeV1NNS5Z9A+ID6jn50VD4A=="], - - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.50.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "sha512-vxd3G/ybKTSlm31MOA96gqvrRGv9RJ7LGtZCn2Vrc5htA0zCDvcMqUkifcjrWNNKXHUU3WCkYOzzVSFBd0wa2w=="], - - "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.50.0", "", { "dependencies": { "@typescript-eslint/types": "8.50.0", "@typescript-eslint/typescript-estree": "8.50.0", "@typescript-eslint/utils": "8.50.0", "debug": "^4.3.4", "ts-api-utils": "^2.1.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-7OciHT2lKCewR0mFoBrvZJ4AXTMe/sYOe87289WAViOocEmDjjv8MvIOT2XESuKj9jp8u3SZYUSh89QA4S1kQw=="], - - "@typescript-eslint/types": ["@typescript-eslint/types@8.50.0", "", {}, "sha512-iX1mgmGrXdANhhITbpp2QQM2fGehBse9LbTf0sidWK6yg/NE+uhV5dfU1g6EYPlcReYmkE9QLPq/2irKAmtS9w=="], - - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.50.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.50.0", "@typescript-eslint/tsconfig-utils": "8.50.0", "@typescript-eslint/types": "8.50.0", "@typescript-eslint/visitor-keys": "8.50.0", "debug": "^4.3.4", "minimatch": "^9.0.4", "semver": "^7.6.0", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.1.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "sha512-W7SVAGBR/IX7zm1t70Yujpbk+zdPq/u4soeFSknWFdXIFuWsBGBOUu/Tn/I6KHSKvSh91OiMuaSnYp3mtPt5IQ=="], - - "@typescript-eslint/utils": ["@typescript-eslint/utils@8.50.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.7.0", "@typescript-eslint/scope-manager": "8.50.0", "@typescript-eslint/types": "8.50.0", "@typescript-eslint/typescript-estree": "8.50.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-87KgUXET09CRjGCi2Ejxy3PULXna63/bMYv72tCAlDJC3Yqwln0HiFJ3VJMst2+mEtNtZu5oFvX4qJGjKsnAgg=="], - - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.50.0", "", { "dependencies": { "@typescript-eslint/types": "8.50.0", "eslint-visitor-keys": "^4.2.1" } }, "sha512-Xzmnb58+Db78gT/CCj/PVCvK+zxbnsw6F+O1oheYszJbBSdEjVhQi3C/Xttzxgi/GLmpvOggRs1RFpiJ8+c34Q=="], - - "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], - - "@unrs/resolver-binding-android-arm-eabi": ["@unrs/resolver-binding-android-arm-eabi@1.11.1", "", { "os": "android", "cpu": "arm" }, "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw=="], - - "@unrs/resolver-binding-android-arm64": ["@unrs/resolver-binding-android-arm64@1.11.1", "", { "os": "android", "cpu": "arm64" }, "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g=="], - - "@unrs/resolver-binding-darwin-arm64": ["@unrs/resolver-binding-darwin-arm64@1.11.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g=="], - - "@unrs/resolver-binding-darwin-x64": ["@unrs/resolver-binding-darwin-x64@1.11.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ=="], - - "@unrs/resolver-binding-freebsd-x64": ["@unrs/resolver-binding-freebsd-x64@1.11.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw=="], - - "@unrs/resolver-binding-linux-arm-gnueabihf": ["@unrs/resolver-binding-linux-arm-gnueabihf@1.11.1", "", { "os": "linux", "cpu": "arm" }, "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw=="], - - "@unrs/resolver-binding-linux-arm-musleabihf": ["@unrs/resolver-binding-linux-arm-musleabihf@1.11.1", "", { "os": "linux", "cpu": "arm" }, "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw=="], - - "@unrs/resolver-binding-linux-arm64-gnu": ["@unrs/resolver-binding-linux-arm64-gnu@1.11.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ=="], - - "@unrs/resolver-binding-linux-arm64-musl": ["@unrs/resolver-binding-linux-arm64-musl@1.11.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w=="], - - "@unrs/resolver-binding-linux-ppc64-gnu": ["@unrs/resolver-binding-linux-ppc64-gnu@1.11.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA=="], - - "@unrs/resolver-binding-linux-riscv64-gnu": ["@unrs/resolver-binding-linux-riscv64-gnu@1.11.1", "", { "os": "linux", "cpu": "none" }, "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ=="], - - "@unrs/resolver-binding-linux-riscv64-musl": ["@unrs/resolver-binding-linux-riscv64-musl@1.11.1", "", { "os": "linux", "cpu": "none" }, "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew=="], - - "@unrs/resolver-binding-linux-s390x-gnu": ["@unrs/resolver-binding-linux-s390x-gnu@1.11.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg=="], - - "@unrs/resolver-binding-linux-x64-gnu": ["@unrs/resolver-binding-linux-x64-gnu@1.11.1", "", { "os": "linux", "cpu": "x64" }, "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w=="], - - "@unrs/resolver-binding-linux-x64-musl": ["@unrs/resolver-binding-linux-x64-musl@1.11.1", "", { "os": "linux", "cpu": "x64" }, "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA=="], - - "@unrs/resolver-binding-wasm32-wasi": ["@unrs/resolver-binding-wasm32-wasi@1.11.1", "", { "dependencies": { "@napi-rs/wasm-runtime": "^0.2.11" }, "cpu": "none" }, "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ=="], - - "@unrs/resolver-binding-win32-arm64-msvc": ["@unrs/resolver-binding-win32-arm64-msvc@1.11.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw=="], - - "@unrs/resolver-binding-win32-ia32-msvc": ["@unrs/resolver-binding-win32-ia32-msvc@1.11.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ=="], - - "@unrs/resolver-binding-win32-x64-msvc": ["@unrs/resolver-binding-win32-x64-msvc@1.11.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g=="], - - "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], - - "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], - - "ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], - - "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], - - "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - - "aria-query": ["aria-query@5.3.2", "", {}, "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw=="], - - "array-buffer-byte-length": ["array-buffer-byte-length@1.0.2", "", { "dependencies": { "call-bound": "^1.0.3", "is-array-buffer": "^3.0.5" } }, "sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw=="], - - "array-includes": ["array-includes@3.1.9", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "define-properties": "^1.2.1", "es-abstract": "^1.24.0", "es-object-atoms": "^1.1.1", "get-intrinsic": "^1.3.0", "is-string": "^1.1.1", "math-intrinsics": "^1.1.0" } }, "sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ=="], - - "array.prototype.findlast": ["array.prototype.findlast@1.2.5", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.2", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0", "es-shim-unscopables": "^1.0.2" } }, "sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ=="], - - "array.prototype.findlastindex": ["array.prototype.findlastindex@1.2.6", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "define-properties": "^1.2.1", "es-abstract": "^1.23.9", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "es-shim-unscopables": "^1.1.0" } }, "sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ=="], - - "array.prototype.flat": ["array.prototype.flat@1.3.3", "", { "dependencies": { "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-shim-unscopables": "^1.0.2" } }, "sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg=="], - - "array.prototype.flatmap": ["array.prototype.flatmap@1.3.3", "", { "dependencies": { "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-shim-unscopables": "^1.0.2" } }, "sha512-Y7Wt51eKJSyi80hFrJCePGGNo5ktJCslFuboqJsbf57CCPcm5zztluPlc4/aD8sWsKvlwatezpV4U1efk8kpjg=="], - - "array.prototype.tosorted": ["array.prototype.tosorted@1.1.4", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.3", "es-errors": "^1.3.0", "es-shim-unscopables": "^1.0.2" } }, "sha512-p6Fx8B7b7ZhL/gmUsAy0D15WhvDccw3mnGNbZpi3pmeJdxtWsj2jEaI4Y6oo3XiHfzuSgPwKc04MYt6KgvC/wA=="], - - "arraybuffer.prototype.slice": ["arraybuffer.prototype.slice@1.0.4", "", { "dependencies": { "array-buffer-byte-length": "^1.0.1", "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "is-array-buffer": "^3.0.4" } }, "sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ=="], - - "ast-types-flow": ["ast-types-flow@0.0.8", "", {}, "sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ=="], - - "async-function": ["async-function@1.0.0", "", {}, "sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA=="], - - "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], - - "axe-core": ["axe-core@4.11.0", "", {}, "sha512-ilYanEU8vxxBexpJd8cWM4ElSQq4QctCLKih0TSfjIfCQTeyH/6zVrmIJfLPrKTKJRbiG+cfnZbQIjAlJmF1jQ=="], - - "axobject-query": ["axobject-query@4.1.0", "", {}, "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ=="], - - "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], - - "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], - - "baseline-browser-mapping": ["baseline-browser-mapping@2.9.8", "", { "bin": { "baseline-browser-mapping": "dist/cli.js" } }, "sha512-Y1fOuNDowLfgKOypdc9SPABfoWXuZHBOyCS4cD52IeZBhr4Md6CLLs6atcxVrzRmQ06E7hSlm5bHHApPKR/byA=="], - - "brace-expansion": ["brace-expansion@1.1.12", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg=="], - - "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - - "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], - - "call-bind": ["call-bind@1.0.8", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", "get-intrinsic": "^1.2.4", "set-function-length": "^1.2.2" } }, "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww=="], - - "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], - - "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], - - "callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="], - - "caniuse-lite": ["caniuse-lite@1.0.30001760", "", {}, "sha512-7AAMPcueWELt1p3mi13HR/LHH0TJLT11cnwDJEs3xA4+CK/PLKeO9Kl1oru24htkyUKtkGCvAx4ohB0Ttry8Dw=="], - - "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], - - "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], - - "character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="], - - "character-entities-html4": ["character-entities-html4@2.1.0", "", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="], - - "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], - - "character-reference-invalid": ["character-reference-invalid@2.0.1", "", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], - - "client-only": ["client-only@0.0.1", "", {}, "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="], - - "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], - - "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], - - "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], - - "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], - - "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], - - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], - - "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], - - "damerau-levenshtein": ["damerau-levenshtein@1.0.8", "", {}, "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA=="], - - "data-view-buffer": ["data-view-buffer@1.0.2", "", { "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", "is-data-view": "^1.0.2" } }, "sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ=="], - - "data-view-byte-length": ["data-view-byte-length@1.0.2", "", { "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", "is-data-view": "^1.0.2" } }, "sha512-tuhGbE6CfTM9+5ANGf+oQb72Ky/0+s3xKUpHvShfiz2RxMFgFPjsXuRLBVMtvMs15awe45SRb83D6wH4ew6wlQ=="], - - "data-view-byte-offset": ["data-view-byte-offset@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "is-data-view": "^1.0.1" } }, "sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ=="], - - "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - - "decode-named-character-reference": ["decode-named-character-reference@1.2.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q=="], - - "deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="], - - "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], - - "define-properties": ["define-properties@1.2.1", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="], - - "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], - - "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - - "devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="], - - "doctrine": ["doctrine@2.1.0", "", { "dependencies": { "esutils": "^2.0.2" } }, "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw=="], - - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - - "electron-to-chromium": ["electron-to-chromium@1.5.267", "", {}, "sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw=="], - - "emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], - - "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], - - "es-abstract": ["es-abstract@1.24.1", "", { "dependencies": { "array-buffer-byte-length": "^1.0.2", "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "data-view-buffer": "^1.0.2", "data-view-byte-length": "^1.0.2", "data-view-byte-offset": "^1.0.1", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "es-set-tostringtag": "^2.1.0", "es-to-primitive": "^1.3.0", "function.prototype.name": "^1.1.8", "get-intrinsic": "^1.3.0", "get-proto": "^1.0.1", "get-symbol-description": "^1.1.0", "globalthis": "^1.0.4", "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", "has-proto": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "internal-slot": "^1.1.0", "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", "is-regex": "^1.2.1", "is-set": "^2.0.3", "is-shared-array-buffer": "^1.0.4", "is-string": "^1.1.1", "is-typed-array": "^1.1.15", "is-weakref": "^1.1.1", "math-intrinsics": "^1.1.0", "object-inspect": "^1.13.4", "object-keys": "^1.1.1", "object.assign": "^4.1.7", "own-keys": "^1.0.1", "regexp.prototype.flags": "^1.5.4", "safe-array-concat": "^1.1.3", "safe-push-apply": "^1.0.0", "safe-regex-test": "^1.1.0", "set-proto": "^1.0.0", "stop-iteration-iterator": "^1.1.0", "string.prototype.trim": "^1.2.10", "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", "typed-array-buffer": "^1.0.3", "typed-array-byte-length": "^1.0.3", "typed-array-byte-offset": "^1.0.4", "typed-array-length": "^1.0.7", "unbox-primitive": "^1.1.0", "which-typed-array": "^1.1.19" } }, "sha512-zHXBLhP+QehSSbsS9Pt23Gg964240DPd6QCf8WpkqEXxQ7fhdZzYsocOr5u7apWonsS5EjZDmTF+/slGMyasvw=="], - - "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], - - "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], - - "es-iterator-helpers": ["es-iterator-helpers@1.2.2", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "define-properties": "^1.2.1", "es-abstract": "^1.24.1", "es-errors": "^1.3.0", "es-set-tostringtag": "^2.1.0", "function-bind": "^1.1.2", "get-intrinsic": "^1.3.0", "globalthis": "^1.0.4", "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", "has-proto": "^1.2.0", "has-symbols": "^1.1.0", "internal-slot": "^1.1.0", "iterator.prototype": "^1.1.5", "safe-array-concat": "^1.1.3" } }, "sha512-BrUQ0cPTB/IwXj23HtwHjS9n7O4h9FX94b4xc5zlTHxeLgTAdzYUDyy6KdExAl9lbN5rtfe44xpjpmj9grxs5w=="], - - "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], - - "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], - - "es-shim-unscopables": ["es-shim-unscopables@1.1.0", "", { "dependencies": { "hasown": "^2.0.2" } }, "sha512-d9T8ucsEhh8Bi1woXCf+TIKDIROLG5WCkxg8geBCbvk22kzwC5G2OnXVMO6FUsvQlgUUXQ2itephWDLqDzbeCw=="], - - "es-to-primitive": ["es-to-primitive@1.3.0", "", { "dependencies": { "is-callable": "^1.2.7", "is-date-object": "^1.0.5", "is-symbol": "^1.0.4" } }, "sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g=="], - - "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], - - "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], - - "eslint": ["eslint@9.39.2", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.21.1", "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "9.39.2", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^8.4.0", "eslint-visitor-keys": "^4.2.1", "espree": "^10.4.0", "esquery": "^1.5.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw=="], - - "eslint-config-next": ["eslint-config-next@16.0.10", "", { "dependencies": { "@next/eslint-plugin-next": "16.0.10", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.32.0", "eslint-plugin-jsx-a11y": "^6.10.0", "eslint-plugin-react": "^7.37.0", "eslint-plugin-react-hooks": "^7.0.0", "globals": "16.4.0", "typescript-eslint": "^8.46.0" }, "peerDependencies": { "eslint": ">=9.0.0", "typescript": ">=3.3.1" }, "optionalPeers": ["typescript"] }, "sha512-BxouZUm0I45K4yjOOIzj24nTi0H2cGo0y7xUmk+Po/PYtJXFBYVDS1BguE7t28efXjKdcN0tmiLivxQy//SsZg=="], - - "eslint-import-resolver-node": ["eslint-import-resolver-node@0.3.9", "", { "dependencies": { "debug": "^3.2.7", "is-core-module": "^2.13.0", "resolve": "^1.22.4" } }, "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g=="], - - "eslint-import-resolver-typescript": ["eslint-import-resolver-typescript@3.10.1", "", { "dependencies": { "@nolyfill/is-core-module": "1.0.39", "debug": "^4.4.0", "get-tsconfig": "^4.10.0", "is-bun-module": "^2.0.0", "stable-hash": "^0.0.5", "tinyglobby": "^0.2.13", "unrs-resolver": "^1.6.2" }, "peerDependencies": { "eslint": "*", "eslint-plugin-import": "*", "eslint-plugin-import-x": "*" }, "optionalPeers": ["eslint-plugin-import", "eslint-plugin-import-x"] }, "sha512-A1rHYb06zjMGAxdLSkN2fXPBwuSaQ0iO5M/hdyS0Ajj1VBaRp0sPD3dn1FhME3c/JluGFbwSxyCfqdSbtQLAHQ=="], - - "eslint-module-utils": ["eslint-module-utils@2.12.1", "", { "dependencies": { "debug": "^3.2.7" } }, "sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw=="], - - "eslint-plugin-import": ["eslint-plugin-import@2.32.0", "", { "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", "array.prototype.findlastindex": "^1.2.6", "array.prototype.flat": "^1.3.3", "array.prototype.flatmap": "^1.3.3", "debug": "^3.2.7", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.9", "eslint-module-utils": "^2.12.1", "hasown": "^2.0.2", "is-core-module": "^2.16.1", "is-glob": "^4.0.3", "minimatch": "^3.1.2", "object.fromentries": "^2.0.8", "object.groupby": "^1.0.3", "object.values": "^1.2.1", "semver": "^6.3.1", "string.prototype.trimend": "^1.0.9", "tsconfig-paths": "^3.15.0" }, "peerDependencies": { "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9" } }, "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA=="], - - "eslint-plugin-jsx-a11y": ["eslint-plugin-jsx-a11y@6.10.2", "", { "dependencies": { "aria-query": "^5.3.2", "array-includes": "^3.1.8", "array.prototype.flatmap": "^1.3.2", "ast-types-flow": "^0.0.8", "axe-core": "^4.10.0", "axobject-query": "^4.1.0", "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", "hasown": "^2.0.2", "jsx-ast-utils": "^3.3.5", "language-tags": "^1.0.9", "minimatch": "^3.1.2", "object.fromentries": "^2.0.8", "safe-regex-test": "^1.0.3", "string.prototype.includes": "^2.0.1" }, "peerDependencies": { "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9" } }, "sha512-scB3nz4WmG75pV8+3eRUQOHZlNSUhFNq37xnpgRkCCELU3XMvXAxLk1eqWWyE22Ki4Q01Fnsw9BA3cJHDPgn2Q=="], - - "eslint-plugin-react": ["eslint-plugin-react@7.37.5", "", { "dependencies": { "array-includes": "^3.1.8", "array.prototype.findlast": "^1.2.5", "array.prototype.flatmap": "^1.3.3", "array.prototype.tosorted": "^1.1.4", "doctrine": "^2.1.0", "es-iterator-helpers": "^1.2.1", "estraverse": "^5.3.0", "hasown": "^2.0.2", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", "object.entries": "^1.1.9", "object.fromentries": "^2.0.8", "object.values": "^1.2.1", "prop-types": "^15.8.1", "resolve": "^2.0.0-next.5", "semver": "^6.3.1", "string.prototype.matchall": "^4.0.12", "string.prototype.repeat": "^1.0.0" }, "peerDependencies": { "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9.7" } }, "sha512-Qteup0SqU15kdocexFNAJMvCJEfa2xUKNV4CC1xsVMrIIqEy3SQ/rqyxCWNzfrd3/ldy6HMlD2e0JDVpDg2qIA=="], - - "eslint-plugin-react-hooks": ["eslint-plugin-react-hooks@7.0.1", "", { "dependencies": { "@babel/core": "^7.24.4", "@babel/parser": "^7.24.4", "hermes-parser": "^0.25.1", "zod": "^3.25.0 || ^4.0.0", "zod-validation-error": "^3.5.0 || ^4.0.0" }, "peerDependencies": { "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" } }, "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA=="], - - "eslint-scope": ["eslint-scope@8.4.0", "", { "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" } }, "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg=="], - - "eslint-visitor-keys": ["eslint-visitor-keys@4.2.1", "", {}, "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ=="], - - "espree": ["espree@10.4.0", "", { "dependencies": { "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^4.2.1" } }, "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ=="], - - "esquery": ["esquery@1.6.0", "", { "dependencies": { "estraverse": "^5.1.0" } }, "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg=="], - - "esrecurse": ["esrecurse@4.3.0", "", { "dependencies": { "estraverse": "^5.2.0" } }, "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag=="], - - "estraverse": ["estraverse@5.3.0", "", {}, "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="], - - "estree-util-is-identifier-name": ["estree-util-is-identifier-name@3.0.0", "", {}, "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg=="], - - "esutils": ["esutils@2.0.3", "", {}, "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="], - - "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], - - "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - - "fast-glob": ["fast-glob@3.3.1", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.4" } }, "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg=="], - - "fast-json-stable-stringify": ["fast-json-stable-stringify@2.1.0", "", {}, "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="], - - "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], - - "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], - - "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], - - "file-entry-cache": ["file-entry-cache@8.0.0", "", { "dependencies": { "flat-cache": "^4.0.0" } }, "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ=="], - - "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], - - "find-up": ["find-up@5.0.0", "", { "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" } }, "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng=="], - - "flat-cache": ["flat-cache@4.0.1", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="], - - "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], - - "for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="], - - "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], - - "function.prototype.name": ["function.prototype.name@1.1.8", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "define-properties": "^1.2.1", "functions-have-names": "^1.2.3", "hasown": "^2.0.2", "is-callable": "^1.2.7" } }, "sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q=="], - - "functions-have-names": ["functions-have-names@1.2.3", "", {}, "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ=="], - - "generator-function": ["generator-function@2.0.1", "", {}, "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g=="], - - "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], - - "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], - - "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], - - "get-symbol-description": ["get-symbol-description@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6" } }, "sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg=="], - - "get-tsconfig": ["get-tsconfig@4.13.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ=="], - - "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], - - "globals": ["globals@16.4.0", "", {}, "sha512-ob/2LcVVaVGCYN+r14cnwnoDPUufjiYgSqRhiFD0Q1iI4Odora5RE8Iv1D24hAz5oMophRGkGz+yuvQmmUMnMw=="], - - "globalthis": ["globalthis@1.0.4", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="], - - "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], - - "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - - "has-bigints": ["has-bigints@1.1.0", "", {}, "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg=="], - - "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], - - "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], - - "has-proto": ["has-proto@1.2.0", "", { "dependencies": { "dunder-proto": "^1.0.0" } }, "sha512-KIL7eQPfHQRC8+XluaIw7BHUwwqL19bQn4hzNgdr+1wXoU0KKj6rufu47lhY7KbJR2C6T6+PfyN0Ea7wkSS+qQ=="], - - "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], - - "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], - - "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], - - "hast-util-to-jsx-runtime": ["hast-util-to-jsx-runtime@2.3.6", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "vfile-message": "^4.0.0" } }, "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg=="], - - "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], - - "hermes-estree": ["hermes-estree@0.25.1", "", {}, "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw=="], - - "hermes-parser": ["hermes-parser@0.25.1", "", { "dependencies": { "hermes-estree": "0.25.1" } }, "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA=="], - - "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], - - "ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - - "import-fresh": ["import-fresh@3.3.1", "", { "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" } }, "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ=="], - - "imurmurhash": ["imurmurhash@0.1.4", "", {}, "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="], - - "inline-style-parser": ["inline-style-parser@0.2.7", "", {}, "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA=="], - - "internal-slot": ["internal-slot@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "hasown": "^2.0.2", "side-channel": "^1.1.0" } }, "sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw=="], - - "is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], - - "is-alphanumerical": ["is-alphanumerical@2.0.1", "", { "dependencies": { "is-alphabetical": "^2.0.0", "is-decimal": "^2.0.0" } }, "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw=="], - - "is-array-buffer": ["is-array-buffer@3.0.5", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A=="], - - "is-async-function": ["is-async-function@2.1.1", "", { "dependencies": { "async-function": "^1.0.0", "call-bound": "^1.0.3", "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" } }, "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ=="], - - "is-bigint": ["is-bigint@1.1.0", "", { "dependencies": { "has-bigints": "^1.0.2" } }, "sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ=="], - - "is-boolean-object": ["is-boolean-object@1.2.2", "", { "dependencies": { "call-bound": "^1.0.3", "has-tostringtag": "^1.0.2" } }, "sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A=="], - - "is-bun-module": ["is-bun-module@2.0.0", "", { "dependencies": { "semver": "^7.7.1" } }, "sha512-gNCGbnnnnFAUGKeZ9PdbyeGYJqewpmc2aKHUEMO5nQPWU9lOmv7jcmQIv+qHD8fXW6W7qfuCwX4rY9LNRjXrkQ=="], - - "is-callable": ["is-callable@1.2.7", "", {}, "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA=="], - - "is-core-module": ["is-core-module@2.16.1", "", { "dependencies": { "hasown": "^2.0.2" } }, "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w=="], - - "is-data-view": ["is-data-view@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "get-intrinsic": "^1.2.6", "is-typed-array": "^1.1.13" } }, "sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw=="], - - "is-date-object": ["is-date-object@1.1.0", "", { "dependencies": { "call-bound": "^1.0.2", "has-tostringtag": "^1.0.2" } }, "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg=="], - - "is-decimal": ["is-decimal@2.0.1", "", {}, "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A=="], - - "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], - - "is-finalizationregistry": ["is-finalizationregistry@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg=="], - - "is-generator-function": ["is-generator-function@1.1.2", "", { "dependencies": { "call-bound": "^1.0.4", "generator-function": "^2.0.0", "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" } }, "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA=="], - - "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], - - "is-hexadecimal": ["is-hexadecimal@2.0.1", "", {}, "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg=="], - - "is-map": ["is-map@2.0.3", "", {}, "sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw=="], - - "is-negative-zero": ["is-negative-zero@2.0.3", "", {}, "sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw=="], - - "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - - "is-number-object": ["is-number-object@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3", "has-tostringtag": "^1.0.2" } }, "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw=="], - - "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], - - "is-regex": ["is-regex@1.2.1", "", { "dependencies": { "call-bound": "^1.0.2", "gopd": "^1.2.0", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g=="], - - "is-set": ["is-set@2.0.3", "", {}, "sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg=="], - - "is-shared-array-buffer": ["is-shared-array-buffer@1.0.4", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A=="], - - "is-string": ["is-string@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3", "has-tostringtag": "^1.0.2" } }, "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA=="], - - "is-symbol": ["is-symbol@1.1.1", "", { "dependencies": { "call-bound": "^1.0.2", "has-symbols": "^1.1.0", "safe-regex-test": "^1.1.0" } }, "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w=="], - - "is-typed-array": ["is-typed-array@1.1.15", "", { "dependencies": { "which-typed-array": "^1.1.16" } }, "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ=="], - - "is-weakmap": ["is-weakmap@2.0.2", "", {}, "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w=="], - - "is-weakref": ["is-weakref@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew=="], - - "is-weakset": ["is-weakset@2.0.4", "", { "dependencies": { "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ=="], - - "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], - - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - - "iterator.prototype": ["iterator.prototype@1.1.5", "", { "dependencies": { "define-data-property": "^1.1.4", "es-object-atoms": "^1.0.0", "get-intrinsic": "^1.2.6", "get-proto": "^1.0.0", "has-symbols": "^1.1.0", "set-function-name": "^2.0.2" } }, "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g=="], - - "jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], - - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - - "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - - "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], - - "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], - - "json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], - - "json-stable-stringify-without-jsonify": ["json-stable-stringify-without-jsonify@1.0.1", "", {}, "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="], - - "json5": ["json5@1.0.2", "", { "dependencies": { "minimist": "^1.2.0" }, "bin": { "json5": "lib/cli.js" } }, "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="], - - "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], - - "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], - - "language-subtag-registry": ["language-subtag-registry@0.3.23", "", {}, "sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ=="], - - "language-tags": ["language-tags@1.0.9", "", { "dependencies": { "language-subtag-registry": "^0.3.20" } }, "sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA=="], - - "levn": ["levn@0.4.1", "", { "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" } }, "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ=="], - - "lightningcss": ["lightningcss@1.30.2", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.30.2", "lightningcss-darwin-arm64": "1.30.2", "lightningcss-darwin-x64": "1.30.2", "lightningcss-freebsd-x64": "1.30.2", "lightningcss-linux-arm-gnueabihf": "1.30.2", "lightningcss-linux-arm64-gnu": "1.30.2", "lightningcss-linux-arm64-musl": "1.30.2", "lightningcss-linux-x64-gnu": "1.30.2", "lightningcss-linux-x64-musl": "1.30.2", "lightningcss-win32-arm64-msvc": "1.30.2", "lightningcss-win32-x64-msvc": "1.30.2" } }, "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ=="], - - "lightningcss-android-arm64": ["lightningcss-android-arm64@1.30.2", "", { "os": "android", "cpu": "arm64" }, "sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A=="], - - "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.30.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ylTcDJBN3Hp21TdhRT5zBOIi73P6/W0qwvlFEk22fkdXchtNTOU4Qc37SkzV+EKYxLouZ6M4LG9NfZ1qkhhBWA=="], - - "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.30.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-oBZgKchomuDYxr7ilwLcyms6BCyLn0z8J0+ZZmfpjwg9fRVZIR5/GMXd7r9RH94iDhld3UmSjBM6nXWM2TfZTQ=="], - - "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.30.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-c2bH6xTrf4BDpK8MoGG4Bd6zAMZDAXS569UxCAGcA7IKbHNMlhGQ89eRmvpIUGfKWNVdbhSbkQaWhEoMGmGslA=="], - - "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.30.2", "", { "os": "linux", "cpu": "arm" }, "sha512-eVdpxh4wYcm0PofJIZVuYuLiqBIakQ9uFZmipf6LF/HRj5Bgm0eb3qL/mr1smyXIS1twwOxNWndd8z0E374hiA=="], - - "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.30.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-UK65WJAbwIJbiBFXpxrbTNArtfuznvxAJw4Q2ZGlU8kPeDIWEX1dg3rn2veBVUylA2Ezg89ktszWbaQnxD/e3A=="], - - "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.30.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA=="], - - "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.30.2", "", { "os": "linux", "cpu": "x64" }, "sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w=="], - - "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.30.2", "", { "os": "linux", "cpu": "x64" }, "sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA=="], - - "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.30.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ=="], - - "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.30.2", "", { "os": "win32", "cpu": "x64" }, "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw=="], - - "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], - - "lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="], - - "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], - - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - - "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], - - "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - - "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], - - "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - - "mdast-util-find-and-replace": ["mdast-util-find-and-replace@3.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "escape-string-regexp": "^5.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg=="], - - "mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA=="], - - "mdast-util-gfm": ["mdast-util-gfm@3.1.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ=="], - - "mdast-util-gfm-autolink-literal": ["mdast-util-gfm-autolink-literal@2.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "ccount": "^2.0.0", "devlop": "^1.0.0", "mdast-util-find-and-replace": "^3.0.0", "micromark-util-character": "^2.0.0" } }, "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ=="], - - "mdast-util-gfm-footnote": ["mdast-util-gfm-footnote@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0" } }, "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ=="], - - "mdast-util-gfm-strikethrough": ["mdast-util-gfm-strikethrough@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg=="], - - "mdast-util-gfm-table": ["mdast-util-gfm-table@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "markdown-table": "^3.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg=="], - - "mdast-util-gfm-task-list-item": ["mdast-util-gfm-task-list-item@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ=="], - - "mdast-util-mdx-expression": ["mdast-util-mdx-expression@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ=="], - - "mdast-util-mdx-jsx": ["mdast-util-mdx-jsx@3.2.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-stringify-position": "^4.0.0", "vfile-message": "^4.0.0" } }, "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q=="], - - "mdast-util-mdxjs-esm": ["mdast-util-mdxjs-esm@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg=="], - - "mdast-util-newline-to-break": ["mdast-util-newline-to-break@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-find-and-replace": "^3.0.0" } }, "sha512-MbgeFca0hLYIEx/2zGsszCSEJJ1JSCdiY5xQxRcLDDGa8EPvlLPupJ4DSajbMPAnC0je8jfb9TiUATnxxrHUog=="], - - "mdast-util-phrasing": ["mdast-util-phrasing@4.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "unist-util-is": "^6.0.0" } }, "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w=="], - - "mdast-util-to-hast": ["mdast-util-to-hast@13.2.1", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@ungap/structured-clone": "^1.0.0", "devlop": "^1.0.0", "micromark-util-sanitize-uri": "^2.0.0", "trim-lines": "^3.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA=="], - - "mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="], - - "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], - - "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], - - "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], - - "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], - - "micromark-extension-gfm": ["micromark-extension-gfm@3.0.0", "", { "dependencies": { "micromark-extension-gfm-autolink-literal": "^2.0.0", "micromark-extension-gfm-footnote": "^2.0.0", "micromark-extension-gfm-strikethrough": "^2.0.0", "micromark-extension-gfm-table": "^2.0.0", "micromark-extension-gfm-tagfilter": "^2.0.0", "micromark-extension-gfm-task-list-item": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w=="], - - "micromark-extension-gfm-autolink-literal": ["micromark-extension-gfm-autolink-literal@2.1.0", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw=="], - - "micromark-extension-gfm-footnote": ["micromark-extension-gfm-footnote@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw=="], - - "micromark-extension-gfm-strikethrough": ["micromark-extension-gfm-strikethrough@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw=="], - - "micromark-extension-gfm-table": ["micromark-extension-gfm-table@2.1.1", "", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg=="], - - "micromark-extension-gfm-tagfilter": ["micromark-extension-gfm-tagfilter@2.0.0", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg=="], - - "micromark-extension-gfm-task-list-item": ["micromark-extension-gfm-task-list-item@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw=="], - - "micromark-factory-destination": ["micromark-factory-destination@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA=="], - - "micromark-factory-label": ["micromark-factory-label@2.0.1", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg=="], - - "micromark-factory-space": ["micromark-factory-space@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg=="], - - "micromark-factory-title": ["micromark-factory-title@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw=="], - - "micromark-factory-whitespace": ["micromark-factory-whitespace@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ=="], - - "micromark-util-character": ["micromark-util-character@2.1.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q=="], - - "micromark-util-chunked": ["micromark-util-chunked@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA=="], - - "micromark-util-classify-character": ["micromark-util-classify-character@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q=="], - - "micromark-util-combine-extensions": ["micromark-util-combine-extensions@2.0.1", "", { "dependencies": { "micromark-util-chunked": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg=="], - - "micromark-util-decode-numeric-character-reference": ["micromark-util-decode-numeric-character-reference@2.0.2", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw=="], - - "micromark-util-decode-string": ["micromark-util-decode-string@2.0.1", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ=="], - - "micromark-util-encode": ["micromark-util-encode@2.0.1", "", {}, "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw=="], - - "micromark-util-html-tag-name": ["micromark-util-html-tag-name@2.0.1", "", {}, "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA=="], - - "micromark-util-normalize-identifier": ["micromark-util-normalize-identifier@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q=="], - - "micromark-util-resolve-all": ["micromark-util-resolve-all@2.0.1", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg=="], - - "micromark-util-sanitize-uri": ["micromark-util-sanitize-uri@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ=="], - - "micromark-util-subtokenize": ["micromark-util-subtokenize@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA=="], - - "micromark-util-symbol": ["micromark-util-symbol@2.0.1", "", {}, "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q=="], - - "micromark-util-types": ["micromark-util-types@2.0.2", "", {}, "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA=="], - - "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], - - "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - - "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], - - "napi-postinstall": ["napi-postinstall@0.3.4", "", { "bin": { "napi-postinstall": "lib/cli.js" } }, "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ=="], - - "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], - - "next": ["next@16.0.10", "", { "dependencies": { "@next/env": "16.0.10", "@swc/helpers": "0.5.15", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.0.10", "@next/swc-darwin-x64": "16.0.10", "@next/swc-linux-arm64-gnu": "16.0.10", "@next/swc-linux-arm64-musl": "16.0.10", "@next/swc-linux-x64-gnu": "16.0.10", "@next/swc-linux-x64-musl": "16.0.10", "@next/swc-win32-arm64-msvc": "16.0.10", "@next/swc-win32-x64-msvc": "16.0.10", "sharp": "^0.34.4" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-RtWh5PUgI+vxlV3HdR+IfWA1UUHu0+Ram/JBO4vWB54cVPentCD0e+lxyAYEsDTqGGMg7qpjhKh6dc6aW7W/sA=="], - - "node-releases": ["node-releases@2.0.27", "", {}, "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA=="], - - "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], - - "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], - - "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], - - "object.assign": ["object.assign@4.1.7", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0", "has-symbols": "^1.1.0", "object-keys": "^1.1.1" } }, "sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw=="], - - "object.entries": ["object.entries@1.1.9", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "define-properties": "^1.2.1", "es-object-atoms": "^1.1.1" } }, "sha512-8u/hfXFRBD1O0hPUjioLhoWFHRmt6tKA4/vZPyckBr18l1KE9uHrFaFaUi8MDRTpi4uak2goyPTSNJLXX2k2Hw=="], - - "object.fromentries": ["object.fromentries@2.0.8", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.2", "es-object-atoms": "^1.0.0" } }, "sha512-k6E21FzySsSK5a21KRADBd/NGneRegFO5pLHfdQLpRDETUNJueLXs3WCzyQ3tFRDYgbq3KHGXfTbi2bs8WQ6rQ=="], - - "object.groupby": ["object.groupby@1.0.3", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.2" } }, "sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ=="], - - "object.values": ["object.values@1.2.1", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" } }, "sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA=="], - - "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], - - "own-keys": ["own-keys@1.0.1", "", { "dependencies": { "get-intrinsic": "^1.2.6", "object-keys": "^1.1.1", "safe-push-apply": "^1.0.0" } }, "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg=="], - - "p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], - - "p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="], - - "parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="], - - "parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="], - - "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], - - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], - - "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], - - "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - - "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], - - "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], - - "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], - - "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], - - "prop-types": ["prop-types@15.8.1", "", { "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", "react-is": "^16.13.1" } }, "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg=="], - - "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], - - "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], - - "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], - - "react": ["react@19.2.1", "", {}, "sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw=="], - - "react-dom": ["react-dom@19.2.1", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.1" } }, "sha512-ibrK8llX2a4eOskq1mXKu/TGZj9qzomO+sNfO98M6d9zIPOEhlBkMkBUBLd1vgS0gQsLDBzA+8jJBVXDnfHmJg=="], - - "react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], - - "react-markdown": ["react-markdown@10.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ=="], - - "reflect.getprototypeof": ["reflect.getprototypeof@1.0.10", "", { "dependencies": { "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.9", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0", "get-intrinsic": "^1.2.7", "get-proto": "^1.0.1", "which-builtin-type": "^1.2.1" } }, "sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw=="], - - "regexp.prototype.flags": ["regexp.prototype.flags@1.5.4", "", { "dependencies": { "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-errors": "^1.3.0", "get-proto": "^1.0.1", "gopd": "^1.2.0", "set-function-name": "^2.0.2" } }, "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA=="], - - "remark-breaks": ["remark-breaks@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-newline-to-break": "^2.0.0", "unified": "^11.0.0" } }, "sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ=="], - - "remark-gfm": ["remark-gfm@4.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-gfm": "^3.0.0", "micromark-extension-gfm": "^3.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg=="], - - "remark-parse": ["remark-parse@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "micromark-util-types": "^2.0.0", "unified": "^11.0.0" } }, "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA=="], - - "remark-rehype": ["remark-rehype@11.1.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], - - "remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], - - "resolve": ["resolve@1.22.11", "", { "dependencies": { "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ=="], - - "resolve-from": ["resolve-from@4.0.0", "", {}, "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g=="], - - "resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="], - - "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], - - "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], - - "safe-array-concat": ["safe-array-concat@1.1.3", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.2", "get-intrinsic": "^1.2.6", "has-symbols": "^1.1.0", "isarray": "^2.0.5" } }, "sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q=="], - - "safe-push-apply": ["safe-push-apply@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "isarray": "^2.0.5" } }, "sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA=="], - - "safe-regex-test": ["safe-regex-test@1.1.0", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "is-regex": "^1.2.1" } }, "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw=="], - - "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], - - "semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - - "set-function-length": ["set-function-length@1.2.2", "", { "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", "has-property-descriptors": "^1.0.2" } }, "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg=="], - - "set-function-name": ["set-function-name@2.0.2", "", { "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "functions-have-names": "^1.2.3", "has-property-descriptors": "^1.0.2" } }, "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ=="], - - "set-proto": ["set-proto@1.0.0", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0" } }, "sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw=="], - - "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], - - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - - "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], - - "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], - - "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], - - "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], - - "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], - - "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], - - "stable-hash": ["stable-hash@0.0.5", "", {}, "sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA=="], - - "stop-iteration-iterator": ["stop-iteration-iterator@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "internal-slot": "^1.1.0" } }, "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ=="], - - "string.prototype.includes": ["string.prototype.includes@2.0.1", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.3" } }, "sha512-o7+c9bW6zpAdJHTtujeePODAhkuicdAryFsfVKwA+wGw89wJ4GTY484WTucM9hLtDEOpOvI+aHnzqnC5lHp4Rg=="], - - "string.prototype.matchall": ["string.prototype.matchall@4.0.12", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "define-properties": "^1.2.1", "es-abstract": "^1.23.6", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0", "get-intrinsic": "^1.2.6", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "internal-slot": "^1.1.0", "regexp.prototype.flags": "^1.5.3", "set-function-name": "^2.0.2", "side-channel": "^1.1.0" } }, "sha512-6CC9uyBL+/48dYizRf7H7VAYCMCNTBeM78x/VTUe9bFEaxBepPJDa1Ow99LqI/1yF7kuy7Q3cQsYMrcjGUcskA=="], - - "string.prototype.repeat": ["string.prototype.repeat@1.0.0", "", { "dependencies": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5" } }, "sha512-0u/TldDbKD8bFCQ/4f5+mNRrXwZ8hg2w7ZR8wa16e8z9XpePWl3eGEcUD0OXpEH/VJH/2G3gjUtR3ZOiBe2S/w=="], - - "string.prototype.trim": ["string.prototype.trim@1.2.10", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.2", "define-data-property": "^1.1.4", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-object-atoms": "^1.0.0", "has-property-descriptors": "^1.0.2" } }, "sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA=="], - - "string.prototype.trimend": ["string.prototype.trimend@1.0.9", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.2", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" } }, "sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ=="], - - "string.prototype.trimstart": ["string.prototype.trimstart@1.0.8", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" } }, "sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg=="], - - "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], - - "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], - - "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], - - "style-to-js": ["style-to-js@1.1.21", "", { "dependencies": { "style-to-object": "1.0.14" } }, "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ=="], - - "style-to-object": ["style-to-object@1.0.14", "", { "dependencies": { "inline-style-parser": "0.2.7" } }, "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw=="], - - "styled-jsx": ["styled-jsx@5.1.6", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], - - "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - - "supports-preserve-symlinks-flag": ["supports-preserve-symlinks-flag@1.0.0", "", {}, "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w=="], - - "tailwindcss": ["tailwindcss@4.1.18", "", {}, "sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw=="], - - "tapable": ["tapable@2.3.0", "", {}, "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg=="], - - "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - - "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], - - "trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="], - - "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], - - "ts-api-utils": ["ts-api-utils@2.1.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ=="], - - "tsconfig-paths": ["tsconfig-paths@3.15.0", "", { "dependencies": { "@types/json5": "^0.0.29", "json5": "^1.0.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" } }, "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg=="], - - "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - - "type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="], - - "typed-array-buffer": ["typed-array-buffer@1.0.3", "", { "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", "is-typed-array": "^1.1.14" } }, "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw=="], - - "typed-array-byte-length": ["typed-array-byte-length@1.0.3", "", { "dependencies": { "call-bind": "^1.0.8", "for-each": "^0.3.3", "gopd": "^1.2.0", "has-proto": "^1.2.0", "is-typed-array": "^1.1.14" } }, "sha512-BaXgOuIxz8n8pIq3e7Atg/7s+DpiYrxn4vdot3w9KbnBhcRQq6o3xemQdIfynqSeXeDrF32x+WvfzmOjPiY9lg=="], - - "typed-array-byte-offset": ["typed-array-byte-offset@1.0.4", "", { "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "for-each": "^0.3.3", "gopd": "^1.2.0", "has-proto": "^1.2.0", "is-typed-array": "^1.1.15", "reflect.getprototypeof": "^1.0.9" } }, "sha512-bTlAFB/FBYMcuX81gbL4OcpH5PmlFHqlCCpAl8AlEzMz5k53oNDvN8p1PNOWLEmI2x4orp3raOFB51tv9X+MFQ=="], - - "typed-array-length": ["typed-array-length@1.0.7", "", { "dependencies": { "call-bind": "^1.0.7", "for-each": "^0.3.3", "gopd": "^1.0.1", "is-typed-array": "^1.1.13", "possible-typed-array-names": "^1.0.0", "reflect.getprototypeof": "^1.0.6" } }, "sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "typescript-eslint": ["typescript-eslint@8.50.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.50.0", "@typescript-eslint/parser": "8.50.0", "@typescript-eslint/typescript-estree": "8.50.0", "@typescript-eslint/utils": "8.50.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-Q1/6yNUmCpH94fbgMUMg2/BSAr/6U7GBk61kZTv1/asghQOWOjTlp9K8mixS5NcJmm2creY+UFfGeW/+OcA64A=="], - - "unbox-primitive": ["unbox-primitive@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "has-bigints": "^1.0.2", "has-symbols": "^1.1.0", "which-boxed-primitive": "^1.1.1" } }, "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw=="], - - "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], - - "unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="], - - "unist-util-is": ["unist-util-is@6.0.1", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g=="], - - "unist-util-position": ["unist-util-position@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA=="], - - "unist-util-stringify-position": ["unist-util-stringify-position@4.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ=="], - - "unist-util-visit": ["unist-util-visit@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg=="], - - "unist-util-visit-parents": ["unist-util-visit-parents@6.0.2", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ=="], - - "unrs-resolver": ["unrs-resolver@1.11.1", "", { "dependencies": { "napi-postinstall": "^0.3.0" }, "optionalDependencies": { "@unrs/resolver-binding-android-arm-eabi": "1.11.1", "@unrs/resolver-binding-android-arm64": "1.11.1", "@unrs/resolver-binding-darwin-arm64": "1.11.1", "@unrs/resolver-binding-darwin-x64": "1.11.1", "@unrs/resolver-binding-freebsd-x64": "1.11.1", "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", "@unrs/resolver-binding-linux-x64-musl": "1.11.1", "@unrs/resolver-binding-wasm32-wasi": "1.11.1", "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" } }, "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg=="], - - "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], - - "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], - - "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], - - "vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], - - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], - - "which-boxed-primitive": ["which-boxed-primitive@1.1.1", "", { "dependencies": { "is-bigint": "^1.1.0", "is-boolean-object": "^1.2.1", "is-number-object": "^1.1.1", "is-string": "^1.1.1", "is-symbol": "^1.1.1" } }, "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA=="], - - "which-builtin-type": ["which-builtin-type@1.2.1", "", { "dependencies": { "call-bound": "^1.0.2", "function.prototype.name": "^1.1.6", "has-tostringtag": "^1.0.2", "is-async-function": "^2.0.0", "is-date-object": "^1.1.0", "is-finalizationregistry": "^1.1.0", "is-generator-function": "^1.0.10", "is-regex": "^1.2.1", "is-weakref": "^1.0.2", "isarray": "^2.0.5", "which-boxed-primitive": "^1.1.0", "which-collection": "^1.0.2", "which-typed-array": "^1.1.16" } }, "sha512-6iBczoX+kDQ7a3+YJBnh3T+KZRxM/iYNPXicqk66/Qfm1b93iu+yOImkg0zHbj5LNOcNv1TEADiZ0xa34B4q6Q=="], - - "which-collection": ["which-collection@1.0.2", "", { "dependencies": { "is-map": "^2.0.3", "is-set": "^2.0.3", "is-weakmap": "^2.0.2", "is-weakset": "^2.0.3" } }, "sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw=="], - - "which-typed-array": ["which-typed-array@1.1.19", "", { "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "for-each": "^0.3.5", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" } }, "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw=="], - - "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], - - "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], - - "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], - - "zod": ["zod@4.2.1", "", {}, "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw=="], - - "zod-validation-error": ["zod-validation-error@4.0.2", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ=="], - - "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], - - "@babel/core/json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], - - "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], - - "@eslint/eslintrc/globals": ["globals@14.0.0", "", {}, "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ=="], - - "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.7.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" }, "bundled": true }, "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg=="], - - "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.7.1", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA=="], - - "@tailwindcss/oxide-wasm32-wasi/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.1.0", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ=="], - - "@tailwindcss/oxide-wasm32-wasi/@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.0", "", { "dependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1", "@tybys/wasm-util": "^0.10.1" }, "bundled": true }, "sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA=="], - - "@tailwindcss/oxide-wasm32-wasi/@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], - - "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - - "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], - - "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], - - "@typescript-eslint/typescript-estree/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - - "eslint-import-resolver-node/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], - - "eslint-module-utils/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], - - "eslint-plugin-import/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], - - "eslint-plugin-react/resolve": ["resolve@2.0.0-next.5", "", { "dependencies": { "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA=="], - - "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - - "is-bun-module/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - - "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], - - "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - - "next/postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], - - "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], - - "sharp/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - - "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], - } -} diff --git a/docs/eslint.config.mjs b/docs/eslint.config.mjs deleted file mode 100644 index 05e726d..0000000 --- a/docs/eslint.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig, globalIgnores } from "eslint/config"; -import nextVitals from "eslint-config-next/core-web-vitals"; -import nextTs from "eslint-config-next/typescript"; - -const eslintConfig = defineConfig([ - ...nextVitals, - ...nextTs, - // Override default ignores of eslint-config-next. - globalIgnores([ - // Default ignores of eslint-config-next: - ".next/**", - "out/**", - "build/**", - "next-env.d.ts", - ]), -]); - -export default eslintConfig; diff --git a/docs/next.config.ts b/docs/next.config.ts deleted file mode 100644 index 33527ec..0000000 --- a/docs/next.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { NextConfig } from "next"; - -const nextConfig: NextConfig = { - output: "export", - basePath: "/docs", - trailingSlash: true, - images: { - unoptimized: true, - }, -}; - -export default nextConfig; diff --git a/docs/package.json b/docs/package.json deleted file mode 100644 index 4bafb7f..0000000 --- a/docs/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "docs-site", - "version": "0.1.0", - "private": true, - "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start", - "lint": "eslint" - }, - "dependencies": { - "next": "16.0.10", - "react": "19.2.1", - "react-dom": "19.2.1", - "react-markdown": "^10.1.0", - "remark-breaks": "^4.0.0", - "remark-gfm": "^4.0.1" - }, - "devDependencies": { - "@tailwindcss/postcss": "^4", - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "eslint": "^9", - "eslint-config-next": "16.0.10", - "tailwindcss": "^4", - "typescript": "^5" - } -} diff --git a/docs/policy-ingredient-status.md b/docs/policy-ingredient-status.md deleted file mode 100644 index e9c79a4..0000000 --- a/docs/policy-ingredient-status.md +++ /dev/null @@ -1,271 +0,0 @@ -# Policy Ingredient Status in API v2 Alpha - -This document summarizes the current state of the policy ingredient implementation in `policyengine-api-v2-alpha`, compares it to API v1, and outlines gaps and recommendations for migration. - -## Current Implementation - -### Database Tables - -The policy system uses a normalized relational model with two main tables: - -#### `policies` table -| Column | Type | Description | -|--------|------|-------------| -| `id` | UUID | Primary key, auto-generated | -| `name` | str | Policy name (required) | -| `description` | str \| None | Optional description | -| `simulation_modifier` | str \| None | Python code for custom variable formulas | -| `created_at` | datetime | Auto-generated UTC timestamp | -| `updated_at` | datetime | Auto-generated UTC timestamp | - -#### `parameter_values` table -| Column | Type | Description | -|--------|------|-------------| -| `id` | UUID | Primary key | -| `parameter_id` | UUID | FK to `parameters` table | -| `policy_id` | UUID \| None | FK to `policies` (NULL = baseline/current law) | -| `dynamic_id` | UUID \| None | FK to `dynamics` table | -| `value_json` | Any | The actual parameter value (JSON) | -| `start_date` | datetime | Effective start date | -| `end_date` | datetime \| None | Effective end date | -| `created_at` | datetime | Auto-generated | - -**Key design:** `parameter_values` stores BOTH baseline values (`policy_id IS NULL`) AND reform values (`policy_id = `). Each policy is self-contained with its own set of parameter value overrides - this is intentional, not a normalization issue. When running simulations, baseline (NULL) vs reform (policy_id = X) values are compared to compute impacts. - -### Supporting Tables - -#### `parameters` table (metadata) -| Column | Type | Description | -|--------|------|-------------| -| `id` | UUID | Primary key | -| `name` | str | Parameter path (e.g., `gov.irs.ctc.max`) | -| `label` | str \| None | Human-readable label | -| `description` | str \| None | Documentation | -| `data_type` | str \| None | Value type | -| `unit` | str \| None | Unit of measurement | -| `tax_benefit_model_version_id` | UUID | FK to model version | -| `created_at` | datetime | Auto-generated | - -#### `tax_benefit_models` table -| Column | Type | Description | -|--------|------|-------------| -| `id` | UUID | Primary key | -| `name` | str | Country identifier ("uk" or "us") | -| `description` | str \| None | Optional | -| `created_at` | datetime | Auto-generated | - -### API Endpoints - -| Method | Path | Status | Description | -|--------|------|--------|-------------| -| `POST` | `/policies` | ✅ Implemented | Create policy with parameter values | -| `GET` | `/policies` | ✅ Implemented | List all policies | -| `GET` | `/policies/{id}` | ✅ Implemented | Get policy by ID | -| `PATCH` | `/policies/{id}` | ❌ Missing | Update policy | -| `DELETE` | `/policies/{id}` | ❌ Missing | Delete policy | - -### Files - -| File | Purpose | -|------|---------| -| `src/policyengine_api/models/policy.py` | SQLModel definitions | -| `src/policyengine_api/models/parameter_value.py` | Parameter value model | -| `src/policyengine_api/api/policies.py` | Policy router/endpoints | -| `src/policyengine_api/api/parameter_values.py` | Parameter values router | -| `tests/test_policies.py` | Basic CRUD tests | - -### Contributors - -- **Nikhil Woodruff** - Initial implementation (Nov 2025) -- **Anthony Volk** - Added parameter_values filtering and indexing (Dec 2025) - ---- - -## Comparison: API v1 vs API v2 Alpha - -### URL Structure - -| Aspect | API v1 | API v2 Alpha | -|--------|--------|--------------| -| Create | `POST /{country_id}/policy` | `POST /policies` | -| Get | `GET /{country_id}/policy/{id}` | `GET /policies/{id}` | -| List | `GET /{country_id}/policies` | `GET /policies` | - -**Key difference:** v1 has country in URL path; v2 does not. - -### Country Handling - -| API v1 | API v2 Alpha | -|--------|--------------| -| Country in URL path (`/us/policy/123`) | Country determined by `tax_benefit_model_name` in request body | -| `@validate_country` decorator | No explicit country validation on policy endpoints | -| Policy implicitly scoped to country | Policy is country-agnostic; country context from analysis request | - -### Policy Storage - -| API v1 | API v2 Alpha | -|--------|--------------| -| `policy_hash` - hash of JSON blob | `parameter_values` - relational FK to parameters | -| Policy content stored as JSON | Policy content normalized in separate table | -| Embedded values in metadata response | Separate queries for parameter values | - -### Metadata Response (Parameters) - -**API v1 - Embedded values:** -```json -{ - "gov.irs.ctc.max": { - "type": "parameter", - "label": "CTC maximum", - "values": { - "2024-01-01": 2000, - "2023-01-01": 2000, - "2022-01-01": 2000 - } - } -} -``` - -**API v2 Alpha - Normalized:** -``` --- parameters table -id | name | label -1 | gov.irs.ctc.max | CTC maximum - --- parameter_values table -id | parameter_id | start_date | value_json | policy_id -1 | 1 | 2024-01-01 | 2000 | NULL (baseline) -2 | 1 | 2024-01-01 | 3000 | -``` - -### User Associations - -| API v1 | API v2 Alpha | -|--------|--------------| -| `user_policies` table exists in code | Not implemented | -| Endpoints NOT exposed (dead code) | No user association tables/endpoints | -| Fields: `user_id`, `reform_id`, `baseline_id`, `year`, `geography`, etc. | N/A | - ---- - -## Pros and Cons - -### API v2 Alpha Advantages - -| Pro | Explanation | -|-----|-------------| -| **Normalized schema** | Easier to query specific parameter changes without parsing JSON | -| **Relational integrity** | FK constraints ensure valid parameter references | -| **Better indexing** | Can index on `parameter_id`, `policy_id`, `start_date` | -| **Audit trail** | Each parameter value has its own `created_at` | -| **Cleaner reform diffs** | Query `WHERE policy_id = X` to see all reform changes | - -### API v2 Alpha Disadvantages - -| Con | Explanation | -|-----|-------------| -| **No country on policy** | Can't filter policies by country at DB level | -| **No user associations** | Must be built from scratch | -| **Missing PATCH/DELETE** | Incomplete CRUD | -| **No label field** | Only `name` + `description`, no user-friendly `label` | -| **More complex queries** | JOIN required to get policy with values | - -### API v1 Advantages - -| Pro | Explanation | -|-----|-------------| -| **Country in URL** | Clear API contract, easy filtering | -| **Simple storage** | Policy is a single JSON blob | -| **User associations designed** | Schema exists (though not exposed) | - -### API v1 Disadvantages - -| Con | Explanation | -|-----|-------------| -| **JSON blob parsing** | Must parse to query specific parameters | -| **No referential integrity** | Policy JSON could reference invalid parameters | -| **Harder to diff** | Must compare two JSON blobs to see changes | - ---- - -## Gaps for Migration - -### Must Have - -1. **User associations table and endpoints** - - `user_policies` table with `user_id`, `policy_id`, `label`, `created_at` - - `POST /user-policies` - Create association - - `GET /user-policies?user_id=X` - List user's policies - - `DELETE /user-policies/{id}` - Remove association - -2. **PATCH endpoint for policies** - - Update `name`, `description` - - Update parameter values - -3. **DELETE endpoint for policies** - - Cascade delete parameter values - -### Should Have - -4. **Country validation** - - Either add `country_id` to policy model OR - - Validate at creation that all parameter_ids belong to same country - -5. **Label field on policy** - - User-friendly display name separate from `name` - -### Nice to Have - -6. **Soft delete** - - `deleted_at` field instead of hard delete - -7. **Policy versioning** - - Track changes over time - ---- - -## Suggested Schema Changes - -### Option A: Add country_id to Policy - -```python -class Policy(PolicyBase, table=True): - # ... existing fields ... - country_id: str # "us" or "uk" -``` - -**Pros:** Simple filtering, matches v1 pattern -**Cons:** Redundant with parameter → tax_benefit_model → name - -### Option B: Derive country from parameters (current approach) - -Keep as-is, derive country from first parameter's tax_benefit_model. - -**Pros:** No schema change, DRY -**Cons:** Requires JOIN to filter by country - -### Recommendation: Option A - -Add explicit `country_id` for simpler queries and clearer data model. - ---- - -## Next Steps - -1. [ ] Add `country_id` to Policy model -2. [ ] Add `label` field to Policy model -3. [ ] Create UserPolicy model and table -4. [ ] Implement `PATCH /policies/{id}` endpoint -5. [ ] Implement `DELETE /policies/{id}` endpoint -6. [ ] Implement user policy association endpoints -7. [ ] Add database migration for schema changes -8. [ ] Update tests - ---- - -## References - -- Policy model: `src/policyengine_api/models/policy.py` -- Policy router: `src/policyengine_api/api/policies.py` -- Parameter value model: `src/policyengine_api/models/parameter_value.py` -- Tests: `tests/test_policies.py` diff --git a/docs/postcss.config.mjs b/docs/postcss.config.mjs deleted file mode 100644 index 61e3684..0000000 --- a/docs/postcss.config.mjs +++ /dev/null @@ -1,7 +0,0 @@ -const config = { - plugins: { - "@tailwindcss/postcss": {}, - }, -}; - -export default config; diff --git a/docs/public/file.svg b/docs/public/file.svg deleted file mode 100644 index 004145c..0000000 --- a/docs/public/file.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/public/globe.svg b/docs/public/globe.svg deleted file mode 100644 index 567f17b..0000000 --- a/docs/public/globe.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/public/next.svg b/docs/public/next.svg deleted file mode 100644 index 5174b28..0000000 --- a/docs/public/next.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/public/vercel.svg b/docs/public/vercel.svg deleted file mode 100644 index 7705396..0000000 --- a/docs/public/vercel.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/public/window.svg b/docs/public/window.svg deleted file mode 100644 index b2b2a44..0000000 --- a/docs/public/window.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/src/app/agent/page.tsx b/docs/src/app/agent/page.tsx deleted file mode 100644 index 7f18816..0000000 --- a/docs/src/app/agent/page.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { PolicyChat } from "../../components/policy-chat"; - -export default function DemoPage() { - return ( -
-

- AI policy analyst -

-

- Ask questions in natural language and get AI-generated policy analysis reports. -

- -
-
-

- How it works -

-

- This demo uses an AI agent powered by Claude that interacts with the PolicyEngine API to answer your questions. - When you ask a question, the agent: -

-
    -
  1. Searches for relevant policy parameters
    2. -
  2. Creates a policy reform based on your question
    3. -
  3. Runs an economy-wide impact analysis
    4. -
  4. Generates a report with the findings
    5. -
-

- Analysis typically takes 30-60 seconds as it runs full microsimulations on population data. -

-
- - - -
-

- Note: This is a demo of the API's capabilities. - Results are from real PolicyEngine microsimulations but should be verified for policy research. - The agent uses Claude Sonnet via a Modal Sandbox for secure, isolated execution. -

-
-
-
- ); -} diff --git a/docs/src/app/architecture/page.tsx b/docs/src/app/architecture/page.tsx deleted file mode 100644 index 9a168c7..0000000 --- a/docs/src/app/architecture/page.tsx +++ /dev/null @@ -1,138 +0,0 @@ -export default function ArchitecturePage() { - return ( -
-

- Architecture -

-

- The PolicyEngine API v2 is a distributed system for running tax-benefit microsimulations with persistence and async processing. -

- -
-
-

Components

- -
-
-

API server

-

- FastAPI application exposing RESTful endpoints for creating and managing datasets, defining policy reforms, queueing simulations, and computing aggregates. The server validates requests, persists to PostgreSQL, and queues background tasks. -

-
- -
-

Database

-

- PostgreSQL (via Supabase) stores all persistent data using SQLModel for type-safe ORM with Pydantic integration. -

-
- {["datasets", "policies", "simulations", "aggregates", "reports", "decile_impacts", "program_statistics", "parameters"].map((table) => ( - - {table} - - ))} -
-
- -
-

Worker

-

- Background workers poll for pending simulations and reports. They load datasets from storage, run PolicyEngine simulations, compute aggregates and impact statistics, then store results to the database. -

-
- -
-

Storage

-

- Dataset files (HDF5 format) are stored in Supabase Storage with local caching for performance. The storage layer handles downloads and caching transparently. -

-
-
-
- -
-

Request flow

- -
- {[ - "Client creates simulation via POST /analysis/economic-impact", - "API validates request and persists simulation + report records", - "API returns pending status immediately", - "Worker picks up pending simulation from queue", - "Worker loads dataset and runs PolicyEngine simulation", - "Worker updates simulation status to completed", - "Worker picks up pending report", - "Worker computes decile impacts and program statistics", - "Client polls GET /analysis/economic-impact/{id} to check status", - "Once complete, response includes full analysis results", - ].map((step, index) => ( -
- - {index + 1} - -

{step}

-
- ))} -
-
- -
-

Data models

-

- All models follow Pydantic/SQLModel patterns for type safety across API, database, and business logic: -

- -
-
- Base -

Shared fields across models

-
-
- Table -

Database model with ID and timestamps

-
-
- Create -

Request schema (no ID)

-
-
- Read -

Response schema (with ID and timestamps)

-
-
-
- -
-

Scaling

- -
-
-

API scaling

-

- Multiple uvicorn workers behind load balancer for horizontal scaling. -

-
-
-

Worker scaling

-

- Increase worker count for parallel simulation processing. -

-
-
-

Database

-

- PostgreSQL supports read replicas for high read throughput. -

-
-
-

Caching

-

- Deterministic UUIDs ensure same requests reuse cached results. -

-
-
-
-
-
- ); -} diff --git a/docs/src/app/design/page.tsx b/docs/src/app/design/page.tsx deleted file mode 100644 index b700d90..0000000 --- a/docs/src/app/design/page.tsx +++ /dev/null @@ -1,151 +0,0 @@ -export default function DesignPage() { - return ( -
-

- API hierarchy design -

-

- Three-level architecture for simulations, analyses, and reports. -

- -
-
-

Levels of analysis

- -
-
-
- Level 2 -

Reports

-
-

- AI-generated documents orchestrating multiple jobs. Future feature. -

-
- -
-
- Level 1 -

Analyses

-
-

- Operations on simulations - thin wrappers around policyengine package functions. -

-
- /analysis/decile-impact/* - /analysis/budget-impact/* - /analysis/winners-losers/* - /analysis/compare/* -
-
- -
-
- Level 0 -

Simulations

-
-

- Single world-state calculations - the foundation for all analyses. -

-
- /simulate/household - /simulate/economy -
-
-
-
- -
-

Modal functions

-

- All compute runs on Modal.com serverless functions with sub-1s cold starts. -

- -
- - - - - - - - - - - - - - - - - - - - - -
FunctionPurpose
simulate_household_uk/usSingle household calculation
simulate_economy_uk/usSingle economy simulation
economy_comparison_uk/usEconomy comparison (decile impacts, budget)
-
-
- -
-

Mapping to policyengine package

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
API endpointpolicyengine function
/simulate/householdcalculate_household_impact()
/simulate/economySimulation.run()
/analysis/decile-impact/*calculate_decile_impacts()
/analysis/budget-impact/*ProgrammeStatistics
/analysis/winners-losers/*ChangeAggregate
-
-
- -
-

Use cases

- -
- {[ - { case: "My tax under current law", endpoint: "/simulate/household" }, - { case: "Reform impact on my household", endpoint: "/analysis/compare/household" }, - { case: "Revenue impact of reform", endpoint: "/analysis/budget-impact/economy" }, - { case: "Decile breakdown of reform", endpoint: "/analysis/decile-impact/economy" }, - { case: "Who wins and loses", endpoint: "/analysis/winners-losers/economy" }, - { case: "Full reform analysis", endpoint: "/analysis/compare/economy" }, - ].map((item) => ( -
- {item.case} - - {item.endpoint} - -
- ))} -
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/aggregates/page.tsx b/docs/src/app/endpoints/aggregates/page.tsx deleted file mode 100644 index 11b5f40..0000000 --- a/docs/src/app/endpoints/aggregates/page.tsx +++ /dev/null @@ -1,127 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function AggregatesPage() { - return ( -
-

- Aggregates -

-

- Compute aggregate statistics from simulation results. Supports sums, means, and counts with optional filtering. Accepts a list of specifications. -

- -
-
-

- List aggregates -

- -
- -
-

- Get aggregate -

- -
- -
-

- Create aggregates (batch) -

- -
- -
-

- Aggregate specification -

-
-
- simulation_id - UUID - Source simulation -
-
- variable - string - Variable name to aggregate -
-
- aggregate_type - sum | mean | count -
-
- entity - string - Entity level (person, household, benunit, etc.) -
-
- filter_config - object | null - Optional quantile filtering -
-
-
- -
-

- Aggregate types -

-
-
- sum - Total across population (weighted) -
-
- mean - Weighted average -
-
- count - Number of entities (weighted) -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/change-aggregates/page.tsx b/docs/src/app/endpoints/change-aggregates/page.tsx deleted file mode 100644 index 980a4e0..0000000 --- a/docs/src/app/endpoints/change-aggregates/page.tsx +++ /dev/null @@ -1,139 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function ChangeAggregatesPage() { - return ( -
-

- Change aggregates -

-

- Compute change statistics comparing baseline vs reform simulations. Accepts a list of specifications. -

- -
-
-

- List change aggregates -

- -
- -
-

- Get change aggregate -

- -
- -
-

- Create change aggregates (batch) -

- -
- -
-

- Change aggregate specification -

-
-
- baseline_simulation_id - UUID - Baseline simulation -
-
- reform_simulation_id - UUID - Reform simulation -
-
- variable - string - Variable name to compare -
-
- aggregate_type - sum | mean | count -
-
- entity - string - Entity level -
-
- change_geq - number | null - Filter: change greater than or equal to -
-
- change_leq - number | null - Filter: change less than or equal to -
-
-
- -
-

- Aggregate types -

-
-
- sum - Total change across population (weighted) -
-
- mean - Weighted average change -
-
- count - Number of entities (use with change_geq/change_leq to count winners/losers) -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/datasets/page.tsx b/docs/src/app/endpoints/datasets/page.tsx deleted file mode 100644 index a260323..0000000 --- a/docs/src/app/endpoints/datasets/page.tsx +++ /dev/null @@ -1,143 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function DatasetsPage() { - return ( -
-

- Datasets -

-

- Manage microdata datasets for tax-benefit simulations. -

- -
-
-

- List datasets -

- -
- -
-

- Get dataset -

- -
- -
-

- Create dataset -

- -
- -
-

- Request parameters -

-
-
- name - string - Human-readable dataset name -
-
- description - string | null - Optional description -
-
- filepath - string - Path to HDF5 file in storage -
-
- year - integer - Simulation year for this dataset -
-
- tax_benefit_model_version_id - UUID - Model version this dataset is compatible with -
-
-
- -
-

- Delete dataset -

- -
- -
-

- Dataset object -

-
-
- id - UUID - Unique identifier -
-
- name - string - Human-readable name -
-
- description - string | null - Optional description -
-
- filepath - string - Path in storage bucket -
-
- year - integer - Simulation year -
-
- created_at - datetime - Creation timestamp -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/dynamics/page.tsx b/docs/src/app/endpoints/dynamics/page.tsx deleted file mode 100644 index 6400a4d..0000000 --- a/docs/src/app/endpoints/dynamics/page.tsx +++ /dev/null @@ -1,138 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function DynamicsPage() { - return ( -
-

- Dynamics -

-

- Define behavioural response models that modify simulation parameters based on policy changes. -

- -
-
-

- List dynamics -

- -
- -
-

- Get dynamic -

- -
- -
-

- Create dynamic -

- -
- -
-

- Request parameters -

-
-
- name - string - Model name -
-
- description - string | null - Optional description -
-
- parameter_values - array - Behavioural parameter modifications (see below) -
-
-
- -
-

- Parameter value object -

-
-
- parameter_id - UUID - Reference to parameter -
-
- value_json - object - New parameter value -
-
- start_date - date - When change takes effect -
-
- end_date - date - When change ends -
-
-
- -
-

- Dynamic object -

-
-
- id - UUID - Unique identifier -
-
- name - string - Model name -
-
- description - string | null - Optional description -
-
- parameter_values - array - Behavioural parameter modifications -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/economic-impact/page.tsx b/docs/src/app/endpoints/economic-impact/page.tsx deleted file mode 100644 index feab5af..0000000 --- a/docs/src/app/endpoints/economic-impact/page.tsx +++ /dev/null @@ -1,272 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; -import { JsonViewer } from "@/components/json-viewer"; - -export default function EconomicImpactPage() { - return ( -
-

- Economic impact -

-

- Calculate the distributional impact of policy reforms. Compares reform scenario against baseline and produces decile-level analysis plus program statistics. -

- -
-

- Recommended endpoint: This is the primary way to analyse policy reforms. It handles simulation creation, comparison, and statistical analysis automatically. -

-
- -
-
-

- Create economic impact analysis -

- -
- -
-

- Get economic impact status -

- -
- -
-

- Request parameters -

-
-
- tax_benefit_model_name - policyengine_uk or policyengine_us -
-
- dataset_id - UUID - Dataset to run simulations on -
-
- policy_id - UUID | null - Reform policy (baseline uses current law) -
-
- dynamic_id - UUID | null - Optional behavioural response model -
-
-
- -
-

- Example response (completed) -

-
- -
-
- -
-

- Decile impact object -

-
-
- decile - integer (1-10) - Income decile -
-
- income_variable - string - Variable used for decile ranking -
-
- baseline_mean - number - Mean income under baseline -
-
- reform_mean - number - Mean income under reform -
-
- absolute_change - number - reform_mean - baseline_mean -
-
- relative_change - number - Percentage change (0.05 = 5%) -
-
- count_better_off - number - People gaining from reform -
-
- count_worse_off - number - People losing from reform -
-
-
- -
-

- Program statistics object -

-
-
- program_name - string - Tax or benefit name -
-
- entity - string - Entity level (person, household, etc.) -
-
- is_tax - boolean - True for taxes, false for benefits -
-
- baseline_total - number - Total revenue/spending under baseline -
-
- reform_total - number - Total revenue/spending under reform -
-
- change - number - Difference (reform - baseline) -
-
- winners - number - Count of people benefiting -
-
- losers - number - Count of people losing out -
-
-
- -
-

- Programs analysed -

-
-
-

UK

-
    -
  • income_tax
    • -
  • national_insurance
    • -
  • vat
    • -
  • council_tax
    • -
  • universal_credit
    • -
  • child_benefit
    • -
  • pension_credit
    • -
  • income_support
    • -
  • working_tax_credit
    • -
  • child_tax_credit
    • -
-
-
-

US

-
    -
  • income_tax
    • -
  • employee_payroll_tax
    • -
  • snap
    • -
  • tanf
    • -
  • ssi
    • -
  • social_security
    • -
-
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/household-impact/page.tsx b/docs/src/app/endpoints/household-impact/page.tsx deleted file mode 100644 index 94f3f62..0000000 --- a/docs/src/app/endpoints/household-impact/page.tsx +++ /dev/null @@ -1,131 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; -import { JsonViewer } from "@/components/json-viewer"; - -export default function HouseholdImpactPage() { - return ( -
-

- Household impact comparison -

-

- Compare a household under baseline (current law) vs a policy reform. Returns both calculations plus computed differences. -

- -
-
-

- Calculate household impact -

- -
- -
-

- Request parameters -

-
-
- tax_benefit_model_name - policyengine_uk or policyengine_us -
-
- people - Array of person objects (age, employment_income, etc.) -
-
- household - Household-level variables (region, etc.) -
-
- year - integer | null - Simulation year -
-
- policy_id - UUID | null - Reform policy to compare against baseline -
-
- dynamic_id - UUID | null - Optional behavioural response model -
-
-
- -
-

- Example response -

-
- -
-
- -
-

- Response structure -

-
-
- baseline - Full household calculation under current law -
-
- reform - Full household calculation with policy applied -
-
- impact - Computed differences (baseline, reform, change) for numeric variables -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/household/page.tsx b/docs/src/app/endpoints/household/page.tsx deleted file mode 100644 index 8b24c1f..0000000 --- a/docs/src/app/endpoints/household/page.tsx +++ /dev/null @@ -1,98 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function HouseholdPage() { - return ( -
-

- Household calculate -

-

- Calculate tax and benefit impacts for a single household. Returns computed values for all variables. -

- -
-
-

- Calculate household -

- -
- -
-

- Request parameters -

-
-
- tax_benefit_model_name - policyengine_uk or policyengine_us -
-
- people - Array of person objects with variable values (e.g., age, employment_income) -
-
- benunit - UK: Benefit unit configuration -
-
- household - Household-level variables -
-
- year - Simulation year (default: 2026 UK, 2024 US) -
-
- policy_id - UUID | null - Optional policy reform to apply -
-
-
- -
-

- US-specific entities -

-
-
- marital_unit - US marital unit configuration -
-
- family - US family configuration -
-
- spm_unit - US SPM unit configuration -
-
- tax_unit - US tax unit configuration -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/parameters/page.tsx b/docs/src/app/endpoints/parameters/page.tsx deleted file mode 100644 index d9f62ad..0000000 --- a/docs/src/app/endpoints/parameters/page.tsx +++ /dev/null @@ -1,103 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function ParametersPage() { - return ( -
-

- Parameters -

-

- Tax-benefit system parameters that can be modified in policy reforms. Each parameter has a name, description, and default values over time. -

- -
-
-

- List parameters -

- -
- -
-

- Get parameter -

- -
- -
-

- List parameter values -

- -
- -
-

- Parameter object -

-
-
- id - UUID - Unique identifier -
-
- name - string - Full parameter path -
-
- label - string - Human-readable label -
-
- description - string | null - Detailed description -
-
- unit - string | null - Unit (currency, percent, etc.) -
-
-
- -
-

- Example UK parameters -

-
- - gov.hmrc.income_tax.allowances.personal_allowance.amount - - - gov.hmrc.income_tax.rates.uk.basic - - - gov.dwp.universal_credit.elements.standard_allowance.single_young - -
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/policies/page.tsx b/docs/src/app/endpoints/policies/page.tsx deleted file mode 100644 index eefba8e..0000000 --- a/docs/src/app/endpoints/policies/page.tsx +++ /dev/null @@ -1,160 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function PoliciesPage() { - return ( -
-

- Policies -

-

- Define policy reforms by modifying tax-benefit system parameters. -

- -
-
-

- List policies -

- -
- -
-

- Get policy -

- -
- -
-

- Create policy -

- -
- -
-

- Request parameters -

-
-
- name - string - Policy name -
-
- description - string | null - Optional description -
-
- parameter_values - array - Parameter modifications (see below) -
-
-
- -
-

- Delete policy -

- -
- -
-

- Policy object -

-
-
- id - UUID - Unique identifier -
-
- name - string - Policy name -
-
- description - string | null - Optional description -
-
- parameter_values - array - Parameter modifications -
-
- created_at - datetime - Creation timestamp -
-
-
- -
-

- Parameter value object -

-
-
- parameter_id - UUID - Reference to parameter -
-
- value_json - object - New parameter value -
-
- start_date - date - When change takes effect -
-
- end_date - date - When change ends -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/simulations/page.tsx b/docs/src/app/endpoints/simulations/page.tsx deleted file mode 100644 index 75e5d8f..0000000 --- a/docs/src/app/endpoints/simulations/page.tsx +++ /dev/null @@ -1,158 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function SimulationsPage() { - return ( -
-

- Simulations -

-

- Run tax-benefit microsimulations on datasets. Simulations are processed asynchronously by background workers. -

- -
-

- Note: Simulations use deterministic UUIDs based on inputs. Requesting the same simulation twice returns the cached result instead of running again. -

-
- -
-
-

- List simulations -

- -
- -
-

- Get simulation -

- -
- -
-

- Create simulation -

- -
- -
-

- Request parameters -

-
-
- dataset_id - UUID - Dataset to run simulation on -
-
- tax_benefit_model_version_id - UUID - Model version to use -
-
- policy_id - UUID | null - Optional policy reform -
-
- dynamic_id - UUID | null - Optional behavioural response model -
-
-
- -
-

- Response object -

-
-
- id - UUID - Deterministic identifier -
-
- dataset_id - UUID - Reference to dataset -
-
- policy_id - UUID | null - Optional policy reform -
-
- dynamic_id - UUID | null - Optional behavioural response -
-
- status - enum - pending | running | completed | failed -
-
- error_message - string | null - Error details if failed -
-
- started_at - datetime | null - When processing started -
-
- completed_at - datetime | null - When processing finished -
-
-
- -
-

- Status values -

-
-
- pending - Queued, waiting for worker -
-
- running - Worker is processing -
-
- completed - Successfully finished -
-
- failed - Error occurred (see error_message) -
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/tax-benefit-models/page.tsx b/docs/src/app/endpoints/tax-benefit-models/page.tsx deleted file mode 100644 index 45a6c06..0000000 --- a/docs/src/app/endpoints/tax-benefit-models/page.tsx +++ /dev/null @@ -1,96 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function TaxBenefitModelsPage() { - return ( -
-

- Tax benefit models -

-

- Available tax-benefit system models (UK and US). Each model has versions with associated variables and parameters. -

- -
-
-

- List models -

- -
- -
-

- Get model -

- -
- -
-

- List model versions -

- -
- -
-

- Get model version -

- -
- -
-

- Available models -

-
-
- policyengine-uk -

- UK tax and benefit system including income tax, NI, benefits, and more. -

-
-
- policyengine-us -

- US federal and state tax system including income tax, payroll tax, SNAP, TANF, etc. -

-
-
-
-
-
- ); -} diff --git a/docs/src/app/endpoints/variables/page.tsx b/docs/src/app/endpoints/variables/page.tsx deleted file mode 100644 index 2eabb27..0000000 --- a/docs/src/app/endpoints/variables/page.tsx +++ /dev/null @@ -1,105 +0,0 @@ -"use client"; - -import { ApiPlayground } from "@/components/api-playground"; - -export default function VariablesPage() { - return ( -
-

- Variables -

-

- Tax-benefit system variables that can be calculated or used as inputs. Variables are associated with model versions. -

- -
-
-

- List variables -

- -
- -
-

- Get variable -

- -
- -
-

- Variable object -

-
-
- id - UUID - Unique identifier -
-
- name - string - Variable name (e.g., income_tax) -
-
- label - string - Human-readable label -
-
- description - string | null - Detailed description -
-
- entity - string - Entity level (person, household, etc.) -
-
- value_type - string - Data type (float, int, bool, enum) -
-
- unit - string | null - Unit of measurement -
-
-
- -
-

- Common UK variables -

-
- {["income_tax", "national_insurance", "universal_credit", "child_benefit", "council_tax", "household_net_income", "employment_income", "pension_income"].map((v) => ( - {v} - ))} -
-
- -
-

- Common US variables -

-
- {["income_tax", "employee_payroll_tax", "snap", "tanf", "ssi", "social_security", "employment_income", "spm_unit_net_income"].map((v) => ( - {v} - ))} -
-
-
-
- ); -} diff --git a/docs/src/app/favicon.ico b/docs/src/app/favicon.ico deleted file mode 100644 index 718d6fea4835ec2d246af9800eddb7ffb276240c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m diff --git a/docs/src/app/globals.css b/docs/src/app/globals.css deleted file mode 100644 index 34f6080..0000000 --- a/docs/src/app/globals.css +++ /dev/null @@ -1,234 +0,0 @@ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap'); -@import "tailwindcss"; - -@theme { - /* PolicyEngine brand colors */ - --color-pe-green: #2c6e49; - --color-pe-green-light: #4c9a6d; - --color-pe-green-dark: #1a4a2e; - --color-pe-blue: #1a365d; - --color-pe-slate: #334155; - --color-pe-gray: #64748b; - - /* Neutral palette */ - --color-surface: #fafbfc; - --color-surface-elevated: #ffffff; - --color-surface-sunken: #f1f5f9; - --color-border: #e2e8f0; - --color-border-strong: #cbd5e1; - - /* Text colors */ - --color-text-primary: #0f172a; - --color-text-secondary: #475569; - --color-text-muted: #94a3b8; - - /* Code colors */ - --color-code-bg: #1e293b; - --color-code-text: #e2e8f0; - - /* Status colors */ - --color-success: #16a34a; - --color-error: #dc2626; - --color-warning: #d97706; - --color-info: #2563eb; - - /* Typography */ - --font-sans: "Inter", ui-sans-serif, system-ui, sans-serif; - --font-mono: "JetBrains Mono", ui-monospace, monospace; - --font-display: "Instrument Serif", Georgia, serif; -} - -body { - font-family: var(--font-sans); - background: var(--color-surface); - color: var(--color-text-primary); - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -/* Custom scrollbar */ -::-webkit-scrollbar { - width: 8px; - height: 8px; -} - -::-webkit-scrollbar-track { - background: var(--color-surface-sunken); -} - -::-webkit-scrollbar-thumb { - background: var(--color-border-strong); - border-radius: 4px; -} - -::-webkit-scrollbar-thumb:hover { - background: var(--color-pe-gray); -} - -/* Code block styling */ -pre { - font-family: var(--font-mono); - font-size: 13px; - line-height: 1.6; -} - -code { - font-family: var(--font-mono); -} - -/* Method badges */ -.method-get { background: #dbeafe; color: #1d4ed8; } -.method-post { background: #dcfce7; color: #16a34a; } -.method-put { background: #fef3c7; color: #d97706; } -.method-delete { background: #fee2e2; color: #dc2626; } - -/* Smooth transitions */ -* { - transition-property: background-color, border-color, color, fill, stroke, opacity, box-shadow, transform; - transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); - transition-duration: 150ms; -} - -/* Focus states */ -*:focus-visible { - outline: 2px solid var(--color-pe-green); - outline-offset: 2px; -} - -/* Selection */ -::selection { - background: var(--color-pe-green); - color: white; -} - -/* Agent response content */ -.response-content { - font-family: var(--font-inter), -apple-system, BlinkMacSystemFont, sans-serif; - font-size: 15px; - line-height: 1.7; - color: #1e293b; - -webkit-font-smoothing: antialiased; -} -.response-content p { - margin: 0.75em 0; -} -.response-content p:first-child { - margin-top: 0; -} -.response-content p:last-child { - margin-bottom: 0; -} -.response-content h1, -.response-content h2, -.response-content h3 { - font-weight: 600; - color: #0f172a; - margin-top: 1.25em; - margin-bottom: 0.5em; - line-height: 1.3; -} -.response-content h1 { - font-size: 1.2em; -} -.response-content h2 { - font-size: 1.1em; -} -.response-content h3 { - font-size: 1em; -} -.response-content h1:first-child, -.response-content h2:first-child, -.response-content h3:first-child { - margin-top: 0; -} -.response-content strong { - font-weight: 600; - color: #0f172a; -} -.response-content ul { - margin: 0.75em 0; - padding-left: 1.25em; - list-style-type: disc; -} -.response-content ol { - margin: 0.75em 0; - padding-left: 1.25em; - list-style-type: decimal; -} -.response-content li { - margin: 0.35em 0; - padding-left: 0.25em; -} -.response-content ul ul { - list-style-type: circle; -} -.response-content ul ul ul { - list-style-type: square; -} -.response-content code { - font-family: "JetBrains Mono", ui-monospace, monospace; - font-size: 0.875em; - background: #f1f5f9; - padding: 0.2em 0.4em; - border-radius: 4px; - color: #334155; -} -.response-content pre { - font-family: "JetBrains Mono", ui-monospace, monospace; - font-size: 13px; - background: #1e293b; - color: #e2e8f0; - padding: 1em 1.25em; - border-radius: 8px; - overflow-x: auto; - margin: 1em 0; - line-height: 1.6; -} -.response-content pre code { - background: none; - padding: 0; - font-size: inherit; - color: inherit; -} -.response-content table { - width: 100%; - border-collapse: collapse; - margin: 1em 0; - font-family: var(--font-inter), -apple-system, sans-serif; - font-size: 14px; -} -.response-content th { - background: #f8fafc; - border: 1px solid #e2e8f0; - padding: 0.625em 0.875em; - text-align: left; - font-weight: 600; - color: #475569; -} -.response-content td { - border: 1px solid #e2e8f0; - padding: 0.625em 0.875em; - color: #334155; -} -.response-content tr:hover td { - background: #f8fafc; -} -.response-content blockquote { - border-left: 3px solid #2c6e49; - padding-left: 1em; - margin: 1em 0; - color: #64748b; - font-style: italic; -} -.response-content a { - color: #2c6e49; - text-decoration: underline; -} -.response-content a:hover { - color: #1a4a2e; -} -.response-content hr { - border: none; - border-top: 1px solid #e2e8f0; - margin: 1.5em 0; -} diff --git a/docs/src/app/layout.tsx b/docs/src/app/layout.tsx deleted file mode 100644 index 77e529c..0000000 --- a/docs/src/app/layout.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import type { Metadata } from "next"; -import { Inter, JetBrains_Mono } from "next/font/google"; -import "./globals.css"; -import { Sidebar } from "@/components/sidebar"; -import { Header } from "@/components/header"; -import { ApiProvider } from "@/components/api-context"; - -const inter = Inter({ - variable: "--font-inter", - subsets: ["latin"], -}); - -const jetbrainsMono = JetBrains_Mono({ - variable: "--font-mono", - subsets: ["latin"], -}); - -export const metadata: Metadata = { - title: "PolicyEngine API v2 | Documentation", - description: "Interactive API documentation for PolicyEngine's tax-benefit microsimulation API", -}; - -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode; -}>) { - return ( - - - -
- -
-
-
- {children} -
-
-
- - - - ); -} diff --git a/docs/src/app/mcp/page.tsx b/docs/src/app/mcp/page.tsx deleted file mode 100644 index 11df87c..0000000 --- a/docs/src/app/mcp/page.tsx +++ /dev/null @@ -1,93 +0,0 @@ -export default function McpPage() { - return ( -
-

- MCP integration -

-

- Use the PolicyEngine API as an MCP server for AI assistants like Claude. -

- -
-
-

- What is MCP? -

-

- The Model Context Protocol (MCP) is a standard for AI assistants to interact with external tools and data sources. - The PolicyEngine API exposes all endpoints as MCP tools at /mcp, - allowing AI assistants to calculate taxes and benefits, run economic impact analyses, and query policy data. -

-
- -
-

- Add to Claude Code -

-

- Run this command: -

- 
-{`claude mcp add --transport http policyengine https://v2.api.policyengine.org/mcp/`}
-
-
- -
-

- Add to Claude Desktop -

-

- Add this to your claude_desktop_config.json file: -

- 
-{`{
-  "mcpServers": {
-    "policyengine": {
-      "type": "url",
-      "url": "https://v2.api.policyengine.org/mcp/"
-    }
-  }
-}`}
-
-
- -
-

- Available tools -

-

- All API endpoints are exposed as MCP tools. Key capabilities include: -

-
    -
  • household_calculate — calculate taxes and benefits for a household
    • -
  • household_impact — compare baseline vs reform policy impact
    • -
  • analysis_economic_impact — run population-wide economic analysis
    • -
  • policies_list / policies_create — manage policy reforms
    • -
  • variables_list / parameters_list — query tax-benefit system metadata
    • -
  • datasets_list — list available population datasets
    • -
-
- -
-

- Example prompts -

-

- Once connected, you can ask Claude things like: -

-
-
- "Calculate the net income for a UK household with two adults earning £40,000 and £30,000" -
-
- "What would happen to this household's benefits if we increased the personal allowance to £15,000?" -
-
- "List all the parameters related to child benefit" -
-
-
-
-
- ); -} diff --git a/docs/src/app/modal/page.tsx b/docs/src/app/modal/page.tsx deleted file mode 100644 index bc50001..0000000 --- a/docs/src/app/modal/page.tsx +++ /dev/null @@ -1,205 +0,0 @@ -export default function ModalPage() { - return ( -
-

- Modal compute -

-

- PolicyEngine uses Modal.com for serverless compute, with two separate apps for different workloads. -

- -
-
-

Why two apps?

-

- The API uses two separate Modal apps rather than one combined app. This separation is intentional and provides several benefits: -

-
-
-

Image size

-

- The policyengine app has massive container images (multiple GB) with the full UK and US tax-benefit models pre-loaded. The policyengine-sandbox app is minimal - just the Anthropic SDK and requests library. -

-
-
-

Cold start optimisation

-

- The main app uses Modal's memory snapshot feature to pre-load PolicyEngine models at build time. When a function cold starts, it restores from the snapshot rather than re-importing the models, achieving sub-1s cold starts for functions that would otherwise take 30+ seconds to import. -

-
-
-

Architectural decoupling

-

- The sandbox/agent calls the public API endpoints, which then trigger the simulation functions. They're independent - the agent doesn't directly import PolicyEngine models, it makes HTTP calls. -

-
-
-

Independent scaling

-

- Simulation workloads scale differently from agent chat sessions. Keeping them separate lets Modal scale each independently based on demand. -

-
-
-
- -
-

policyengine app

-

- The main compute app for running simulations. Located at src/policyengine_api/modal_app.py. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FunctionImageMemoryPurpose
simulate_household_ukuk_image4GBSingle UK household calculation
simulate_household_usus_image4GBSingle US household calculation
simulate_economy_ukuk_image8GBUK economy simulation
simulate_economy_usus_image8GBUS economy simulation
economy_comparison_ukuk_image8GBUK decile impacts, budget impact
economy_comparison_usus_image8GBUS decile impacts, budget impact
-
- -
-

- Deploy with: modal deploy src/policyengine_api/modal_app.py -

-
-
- -
-

policyengine-sandbox app

-

- Lightweight app for the AI agent. Located at src/policyengine_api/agent_sandbox.py. -

- -
- - - - - - - - - - - - - - - -
FunctionDependenciesPurpose
run_agentanthropic, requestsAgentic loop using Claude with API tools
-
- -

- The agent dynamically generates Claude tools from the OpenAPI spec, then executes an agentic loop to answer policy questions by making API calls. It doesn't import PolicyEngine directly. -

- -
-

- Deploy with: modal deploy src/policyengine_api/agent_sandbox.py -

-
-
- -
-

Memory snapshots

-

- The policyengine app uses Modal's run_function to snapshot the Python interpreter state after importing the models: -

- 
-{`def _import_uk():
-    from policyengine.tax_benefit_models.uk import uk_latest
-    print("UK model loaded and snapshotted")
-
-uk_image = base_image.run_commands(
-    "uv pip install --system policyengine-uk>=2.0.0"
-).run_function(_import_uk)`}
-
-

- When a cold start happens, Modal restores from this snapshot rather than re-running the imports. This turns a 30+ second import into sub-second startup. -

-
- -
-

Secrets

-

- Each app uses different Modal secrets: -

-
-
- policyengine-db -

Database credentials for the main app (DATABASE_URL, SUPABASE_URL, SUPABASE_KEY)

-
-
- anthropic-api-key -

Anthropic API key for the sandbox app (ANTHROPIC_API_KEY)

-
-
-
- -
-

Request flow

-
- {[ - "Client calls API endpoint (e.g. POST /household/calculate)", - "FastAPI validates request and creates job record in Supabase", - "FastAPI triggers Modal function asynchronously", - "API returns job ID immediately", - "Modal function runs calculation with pre-loaded models", - "Modal function writes results directly to Supabase", - "Client polls API until job status = completed", - ].map((step, index) => ( -
- - {index + 1} - -

{step}

-
- ))} -
-
-
-
- ); -} diff --git a/docs/src/app/page.tsx b/docs/src/app/page.tsx deleted file mode 100644 index a77ef67..0000000 --- a/docs/src/app/page.tsx +++ /dev/null @@ -1,103 +0,0 @@ -import Link from "next/link"; - -export default function Home() { - return ( -
-

- PolicyEngine API v2 -

-

- A distributed system for running tax-benefit microsimulations with persistence and async processing. -

- -
- -

- Quick start -

-

- Get up and running with the API in minutes. Complete walkthrough from setup to your first simulation. -

- - - -

- Economic impact -

-

- Calculate distributional impacts, decile analysis, and program statistics for policy reforms. -

- - - -

- Architecture -

-

- Understand the system design: API server, worker, database, and how they work together. -

- - - -

- API reference -

-

- Interactive documentation for all endpoints with live testing capabilities. -

- -
- -
-

- Features -

-
-
-

Async processing

-

- Simulations run asynchronously via a worker queue. Poll for status or use webhooks. -

-
-
-

Deterministic caching

-

- Same inputs produce the same simulation ID. Results are cached and reused automatically. -

-
-
-

UK and US models

-

- Full support for PolicyEngine UK and US tax-benefit systems with comprehensive datasets. -

-
-
-

Policy reforms

-

- Define custom parameter changes and compare reform scenarios against baselines. -

-
-
-
- -
-

- Tip: Use the base URL input in the header to switch between local and production environments. - The default is https://v2.api.policyengine.org. -

-
-
- ); -} diff --git a/docs/src/app/quickstart/page.tsx b/docs/src/app/quickstart/page.tsx deleted file mode 100644 index 4785324..0000000 --- a/docs/src/app/quickstart/page.tsx +++ /dev/null @@ -1,149 +0,0 @@ -import { JsonViewer } from "@/components/json-viewer"; - -export default function QuickstartPage() { - return ( -
-

- Quick start -

-

- Get up and running with the PolicyEngine API in minutes. -

- -
-
-

1. Start the services

-

- Clone the repository and start the services using Docker Compose: -

- 
-{`git clone https://github.com/PolicyEngine/policyengine-api-v2
-cd policyengine-api-v2
-docker compose up -d`}
-
-

- Wait for all services to be healthy before proceeding. -

-
- -
-

2. Create a policy reform

-

- Define a policy reform by specifying parameter changes: -

- 
-{`curl -X POST https://v2.api.policyengine.org/policies \\
-  -H "Content-Type: application/json" \\
-  -d '{
-    "name": "Increased personal allowance",
-    "description": "Raises personal allowance to £15,000",
-    "parameter_values": [{
-      "parameter_id": "",
-      "value_json": {"value": 15000},
-      "start_date": "2026-01-01",
-      "end_date": "2026-12-31"
-    }]
-  }'`}
-
-
- -
-

3. Run economic impact analysis

-

- Submit an economic impact analysis request. This creates baseline and reform simulations and queues them for processing: -

- 
-{`curl -X POST https://v2.api.policyengine.org/analysis/economic-impact \\
-  -H "Content-Type: application/json" \\
-  -d '{
-    "tax_benefit_model_name": "policyengine_uk",
-    "dataset_id": "",
-    "policy_id": ""
-  }'`}
-
-

- The response includes a report_id and initial status. -

-
- -
-

4. Poll for results

-

- Check the status until the report is complete: -

- 
-{`curl https://v2.api.policyengine.org/analysis/economic-impact/`}
-
-

- Once complete, the response includes decile impacts and program statistics: -

-
- -
-
- -
-

Python example

-

- Complete workflow using httpx: -

- 
-{`import httpx
-import time
-
-BASE_URL = "https://v2.api.policyengine.org"
-
-# Create economic impact analysis
-response = httpx.post(
-    f"{BASE_URL}/analysis/economic-impact",
-    json={
-        "tax_benefit_model_name": "policyengine_uk",
-        "dataset_id": "",
-        "policy_id": "",
-    },
-).json()
-
-report_id = response["report_id"]
-
-# Poll for completion
-while True:
-    result = httpx.get(f"{BASE_URL}/analysis/economic-impact/{report_id}").json()
-    if result["status"] == "completed":
-        break
-    elif result["status"] == "failed":
-        raise Exception(f"Analysis failed: {result['error_message']}")
-    time.sleep(5)
-
-# Access results
-for decile in result["decile_impacts"]:
-    print(f"Decile {decile['decile']}: {decile['relative_change']:.1%} change")
-
-for prog in result["program_statistics"]:
-    print(f"{prog['program_name']}: £{prog['change']/1e9:.1f}bn change")`}
-
-
-
-
- ); -} diff --git a/docs/src/app/reference/models/page.tsx b/docs/src/app/reference/models/page.tsx deleted file mode 100644 index 7c66417..0000000 --- a/docs/src/app/reference/models/page.tsx +++ /dev/null @@ -1,172 +0,0 @@ -export default function ModelsPage() { - const models = [ - { - name: "Dataset", - description: "Microdata file metadata", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "name", type: "string", description: "Human-readable name" }, - { name: "description", type: "string | null", description: "Optional description" }, - { name: "filepath", type: "string", description: "Path in storage bucket" }, - { name: "year", type: "integer", description: "Simulation year" }, - { name: "tax_benefit_model_version_id", type: "UUID", description: "Associated model version" }, - { name: "created_at", type: "datetime", description: "Creation timestamp" }, - ], - }, - { - name: "Policy", - description: "Parameter reform definition", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "name", type: "string", description: "Policy name" }, - { name: "description", type: "string | null", description: "Optional description" }, - { name: "parameter_values", type: "ParameterValue[]", description: "Parameter modifications" }, - { name: "created_at", type: "datetime", description: "Creation timestamp" }, - ], - }, - { - name: "ParameterValue", - description: "Single parameter modification", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "parameter_id", type: "UUID", description: "Reference to parameter" }, - { name: "value_json", type: "object", description: "New parameter value" }, - { name: "start_date", type: "date", description: "Effective start date" }, - { name: "end_date", type: "date", description: "Effective end date" }, - ], - }, - { - name: "Simulation", - description: "Tax-benefit microsimulation run", - fields: [ - { name: "id", type: "UUID", description: "Deterministic identifier" }, - { name: "dataset_id", type: "UUID", description: "Input dataset" }, - { name: "tax_benefit_model_version_id", type: "UUID", description: "Model version" }, - { name: "policy_id", type: "UUID | null", description: "Optional reform policy" }, - { name: "dynamic_id", type: "UUID | null", description: "Optional behavioural model" }, - { name: "status", type: "SimulationStatus", description: "Processing status" }, - { name: "error_message", type: "string | null", description: "Error if failed" }, - { name: "started_at", type: "datetime | null", description: "Processing start time" }, - { name: "completed_at", type: "datetime | null", description: "Processing end time" }, - ], - }, - { - name: "Report", - description: "Economic impact analysis container", - fields: [ - { name: "id", type: "UUID", description: "Deterministic identifier" }, - { name: "label", type: "string", description: "Human-readable label" }, - { name: "baseline_simulation_id", type: "UUID", description: "Baseline simulation" }, - { name: "reform_simulation_id", type: "UUID", description: "Reform simulation" }, - { name: "status", type: "ReportStatus", description: "Processing status" }, - { name: "error_message", type: "string | null", description: "Error if failed" }, - { name: "created_at", type: "datetime", description: "Creation timestamp" }, - ], - }, - { - name: "DecileImpact", - description: "Distributional impact by income decile", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "report_id", type: "UUID", description: "Parent report" }, - { name: "decile", type: "integer", description: "Income decile (1-10)" }, - { name: "income_variable", type: "string", description: "Variable for ranking" }, - { name: "baseline_mean", type: "number", description: "Baseline mean income" }, - { name: "reform_mean", type: "number", description: "Reform mean income" }, - { name: "absolute_change", type: "number", description: "Absolute difference" }, - { name: "relative_change", type: "number", description: "Percentage change" }, - { name: "count_better_off", type: "number", description: "Winners count" }, - { name: "count_worse_off", type: "number", description: "Losers count" }, - { name: "count_no_change", type: "number", description: "No change count" }, - ], - }, - { - name: "ProgramStatistics", - description: "Tax/benefit program impact statistics", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "report_id", type: "UUID", description: "Parent report" }, - { name: "program_name", type: "string", description: "Program name" }, - { name: "entity", type: "string", description: "Entity level" }, - { name: "is_tax", type: "boolean", description: "True for taxes" }, - { name: "baseline_total", type: "number", description: "Baseline total" }, - { name: "reform_total", type: "number", description: "Reform total" }, - { name: "change", type: "number", description: "Difference" }, - { name: "baseline_count", type: "number", description: "Baseline recipients" }, - { name: "reform_count", type: "number", description: "Reform recipients" }, - { name: "winners", type: "number", description: "Winners count" }, - { name: "losers", type: "number", description: "Losers count" }, - ], - }, - { - name: "AggregateOutput", - description: "Computed aggregate statistic", - fields: [ - { name: "id", type: "UUID", description: "Unique identifier" }, - { name: "simulation_id", type: "UUID", description: "Source simulation" }, - { name: "variable", type: "string", description: "Variable name" }, - { name: "aggregate_type", type: "AggregateType", description: "Calculation type" }, - { name: "entity", type: "string", description: "Entity level" }, - { name: "filter_config", type: "object | null", description: "Optional filter" }, - { name: "result", type: "number", description: "Computed value" }, - ], - }, - ]; - - return ( -
-

- Models -

-

- Data models used throughout the API. All models use UUID identifiers and include timestamps. -

- -
- {models.map((model) => ( -
-

- {model.name} -

-

- {model.description} -

-
- - - - - - - - - - {model.fields.map((field) => ( - - - - - - ))} - -
- Field - - Type - - Description -
- {field.name} - - {field.type} - {field.description}
-
-
- ))} -
-
- ); -} diff --git a/docs/src/app/reference/status-codes/page.tsx b/docs/src/app/reference/status-codes/page.tsx deleted file mode 100644 index 83d58a4..0000000 --- a/docs/src/app/reference/status-codes/page.tsx +++ /dev/null @@ -1,173 +0,0 @@ -export default function StatusCodesPage() { - const statusCodes = [ - { - code: "200", - name: "OK", - description: "Request succeeded. Response body contains the requested data.", - color: "bg-green-100 text-green-700", - }, - { - code: "201", - name: "Created", - description: "Resource created successfully. Response body contains the new resource.", - color: "bg-green-100 text-green-700", - }, - { - code: "204", - name: "No Content", - description: "Request succeeded with no response body (e.g., DELETE operations).", - color: "bg-green-100 text-green-700", - }, - { - code: "400", - name: "Bad Request", - description: "Invalid request body or parameters. Check the error message for details.", - color: "bg-red-100 text-red-700", - }, - { - code: "404", - name: "Not Found", - description: "The requested resource does not exist.", - color: "bg-red-100 text-red-700", - }, - { - code: "422", - name: "Unprocessable Entity", - description: "Request body failed validation. Response contains field-level errors.", - color: "bg-red-100 text-red-700", - }, - { - code: "500", - name: "Internal Server Error", - description: "Unexpected server error. Please report if persistent.", - color: "bg-red-100 text-red-700", - }, - ]; - - const simulationStatuses = [ - { - status: "pending", - description: "Queued and waiting for a worker to pick up", - color: "bg-gray-100 text-gray-700", - }, - { - status: "running", - description: "Currently being processed by a worker", - color: "bg-blue-100 text-blue-700", - }, - { - status: "completed", - description: "Successfully finished processing", - color: "bg-green-100 text-green-700", - }, - { - status: "failed", - description: "An error occurred during processing", - color: "bg-red-100 text-red-700", - }, - ]; - - return ( -
-

- Status codes -

-

- HTTP status codes and resource statuses used by the API. -

- -
-
-

- HTTP status codes -

-
- {statusCodes.map((item) => ( -
- - {item.code} - -
- {item.name} -

- {item.description} -

-
-
- ))} -
-
- -
-

- Simulation status -

-

- Simulations and reports progress through these statuses: -

-
- {simulationStatuses.map((item) => ( -
- - {item.status} - -

- {item.description} -

-
- ))} -
-
- -
-

- Error response format -

-

- Error responses follow a consistent format: -

- 
-{`{
-  "detail": "Error message describing what went wrong"
-}`}
-
-

- Validation errors (422) include field-level details: -

- 
-{`{
-  "detail": [
-    {
-      "loc": ["body", "dataset_id"],
-      "msg": "field required",
-      "type": "value_error.missing"
-    }
-  ]
-}`}
-
-
- -
-

- Polling strategy -

-

- For async operations like simulations and reports: -

-
    -
  1. Submit the request (POST) and receive the resource with pending status
    2. -
  2. Poll the GET endpoint every 2-5 seconds
    3. -
  3. Check the status field in the response
    4. -
  4. Stop polling when status is completed or failed
    5. -
  5. If failed, check the error_message field for details
    6. -
-
-

- Tip: Use exponential backoff for production systems to avoid overwhelming the API during high load. -

-
-
-
-
- ); -} diff --git a/docs/src/app/setup/page.tsx b/docs/src/app/setup/page.tsx deleted file mode 100644 index cad7242..0000000 --- a/docs/src/app/setup/page.tsx +++ /dev/null @@ -1,198 +0,0 @@ -export default function SetupPage() { - return ( -
-

- Environment setup -

-

- Environment variables across services: local dev, Cloud Run, Modal, and GitHub Actions. -

- -
-
-

Overview

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ServiceConfig locationSecrets storage
Local dev.env fileLocal file
Cloud Run (API)TerraformGitHub Secrets
Modal.comModal secretsModal dashboard
GitHub ActionsWorkflow filesGitHub Secrets
-
-
- -
-

Local development

-

- Copy .env.example to .env and configure: -

- -
- 
{`# Supabase (from \`supabase start\` output)
-SUPABASE_URL=http://127.0.0.1:54321
-SUPABASE_KEY=eyJ...
-SUPABASE_SERVICE_KEY=eyJ...
-SUPABASE_DB_URL=postgresql://postgres:postgres@127.0.0.1:54322/postgres
-
-# Storage
-STORAGE_BUCKET=datasets
-
-# API
-API_TITLE=PolicyEngine API
-API_VERSION=0.1.0
-API_PORT=8000
-DEBUG=true
-
-# Observability
-LOGFIRE_TOKEN=...
-LOGFIRE_ENVIRONMENT=local
-
-# Modal (for local testing)
-MODAL_TOKEN_ID=ak-...
-MODAL_TOKEN_SECRET=as-...`}
-
-
- -
-

Modal.com secrets

-

- Modal functions read from a secret named policyengine-db: -

- -
- 
{`modal secret create policyengine-db \\
-  DATABASE_URL="postgresql://..." \\
-  SUPABASE_URL="https://xxx.supabase.co" \\
-  SUPABASE_KEY="eyJ..." \\
-  STORAGE_BUCKET="datasets"`}
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - -
KeyDescription
DATABASE_URLSupabase Postgres (use connection pooler)
SUPABASE_URLSupabase project URL
SUPABASE_KEYSupabase anon or service key
STORAGE_BUCKETSupabase storage bucket name
-
-
- -
-

GitHub Actions

-

- Required secrets for CI/CD (Settings → Secrets): -

- -
-
-

Secrets

-
- {["SUPABASE_URL", "SUPABASE_KEY", "SUPABASE_DB_URL", "LOGFIRE_TOKEN", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "GCP_WORKLOAD_IDENTITY_PROVIDER", "GCP_SERVICE_ACCOUNT"].map((secret) => ( -
- {secret} -
- ))} -
-
-
-

Variables

-
- {["GCP_PROJECT_ID", "GCP_REGION", "PROJECT_NAME", "API_SERVICE_NAME"].map((variable) => ( -
- {variable} -
- ))} -
-
-
-
- -
-

Database URLs

-

- Supabase provides multiple connection options: -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - -
TypeUse casePort
DirectLocal dev54322
Pooler (transaction)Cloud Run, Modal6543
Pooler (session)Long connections5432
-
- -

- Use the transaction pooler (port 6543) for serverless environments - handles IPv4 and connection limits. -

-
-
-
- ); -} diff --git a/docs/src/components/api-context.tsx b/docs/src/components/api-context.tsx deleted file mode 100644 index a76573d..0000000 --- a/docs/src/components/api-context.tsx +++ /dev/null @@ -1,29 +0,0 @@ -"use client"; - -import { createContext, useContext, useState, ReactNode } from "react"; - -interface ApiContextType { - baseUrl: string; - setBaseUrl: (url: string) => void; -} - -const defaultBaseUrl = process.env.NEXT_PUBLIC_API_URL || "https://v2.api.policyengine.org"; - -const ApiContext = createContext({ - baseUrl: defaultBaseUrl, - setBaseUrl: () => {}, -}); - -export function ApiProvider({ children }: { children: ReactNode }) { - const [baseUrl, setBaseUrl] = useState(defaultBaseUrl); - - return ( - - {children} - - ); -} - -export function useApi() { - return useContext(ApiContext); -} diff --git a/docs/src/components/api-playground.tsx b/docs/src/components/api-playground.tsx deleted file mode 100644 index 3ea693b..0000000 --- a/docs/src/components/api-playground.tsx +++ /dev/null @@ -1,184 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { useApi } from "./api-context"; -import { JsonViewer } from "./json-viewer"; - -type HttpMethod = "GET" | "POST" | "PUT" | "DELETE"; - -interface ApiPlaygroundProps { - method: HttpMethod; - endpoint: string; - description: string; - defaultBody?: Record | Record[]; - pathParams?: { name: string; description: string; example: string }[]; -} - -export function ApiPlayground({ - method, - endpoint, - description, - defaultBody, - pathParams = [], -}: ApiPlaygroundProps) { - const { baseUrl } = useApi(); - const [body, setBody] = useState(defaultBody ? JSON.stringify(defaultBody, null, 2) : ""); - const [response, setResponse] = useState<{ status: number; data: unknown } | null>(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - const [pathValues, setPathValues] = useState>(() => { - const initial: Record = {}; - pathParams.forEach((p) => { - initial[p.name] = p.example; - }); - return initial; - }); - - const resolvedEndpoint = endpoint.replace(/:(\w+)/g, (_, name) => pathValues[name] || `:${name}`); - - const handleSubmit = async () => { - setLoading(true); - setError(null); - setResponse(null); - - try { - const url = `${baseUrl}${resolvedEndpoint}`; - const options: RequestInit = { - method, - headers: { - "Content-Type": "application/json", - }, - }; - - if (body && (method === "POST" || method === "PUT")) { - options.body = body; - } - - const res = await fetch(url, options); - const data = await res.json().catch(() => null); - - setResponse({ - status: res.status, - data, - }); - } catch (err) { - setError(err instanceof Error ? err.message : "Request failed"); - } finally { - setLoading(false); - } - }; - - const methodColors: Record = { - GET: "bg-blue-100 text-blue-700", - POST: "bg-green-100 text-green-700", - PUT: "bg-amber-100 text-amber-700", - DELETE: "bg-red-100 text-red-700", - }; - - return ( -
- {/* Header */} -
-
- - {method} - - {endpoint} -
-

{description}

-
- - {/* Path parameters */} - {pathParams.length > 0 && ( -
-

- Path parameters -

-
- {pathParams.map((param) => ( -
- - - setPathValues((prev) => ({ ...prev, [param.name]: e.target.value })) - } - className="w-full px-3 py-2 text-sm border border-[var(--color-border)] rounded-lg font-mono focus:outline-none focus:ring-2 focus:ring-[var(--color-pe-green)]" - /> -
- ))} -
-
- )} - - {/* Request body */} - {(method === "POST" || method === "PUT") && ( -
-

- Request body -

-