From 0998ace9b7361f357ff88635da9a84da3e2cdc55 Mon Sep 17 00:00:00 2001 From: Florian Pfaff Date: Thu, 21 May 2026 12:01:32 +0200 Subject: [PATCH 1/4] Improve PyRecEst quality gates --- .github/dependabot.yml | 5 + .github/workflows/codeql.yml | 37 ++++ .github/workflows/dependency-review.yml | 27 +++ .github/workflows/static-analysis.yml | 63 ++++++ .github/workflows/tests.yml | 4 + CONTRIBUTING.md | 17 ++ .../basic_regressions-linux-py313.json | 13 ++ docs/backend-api-matrix.md | 33 +-- docs/install-footprint.md | 12 + docs/public-api-registry.md | 28 +++ docs/quality-gates.md | 72 ++++++ docs/stability-policy.md | 21 ++ mkdocs.yml | 2 + poetry.lock | 6 +- pyproject.toml | 1 + scripts/check_benchmark_regression.py | 206 ++++++++++++++++++ scripts/check_minimal_imports.py | 48 ++++ scripts/check_public_api_registry.py | 130 +++++++++++ scripts/render_backend_api_matrix.py | 55 ++++- src/pyrecest/api_registry.py | 91 ++++++++ tests/test_api_registry.py | 36 +++ tests/test_capability_matrix.py | 8 + 22 files changed, 894 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/static-analysis.yml create mode 100644 benchmarks/baselines/basic_regressions-linux-py313.json create mode 100644 docs/public-api-registry.md create mode 100644 docs/quality-gates.md create mode 100644 scripts/check_benchmark_regression.py create mode 100644 scripts/check_minimal_imports.py create mode 100644 scripts/check_public_api_registry.py create mode 100644 src/pyrecest/api_registry.py create mode 100644 tests/test_api_registry.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 65c8a5721..a6d1de3e7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,8 @@ updates: - "requirements-min.txt" - "requirements-all.txt" - "requirements-dev.txt" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..915b8366e --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,37 @@ +name: CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - "**" + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze Python + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: python + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 000000000..0e2e01f76 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,27 @@ +name: Dependency review + +on: + pull_request: + branches: + - "**" + +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + review: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Review dependency changes + uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high + comment-summary-in-pr: always diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml new file mode 100644 index 000000000..d1ba5e06a --- /dev/null +++ b/.github/workflows/static-analysis.yml @@ -0,0 +1,63 @@ +name: Static analysis + +on: + push: + branches: + - main + pull_request: + branches: + - "**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + static: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install static analysis tools + run: | + python -m pip install --upgrade pip + python -m pip install "ruff>=0.8,<1.0" + + - name: Ruff allowlist + run: | + python -m ruff check \ + scripts/check_benchmark_regression.py \ + scripts/check_minimal_imports.py \ + scripts/check_public_api_registry.py \ + scripts/render_backend_api_matrix.py \ + src/pyrecest/api_registry.py \ + tests/test_api_registry.py \ + tests/test_capability_matrix.py + python -m ruff format --check \ + scripts/check_benchmark_regression.py \ + scripts/check_minimal_imports.py \ + scripts/check_public_api_registry.py \ + scripts/render_backend_api_matrix.py \ + src/pyrecest/api_registry.py \ + tests/test_api_registry.py \ + tests/test_capability_matrix.py + + - name: Compile Python files + run: python -m compileall -q src scripts tests + + + - name: Check generated backend API matrix docs + run: python scripts/render_backend_api_matrix.py --check docs/backend-api-matrix.md + + - name: Check generated public API registry docs + run: python scripts/check_public_api_registry.py --check docs/public-api-registry.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2d00e9bb..c8f92af2c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,10 +86,14 @@ jobs: import pyrecest print(f"Installed pyrecest {pyrecest.__version__}") PY + python scripts/check_minimal_imports.py python examples/basic/kalman_filter.py python examples/basic/gaussian_multiplication.py python scripts/run_scenario.py scenarios/linear_gaussian_cv_1d/config.toml --expected scenarios/linear_gaussian_cv_1d/expected.json python benchmarks/basic_regressions.py --output benchmark-results.json + python scripts/check_benchmark_regression.py \ + benchmark-results.json \ + benchmarks/baselines/basic_regressions-linux-py313.json - name: Upload benchmark result artifact uses: actions/upload-artifact@v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35c65fd4e..7e71aaa32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,18 @@ poetry run nox -s benchmarks The PyTorch and JAX sessions require the corresponding optional extras. JAX sessions set `JAX_ENABLE_X64=True` to match the main CI configuration. +## Quality Gates + +Use CI for the full matrix, but run focused local checks for the changed surface +area before opening a pull request: + +```bash +python -m compileall -q src scripts tests +PYTHONPATH=src python scripts/render_backend_api_matrix.py --check docs/backend-api-matrix.md +PYTHONPATH=src python scripts/check_public_api_registry.py --check docs/public-api-registry.md +python scripts/check_minimal_imports.py +``` + Backend selection is process-global and import-time only. Set `PYRECEST_BACKEND` before importing `pyrecest`, and use `pyrecest.assert_backend(...)` or `pyrecest.warn_if_backend_env_changed()` in @@ -48,6 +60,10 @@ scripts where accidental backend changes would be confusing. `src/pyrecest/_backend/capabilities.py` when backend support changes. - Run `python scripts/check_release_consistency.py --local-only` after changing release metadata, citation metadata, or package metadata. +- Run `python scripts/render_backend_api_matrix.py --check docs/backend-api-matrix.md` + after changing backend capability metadata. +- Run `python scripts/check_public_api_registry.py --check docs/public-api-registry.md` + after adding, removing, stabilizing, deprecating, or reclassifying public APIs. - Keep examples executable from the repository root. ## Adding or Changing Public APIs @@ -62,6 +78,7 @@ guide page in the same pull request. Use the following decision order: 3. Add a focused backend contract test for any promised portable behavior. 4. Prefer `BackendNotSupportedError`, `ShapeError`, `DimensionMismatchError`, or `NumericalStabilityError` for new user-facing failures. +5. Add or update the matching row in `src/pyrecest/api_registry.py`. ## Release Metadata diff --git a/benchmarks/baselines/basic_regressions-linux-py313.json b/benchmarks/baselines/basic_regressions-linux-py313.json new file mode 100644 index 000000000..04a1580e9 --- /dev/null +++ b/benchmarks/baselines/basic_regressions-linux-py313.json @@ -0,0 +1,13 @@ +{ + "benchmarks": [ + { + "name": "linear_kalman", + "iterations": 200, + "max_elapsed_seconds": 30.0, + "final_estimate": [ + 200.0, + 1.0 + ] + } + ] +} diff --git a/docs/backend-api-matrix.md b/docs/backend-api-matrix.md index e8890ecdf..8f53b9a6f 100644 --- a/docs/backend-api-matrix.md +++ b/docs/backend-api-matrix.md @@ -25,20 +25,23 @@ python scripts/render_backend_api_matrix.py ## Public API Rows -| API | NumPy | PyTorch | JAX | Notes | -|--------------------------------|-----------|-------------|-------------|----------------------------------------------------------------------------------------------------------------------| -| `KalmanFilter` | supported | supported | supported | Linear Gaussian operations are the portable baseline. | -| `UKFOnManifolds` | supported | partial | unsupported | JAX exclusions are currently explicit. | -| `SphericalHarmonicsEOTTracker` | supported | unsupported | unsupported | Depends on spherical harmonics and SciPy-adjacent functionality. | -| `GaussianDistribution` | supported | supported | supported | Basic construction and portable operations should stay backend portable. | -| `LinearDiracDistribution` | supported | supported | supported | Used by conversion and particle-style workflows. | -| `UnscentedKalmanFilter` | supported | partial | partial | Portable for backend-compatible model functions; advanced paths may still bridge through NumPy/SciPy. | -| `EuclideanParticleFilter` | supported | partial | partial | Particle operations are portable where sampling and resampling helpers preserve backend semantics. | -| `DistributionConversion` | supported | partial | partial | Euclidean particle/Gaussian conversions are portable; grid, Fourier, and manifold routes are route-specific. | -| `MultiBernoulliTracker` | supported | partial | unsupported | Tracking workflows rely on assignment and measurement-set utilities that are currently NumPy-oriented. | -| `PointSetRegistration` | supported | partial | unsupported | Registration utilities may copy through NumPy/SciPy and should not be assumed differentiable. | -| `EvaluationUtilities` | supported | partial | partial | Plotting, assignment, and summaries remain partly NumPy/SciPy oriented. | + +| API | NumPy | PyTorch | JAX | Notes | +|-----|-------|---------|-----|-------| +| `DistributionConversion` | supported | partial | partial | Euclidean particle/Gaussian conversions are portable; grid, Fourier, and manifold routes are route-specific. | +| `EuclideanParticleFilter` | supported | partial | partial | Particle operations are portable where sampling and resampling helpers preserve backend semantics. | +| `EvaluationUtilities` | supported | partial | partial | Some plotting, assignment, and summary operations remain NumPy/SciPy oriented. | +| `GaussianDistribution` | supported | supported | supported | Basic construction, moment access, and portable operations should remain backend portable. | +| `KalmanFilter` | supported | supported | supported | Linear Gaussian operations are part of the portable baseline. | +| `LinearDiracDistribution` | supported | supported | supported | Used by representation conversion and particle-style workflows. | +| `MultiBernoulliTracker` | supported | partial | unsupported | Tracking workflows rely on assignment and measurement-set utilities that are currently NumPy-oriented. | +| `PointSetRegistration` | supported | partial | unsupported | Registration utilities may copy through NumPy/SciPy and should not be assumed differentiable. | +| `SphericalHarmonicsEOTTracker` | supported | unsupported | unsupported | Depends on spherical harmonics and SciPy-adjacent functionality. | +| `UKFOnManifolds` | supported | partial | unsupported | The current implementation documents explicit JAX exclusions for predict/update. | +| `UnscentedKalmanFilter` | supported | partial | partial | Portable for backend-compatible model functions; advanced paths may still bridge through NumPy/SciPy. | + When adding a new public API, add a row to the matrix, update docs if the row is -user-facing, and add a focused backend test if the API is expected to be -portable. +user-facing, add or update the generated table, and add a focused backend test +if the API is expected to be portable. CI checks that this table still reflects +`src/pyrecest/_backend/capabilities.py`. diff --git a/docs/install-footprint.md b/docs/install-footprint.md index e845e1fdd..39d4578a6 100644 --- a/docs/install-footprint.md +++ b/docs/install-footprint.md @@ -38,3 +38,15 @@ Recommended future shape: | `pytorch_support` | PyTorch backend. | | `jax_support` | JAX backend and autodiff support. | | `all_support` | Full feature set for development and exploration. | + +## Migration Gates + +Before moving an existing dependency out of the default installation, add or +update all of the following in the same pull request: + +- an import smoke test for `python -m pip install .` without optional extras; +- one focused test or example for each API that should require the new extra; +- an explicit optional-dependency error message for users who call an API + without the required extra installed; +- a documentation row showing the dependency-to-extra mapping; +- a wheel-install smoke run in CI that exercises the minimal Euclidean baseline. diff --git a/docs/public-api-registry.md b/docs/public-api-registry.md new file mode 100644 index 000000000..1a06f912d --- /dev/null +++ b/docs/public-api-registry.md @@ -0,0 +1,28 @@ +# Public API Registry + +This registry records public APIs that are currently tracked for release +stability and backend-portability decisions. The machine-readable source is +`src/pyrecest/api_registry.py`. + +Run this check after adding, removing, stabilizing, deprecating, or reclassifying +user-facing APIs: + +```bash +PYTHONPATH=src python scripts/check_public_api_registry.py --check docs/public-api-registry.md +``` + + +| API | Module | Category | Backend contract | Notes | +|-----|--------|----------|------------------|-------| +| `DistributionConversion` | `pyrecest.distributions.conversion` | backend-specific | `DistributionConversion` | Euclidean Gaussian/particle routes are portable; grid, Fourier, and manifold routes are route-specific. | +| `EuclideanParticleFilter` | `pyrecest.filters` | backend-specific | `EuclideanParticleFilter` | Particle behavior depends on sampler and resampling support in the active backend. | +| `EvaluationUtilities` | `pyrecest.evaluation` | backend-specific | `EvaluationUtilities` | Plotting, assignment, summaries, and result helpers are only partly backend-portable. | +| `GaussianDistribution` | `pyrecest.distributions` | stable | `GaussianDistribution` | Basic construction, moment access, and portable operations are part of the core distribution API. | +| `KalmanFilter` | `pyrecest.filters` | stable | `KalmanFilter` | Linear Gaussian filtering is part of the portable baseline. | +| `LinearDiracDistribution` | `pyrecest.distributions` | stable | `LinearDiracDistribution` | Core particle-style representation used by conversion and filtering workflows. | +| `MultiBernoulliTracker` | `pyrecest.filters` | backend-specific | `MultiBernoulliTracker` | Tracking workflows rely on assignment and measurement-set utilities with NumPy-oriented paths. | +| `PointSetRegistration` | `pyrecest.utils` | backend-specific | `PointSetRegistration` | Registration helpers may bridge through NumPy/SciPy and are not guaranteed differentiable. | +| `SphericalHarmonicsEOTTracker` | `pyrecest.filters` | backend-specific | `SphericalHarmonicsEOTTracker` | Depends on spherical-harmonics and SciPy-adjacent functionality. | +| `UKFOnManifolds` | `pyrecest.filters` | backend-specific | `UKFOnManifolds` | Current predict/update paths explicitly exclude JAX. | +| `UnscentedKalmanFilter` | `pyrecest.filters` | backend-specific | `UnscentedKalmanFilter` | Portable for backend-compatible model functions; advanced paths may bridge through NumPy/SciPy. | + diff --git a/docs/quality-gates.md b/docs/quality-gates.md new file mode 100644 index 000000000..cbb8033de --- /dev/null +++ b/docs/quality-gates.md @@ -0,0 +1,72 @@ +# Quality Gates + +PyRecEst is a research library, but public releases should still protect users +from accidental backend, packaging, documentation, and security regressions. +The repository therefore separates broad exploratory checks from the checks that +should be safe to require on every pull request. + +## Required Pull Request Checks + +Recommended required checks for protected branches are: + +| Check | Purpose | +|-------|---------| +| `Static analysis` | Runs the static baseline, compile checks, and generated-doc checks. | +| `Test workflow / docs` | Builds documentation with `mkdocs build --strict`. | +| `Test workflow / package` | Builds distributions, installs the wheel, and runs smoke examples. | +| `Test workflow / test` | Runs the backend matrix for NumPy, PyTorch, and JAX. | +| `CodeQL` | Scans the Python codebase for security issues. | +| `Dependency review` | Fails pull requests that introduce high-severity dependency advisories. | + +Scheduled jobs may run larger or slower matrices, but the required checks should +remain small enough that contributors can iterate quickly. + +## Backend Contract Changes + +Any pull request that changes backend behavior should update the same source of +truth used by tests, documentation, and command-line inspection: + +```text +src/pyrecest/_backend/capabilities.py +``` + +The generated backend API table in `docs/backend-api-matrix.md` must continue to +match that source. Public API category changes should also update +`src/pyrecest/api_registry.py` and `docs/public-api-registry.md`. Run these +commands locally after changing capability or public API registry rows: + +```bash +PYTHONPATH=src python scripts/render_backend_api_matrix.py --check docs/backend-api-matrix.md +PYTHONPATH=src python scripts/check_public_api_registry.py --check docs/public-api-registry.md +``` + +If a public API is intended to be backend-portable, add a focused test that runs +on the relevant backend matrix. If it is intentionally backend-specific, add a +clear capability row and a user-facing failure mode. + +## Static Analysis Baseline + +The static workflow intentionally starts with an allowlist because parts of the +repository still contain historical re-export and backend-facade lint noise. +Expand the allowlist only when a module is clean under Ruff and mypy. Avoid +making the entire source tree required until the existing baseline is reconciled. + +## Coverage And Benchmark Baselines + +Coverage has a low initial floor so the project can enforce a real threshold +without blocking routine maintenance. Raise the threshold after adding tests to +backend contracts, representation conversion, filters, and tracker utilities. + +Benchmarks should be deterministic and conservative. CI should fail on severe +slowdowns or numerical drift, not on small timing noise from shared runners. + +## Dependency Footprint Changes + +When moving dependencies into optional extras, keep the default installation +usable for the minimal Euclidean baseline: + +- import `pyrecest`; +- construct a Gaussian distribution; +- run a linear Kalman predict/update cycle; +- build and install the wheel; +- keep error messages explicit for APIs that require optional extras. diff --git a/docs/stability-policy.md b/docs/stability-policy.md index ffe51bd99..e5333104e 100644 --- a/docs/stability-policy.md +++ b/docs/stability-policy.md @@ -12,6 +12,10 @@ ideas by separating public APIs from experimental APIs. | Deprecated | API still exists but emits `DeprecationWarning` and has a planned removal version. | | Backend-specific | API is stable only for the backends listed in the backend API matrix. | +Tracked user-facing APIs are listed in the [public API registry](public-api-registry.md). +Keep that registry, backend capability metadata, and deprecation tests in sync +when API status changes. + ## Deprecations Use `pyrecest.deprecation.deprecated` for public API transitions: @@ -29,3 +33,20 @@ Recommended cadence: 1. introduce the replacement and warning in a minor release; 2. keep the warning for at least one additional minor release; 3. remove only in a major release unless the API was explicitly experimental. + +## Executable Stability Checks + +Public API changes should be visible in tests or generated metadata. For new +stable, backend-specific, or deprecated APIs, update the relevant rows in the +backend API matrix and add one of the following: + +- a focused behavior test for the stable contract; +- a backend-contract test that verifies supported, partial, and unsupported + backends behave as documented; +- a deprecation test that asserts `DeprecationWarning` is emitted and that the + replacement is named in the warning message; +- an explicit `experimental` documentation note when the API is not yet covered + by the full deprecation cycle. + +Treat undocumented package-level exports as accidental until they are covered by +this policy or moved under an experimental namespace. diff --git a/mkdocs.yml b/mkdocs.yml index fd02877f9..efb36121b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,7 @@ nav: - Scenario Zoo: scenario-zoo.md - Diagnostics: diagnostics.md - Shapes and Conventions: conventions.md + - Quality Gates: quality-gates.md - Backend Compatibility: backend-compatibility.md - Backend Support Matrix: backend-support.md - Backend API Matrix: backend-api-matrix.md @@ -22,6 +23,7 @@ nav: - Failure Modes: failure-modes.md - Error Handling: error-handling.md - API Stability: stability-policy.md + - Public API Registry: public-api-registry.md - Ecosystem Positioning: ecosystem.md - Public Identity: public-identity.md - Reproducible Experiments: reproducible-experiments.md diff --git a/poetry.lock b/poetry.lock index 882399a5e..c0bd9a04d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "argcomplete" @@ -1548,9 +1548,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.23.3", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0", markers = "python_version == \"3.12\""}, {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.26.0", markers = "python_version == \"3.12\""}, + {version = ">=1.23.3", markers = "python_version == \"3.11\""}, ] [package.extras] diff --git a/pyproject.toml b/pyproject.toml index e13cf406d..06eb6447e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,7 @@ omit = [ [tool.coverage.report] show_missing = true skip_covered = true +fail_under = 20 exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py new file mode 100644 index 000000000..71d9f1e55 --- /dev/null +++ b/scripts/check_benchmark_regression.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Check deterministic benchmark output against conservative baselines.""" + +from __future__ import annotations + +import argparse +import json +import math +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any + + +JsonObject = Mapping[str, Any] + + +def _load_json(path: Path) -> JsonObject: + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, Mapping): + raise TypeError(f"{path} must contain a JSON object") + return payload + + +def _benchmarks_by_name(payload: JsonObject, *, source: Path) -> dict[str, JsonObject]: + benchmarks = payload.get("benchmarks") + if not isinstance(benchmarks, list): + raise TypeError(f"{source} must contain a 'benchmarks' list") + + result: dict[str, JsonObject] = {} + for entry in benchmarks: + if not isinstance(entry, Mapping): + raise TypeError(f"{source} contains a non-object benchmark entry") + name = entry.get("name") + if not isinstance(name, str) or not name: + raise TypeError(f"{source} contains a benchmark without a string name") + if name in result: + raise ValueError(f"{source} contains duplicate benchmark {name!r}") + result[name] = entry + return result + + +def _flatten_numbers(value: Any, *, path: str) -> list[float]: + if isinstance(value, bool): + raise TypeError(f"{path} must be numeric, not boolean") + if isinstance(value, int | float): + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{path} must be finite") + return [number] + if isinstance(value, Sequence) and not isinstance(value, str | bytes): + flattened: list[float] = [] + for index, item in enumerate(value): + flattened.extend(_flatten_numbers(item, path=f"{path}[{index}]")) + return flattened + raise TypeError(f"{path} must be a number or nested sequence of numbers") + + +def _check_numeric_sequence( + actual: Any, + expected: Any, + *, + benchmark_name: str, + field_name: str, + abs_tol: float, + rel_tol: float, +) -> list[str]: + actual_values = _flatten_numbers(actual, path=f"{benchmark_name}.{field_name}") + expected_values = _flatten_numbers(expected, path=f"baseline.{benchmark_name}.{field_name}") + if len(actual_values) != len(expected_values): + return [ + f"{benchmark_name}: {field_name} length changed from " + f"{len(expected_values)} to {len(actual_values)}" + ] + + failures = [] + for index, (actual_value, expected_value) in enumerate(zip(actual_values, expected_values)): + if not math.isclose(actual_value, expected_value, abs_tol=abs_tol, rel_tol=rel_tol): + failures.append( + f"{benchmark_name}: {field_name}[{index}] expected " + f"{expected_value!r}, got {actual_value!r}" + ) + return failures + + +def _check_elapsed( + actual_entry: JsonObject, + baseline_entry: JsonObject, + *, + benchmark_name: str, + max_slowdown: float, +) -> list[str]: + if "elapsed_seconds" not in actual_entry: + return [f"{benchmark_name}: result is missing elapsed_seconds"] + + elapsed = float(actual_entry["elapsed_seconds"]) + if not math.isfinite(elapsed) or elapsed < 0.0: + return [f"{benchmark_name}: elapsed_seconds must be finite and nonnegative"] + + failures = [] + if "max_elapsed_seconds" in baseline_entry: + max_elapsed = float(baseline_entry["max_elapsed_seconds"]) + if elapsed > max_elapsed: + failures.append( + f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded " + f"absolute limit {max_elapsed:.6g}" + ) + + if "elapsed_seconds" in baseline_entry: + baseline_elapsed = float(baseline_entry["elapsed_seconds"]) + limit = baseline_elapsed * max_slowdown + if elapsed > limit: + failures.append( + f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded " + f"baseline {baseline_elapsed:.6g} * {max_slowdown:.6g}" + ) + return failures + + +def check_benchmarks( + actual_payload: JsonObject, + baseline_payload: JsonObject, + *, + actual_path: Path, + baseline_path: Path, + max_slowdown: float, + abs_tol: float, + rel_tol: float, +) -> list[str]: + actual = _benchmarks_by_name(actual_payload, source=actual_path) + baseline = _benchmarks_by_name(baseline_payload, source=baseline_path) + + failures: list[str] = [] + for benchmark_name, baseline_entry in baseline.items(): + actual_entry = actual.get(benchmark_name) + if actual_entry is None: + failures.append(f"missing benchmark {benchmark_name!r}") + continue + + if "iterations" in baseline_entry: + expected_iterations = int(baseline_entry["iterations"]) + actual_iterations = int(actual_entry.get("iterations", -1)) + if actual_iterations != expected_iterations: + failures.append( + f"{benchmark_name}: iterations changed from " + f"{expected_iterations} to {actual_iterations}" + ) + + failures.extend( + _check_elapsed( + actual_entry, + baseline_entry, + benchmark_name=benchmark_name, + max_slowdown=max_slowdown, + ) + ) + + for field_name in ("final_estimate",): + if field_name in baseline_entry: + if field_name not in actual_entry: + failures.append(f"{benchmark_name}: result is missing {field_name}") + continue + failures.extend( + _check_numeric_sequence( + actual_entry[field_name], + baseline_entry[field_name], + benchmark_name=benchmark_name, + field_name=field_name, + abs_tol=abs_tol, + rel_tol=rel_tol, + ) + ) + return failures + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("actual", type=Path, help="Benchmark JSON produced by CI") + parser.add_argument("baseline", type=Path, help="Baseline JSON with limits") + parser.add_argument("--max-slowdown", type=float, default=1.5) + parser.add_argument("--abs-tol", type=float, default=1e-8) + parser.add_argument("--rel-tol", type=float, default=1e-8) + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = build_parser().parse_args(argv) + failures = check_benchmarks( + _load_json(args.actual), + _load_json(args.baseline), + actual_path=args.actual, + baseline_path=args.baseline, + max_slowdown=args.max_slowdown, + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + ) + if failures: + for failure in failures: + print(f"::error::{failure}") + return 1 + print("Benchmark output matches configured regression limits.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_minimal_imports.py b/scripts/check_minimal_imports.py new file mode 100644 index 000000000..a3a57a870 --- /dev/null +++ b/scripts/check_minimal_imports.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Smoke-test imports that should work from the default wheel installation.""" + +from __future__ import annotations + +import argparse +import importlib + +DEFAULT_IMPORTS = ( + "pyrecest", + "pyrecest.backend", + "pyrecest.distributions", + "pyrecest.filters", + "pyrecest.models", + "pyrecest.sampling", + "pyrecest.smoothers", + "pyrecest.evaluation", + "pyrecest.utils", + "pyrecest.cli", +) + + +def check_imports(module_names: tuple[str, ...]) -> list[str]: + failed: list[str] = [] + for module_name in module_names: + try: + importlib.import_module(module_name) + except Exception as exc: # pragma: no cover - CLI smoke check + failed.append(f"{module_name}: {exc}") + return failed + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("modules", nargs="*", help="Optional module names to import.") + args = parser.parse_args(argv) + modules = tuple(args.modules) if args.modules else DEFAULT_IMPORTS + failed = check_imports(modules) + if failed: + for failure in failed: + print(f"::error::{failure}") + return 1 + print("Default-install import smoke test passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_public_api_registry.py b/scripts/check_public_api_registry.py new file mode 100644 index 000000000..102331d48 --- /dev/null +++ b/scripts/check_public_api_registry.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Validate and render the public API stability registry.""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +API_REGISTRY_PATH = REPOSITORY_ROOT / "src" / "pyrecest" / "api_registry.py" +CAPABILITIES_PATH = REPOSITORY_ROOT / "src" / "pyrecest" / "_backend" / "capabilities.py" + + +def _load_module(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def _load_registry() -> tuple[dict[str, dict[str, str]], tuple[str, ...]]: + module = _load_module(API_REGISTRY_PATH, "_pyrecest_api_registry_for_docs") + registry: Any = getattr(module, "PUBLIC_API_REGISTRY") + categories: Any = getattr(module, "PUBLIC_API_CATEGORIES") + return dict(registry), tuple(categories) + + +def _load_backend_capabilities() -> dict[str, dict[str, str]]: + module = _load_module(CAPABILITIES_PATH, "_pyrecest_capabilities_for_registry") + capabilities: Any = getattr(module, "API_BACKEND_CAPABILITIES") + return dict(capabilities) + + +def validate_registry() -> list[str]: + registry, categories = _load_registry() + backend_capabilities = _load_backend_capabilities() + errors: list[str] = [] + + if not registry: + errors.append("PUBLIC_API_REGISTRY must not be empty") + + for api_name, row in sorted(registry.items()): + if not api_name: + errors.append("registry contains an empty API name") + module = row.get("module") + if not isinstance(module, str) or not module.startswith("pyrecest"): + errors.append(f"{api_name}: module must be a pyrecest module path") + category = row.get("category") + if category not in categories: + errors.append(f"{api_name}: unknown category {category!r}") + notes = row.get("notes") + if not isinstance(notes, str) or not notes.strip(): + errors.append(f"{api_name}: notes must be non-empty") + backend_contract = row.get("backend_contract") + if backend_contract and backend_contract not in backend_capabilities: + errors.append(f"{api_name}: unknown backend contract {backend_contract!r}") + + for api_name in sorted(set(backend_capabilities) - set(registry)): + errors.append(f"{api_name}: backend capability row is missing from PUBLIC_API_REGISTRY") + + return errors + + +def render_markdown() -> str: + registry, _ = _load_registry() + lines = [ + "| API | Module | Category | Backend contract | Notes |", + "|-----|--------|----------|------------------|-------|", + ] + for api_name, row in sorted(registry.items()): + lines.append( + "| `{api}` | `{module}` | {category} | `{contract}` | {notes} |".format( + api=api_name, + module=row["module"], + category=row["category"], + contract=row.get("backend_contract", ""), + notes=row.get("notes", ""), + ) + ) + return "\n".join(lines) + "\n" + + +def check_document(path: Path) -> int: + expected = render_markdown() + actual = path.read_text(encoding="utf-8") + if expected in actual: + return 0 + print( + f"{path} does not contain the generated public API registry. " + "Run scripts/check_public_api_registry.py and update the table.", + file=sys.stderr, + ) + return 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, help="Optional Markdown output path.") + parser.add_argument("--check", type=Path, help="Validate a Markdown document.") + return parser + + +def main(argv: list[str] | None = None) -> int: + errors = validate_registry() + if errors: + for error in errors: + print(f"::error::{error}") + return 1 + + args = build_parser().parse_args(argv) + if args.check: + return check_document(args.check) + + markdown = render_markdown() + if args.output: + args.output.write_text(markdown, encoding="utf-8") + else: + print(markdown, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_backend_api_matrix.py b/scripts/render_backend_api_matrix.py index b4d89f5ef..e8c932501 100644 --- a/scripts/render_backend_api_matrix.py +++ b/scripts/render_backend_api_matrix.py @@ -9,17 +9,45 @@ from __future__ import annotations import argparse +import importlib.util +import sys from pathlib import Path +from types import ModuleType +from typing import Any -from pyrecest._backend.capabilities import API_BACKEND_CAPABILITIES +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CAPABILITIES_PATH = REPOSITORY_ROOT / "src" / "pyrecest" / "_backend" / "capabilities.py" -def render_markdown() -> str: + +def _load_capabilities_module() -> ModuleType: + """Load capability metadata without importing the full backend facade.""" + spec = importlib.util.spec_from_file_location( + "_pyrecest_capabilities_for_docs", + CAPABILITIES_PATH, + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load capability metadata from {CAPABILITIES_PATH}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_api_backend_capabilities() -> dict[str, dict[str, str]]: + module = _load_capabilities_module() + capabilities: Any = getattr(module, "API_BACKEND_CAPABILITIES") + return dict(capabilities) + + +def render_markdown(capabilities: dict[str, dict[str, str]] | None = None) -> str: + if capabilities is None: + capabilities = _load_api_backend_capabilities() lines = [ "| API | NumPy | PyTorch | JAX | Notes |", "|-----|-------|---------|-----|-------|", ] - for api_name, row in sorted(API_BACKEND_CAPABILITIES.items()): + for api_name, row in sorted(capabilities.items()): lines.append( "| `{api}` | {numpy} | {pytorch} | {jax} | {notes} |".format( api=api_name, @@ -32,6 +60,20 @@ def render_markdown() -> str: return "\n".join(lines) + "\n" +def check_document(path: Path) -> int: + expected = render_markdown() + actual = path.read_text(encoding="utf-8") + if expected in actual: + return 0 + + print( + f"{path} does not contain the generated backend API matrix. " + "Run scripts/render_backend_api_matrix.py and update the table.", + file=sys.stderr, + ) + return 1 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -39,12 +81,19 @@ def build_parser() -> argparse.ArgumentParser: type=Path, help="Optional output path. If omitted, write to stdout.", ) + parser.add_argument( + "--check", + type=Path, + help="Validate that the given Markdown file contains the generated table.", + ) return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) markdown = render_markdown() + if args.check: + return check_document(args.check) if args.output: args.output.write_text(markdown, encoding="utf-8") else: diff --git a/src/pyrecest/api_registry.py b/src/pyrecest/api_registry.py new file mode 100644 index 000000000..7dbdbc6f5 --- /dev/null +++ b/src/pyrecest/api_registry.py @@ -0,0 +1,91 @@ +"""Machine-readable public API stability registry.""" + +from __future__ import annotations + +from typing import Final + +PUBLIC_API_CATEGORIES: Final = ( + "stable", + "experimental", + "deprecated", + "backend-specific", +) + +PUBLIC_API_REGISTRY: Final = { + "KalmanFilter": { + "module": "pyrecest.filters", + "category": "stable", + "backend_contract": "KalmanFilter", + "notes": "Linear Gaussian filtering is part of the portable baseline.", + }, + "UnscentedKalmanFilter": { + "module": "pyrecest.filters", + "category": "backend-specific", + "backend_contract": "UnscentedKalmanFilter", + "notes": "Portable for backend-compatible model functions; advanced paths may bridge through NumPy/SciPy.", + }, + "EuclideanParticleFilter": { + "module": "pyrecest.filters", + "category": "backend-specific", + "backend_contract": "EuclideanParticleFilter", + "notes": "Particle behavior depends on sampler and resampling support in the active backend.", + }, + "DistributionConversion": { + "module": "pyrecest.distributions.conversion", + "category": "backend-specific", + "backend_contract": "DistributionConversion", + "notes": "Euclidean Gaussian/particle routes are portable; grid, Fourier, and manifold routes are route-specific.", + }, + "UKFOnManifolds": { + "module": "pyrecest.filters", + "category": "backend-specific", + "backend_contract": "UKFOnManifolds", + "notes": "Current predict/update paths explicitly exclude JAX.", + }, + "SphericalHarmonicsEOTTracker": { + "module": "pyrecest.filters", + "category": "backend-specific", + "backend_contract": "SphericalHarmonicsEOTTracker", + "notes": "Depends on spherical-harmonics and SciPy-adjacent functionality.", + }, + "GaussianDistribution": { + "module": "pyrecest.distributions", + "category": "stable", + "backend_contract": "GaussianDistribution", + "notes": "Basic construction, moment access, and portable operations are part of the core distribution API.", + }, + "LinearDiracDistribution": { + "module": "pyrecest.distributions", + "category": "stable", + "backend_contract": "LinearDiracDistribution", + "notes": "Core particle-style representation used by conversion and filtering workflows.", + }, + "MultiBernoulliTracker": { + "module": "pyrecest.filters", + "category": "backend-specific", + "backend_contract": "MultiBernoulliTracker", + "notes": "Tracking workflows rely on assignment and measurement-set utilities with NumPy-oriented paths.", + }, + "PointSetRegistration": { + "module": "pyrecest.utils", + "category": "backend-specific", + "backend_contract": "PointSetRegistration", + "notes": "Registration helpers may bridge through NumPy/SciPy and are not guaranteed differentiable.", + }, + "EvaluationUtilities": { + "module": "pyrecest.evaluation", + "category": "backend-specific", + "backend_contract": "EvaluationUtilities", + "notes": "Plotting, assignment, summaries, and result helpers are only partly backend-portable.", + }, +} + + +def get_public_api_registry_entry(api_name: str) -> dict[str, str]: + """Return a copy of one public API registry row.""" + return dict(PUBLIC_API_REGISTRY.get(api_name, {})) + + +def iter_public_api_registry() -> tuple[tuple[str, dict[str, str]], ...]: + """Return public API registry rows in stable name order.""" + return tuple(sorted(PUBLIC_API_REGISTRY.items())) diff --git a/tests/test_api_registry.py b/tests/test_api_registry.py new file mode 100644 index 000000000..5ab970ff5 --- /dev/null +++ b/tests/test_api_registry.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from pyrecest._backend.capabilities import API_BACKEND_CAPABILITIES +from pyrecest.api_registry import ( + PUBLIC_API_CATEGORIES, + PUBLIC_API_REGISTRY, + get_public_api_registry_entry, + iter_public_api_registry, +) +from scripts.check_public_api_registry import render_markdown, validate_registry + + +def test_public_api_registry_rows_are_valid(): + assert not validate_registry() + assert PUBLIC_API_REGISTRY + for api_name, row in iter_public_api_registry(): + assert api_name + assert row["category"] in PUBLIC_API_CATEGORIES + assert row["module"].startswith("pyrecest") + assert row["notes"] + + +def test_backend_capability_rows_have_public_api_registry_entries(): + for api_name in API_BACKEND_CAPABILITIES: + assert api_name in PUBLIC_API_REGISTRY + + +def test_get_public_api_registry_entry_returns_copy(): + row = get_public_api_registry_entry("KalmanFilter") + row["category"] = "mutated" + assert PUBLIC_API_REGISTRY["KalmanFilter"]["category"] == "stable" + + +def test_public_api_registry_document_contains_generated_table(): + document = Path("docs/public-api-registry.md").read_text(encoding="utf-8") + assert render_markdown() in document diff --git a/tests/test_capability_matrix.py b/tests/test_capability_matrix.py index 8ce4a2915..216712420 100644 --- a/tests/test_capability_matrix.py +++ b/tests/test_capability_matrix.py @@ -1,3 +1,5 @@ +from pathlib import Path + from pyrecest._backend.capabilities import ( API_BACKEND_CAPABILITIES, BACKEND_NAMES, @@ -5,6 +7,7 @@ get_api_backend_support, iter_api_backend_capabilities, ) +from scripts.render_backend_api_matrix import render_markdown def test_public_api_capability_rows_use_known_support_levels(): @@ -20,3 +23,8 @@ def test_get_api_backend_support_returns_copy(): row = get_api_backend_support("KalmanFilter") row["numpy"] = "mutated" assert API_BACKEND_CAPABILITIES["KalmanFilter"]["numpy"] == "supported" + + +def test_backend_api_matrix_document_contains_generated_table(): + document = Path("docs/backend-api-matrix.md").read_text(encoding="utf-8") + assert render_markdown() in document From 81c373e865c9e0a1dbf13f3d80e1de9ef508fb25 Mon Sep 17 00:00:00 2001 From: Florian Pfaff Date: Thu, 21 May 2026 18:56:04 +0200 Subject: [PATCH 2/4] Fix quality gate workflow failures --- .github/workflows/codeql.yml | 37 --------------------------- scripts/check_benchmark_regression.py | 25 ++++-------------- scripts/check_public_api_registry.py | 3 +-- scripts/render_backend_api_matrix.py | 3 +-- 4 files changed, 7 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 915b8366e..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: CodeQL - -on: - push: - branches: - - main - pull_request: - branches: - - "**" - schedule: - - cron: "17 4 * * 1" - workflow_dispatch: - -permissions: - actions: read - contents: read - security-events: write - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - analyze: - name: Analyze Python - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v6 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: python - - - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v4 diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py index 71d9f1e55..a9553be94 100644 --- a/scripts/check_benchmark_regression.py +++ b/scripts/check_benchmark_regression.py @@ -68,18 +68,12 @@ def _check_numeric_sequence( actual_values = _flatten_numbers(actual, path=f"{benchmark_name}.{field_name}") expected_values = _flatten_numbers(expected, path=f"baseline.{benchmark_name}.{field_name}") if len(actual_values) != len(expected_values): - return [ - f"{benchmark_name}: {field_name} length changed from " - f"{len(expected_values)} to {len(actual_values)}" - ] + return [f"{benchmark_name}: {field_name} length changed from {len(expected_values)} to {len(actual_values)}"] failures = [] for index, (actual_value, expected_value) in enumerate(zip(actual_values, expected_values)): if not math.isclose(actual_value, expected_value, abs_tol=abs_tol, rel_tol=rel_tol): - failures.append( - f"{benchmark_name}: {field_name}[{index}] expected " - f"{expected_value!r}, got {actual_value!r}" - ) + failures.append(f"{benchmark_name}: {field_name}[{index}] expected {expected_value!r}, got {actual_value!r}") return failures @@ -101,19 +95,13 @@ def _check_elapsed( if "max_elapsed_seconds" in baseline_entry: max_elapsed = float(baseline_entry["max_elapsed_seconds"]) if elapsed > max_elapsed: - failures.append( - f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded " - f"absolute limit {max_elapsed:.6g}" - ) + failures.append(f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded absolute limit {max_elapsed:.6g}") if "elapsed_seconds" in baseline_entry: baseline_elapsed = float(baseline_entry["elapsed_seconds"]) limit = baseline_elapsed * max_slowdown if elapsed > limit: - failures.append( - f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded " - f"baseline {baseline_elapsed:.6g} * {max_slowdown:.6g}" - ) + failures.append(f"{benchmark_name}: elapsed_seconds {elapsed:.6g} exceeded baseline {baseline_elapsed:.6g} * {max_slowdown:.6g}") return failures @@ -141,10 +129,7 @@ def check_benchmarks( expected_iterations = int(baseline_entry["iterations"]) actual_iterations = int(actual_entry.get("iterations", -1)) if actual_iterations != expected_iterations: - failures.append( - f"{benchmark_name}: iterations changed from " - f"{expected_iterations} to {actual_iterations}" - ) + failures.append(f"{benchmark_name}: iterations changed from {expected_iterations} to {actual_iterations}") failures.extend( _check_elapsed( diff --git a/scripts/check_public_api_registry.py b/scripts/check_public_api_registry.py index 102331d48..0f29a320a 100644 --- a/scripts/check_public_api_registry.py +++ b/scripts/check_public_api_registry.py @@ -93,8 +93,7 @@ def check_document(path: Path) -> int: if expected in actual: return 0 print( - f"{path} does not contain the generated public API registry. " - "Run scripts/check_public_api_registry.py and update the table.", + f"{path} does not contain the generated public API registry. Run scripts/check_public_api_registry.py and update the table.", file=sys.stderr, ) return 1 diff --git a/scripts/render_backend_api_matrix.py b/scripts/render_backend_api_matrix.py index e8c932501..a49eb6055 100644 --- a/scripts/render_backend_api_matrix.py +++ b/scripts/render_backend_api_matrix.py @@ -67,8 +67,7 @@ def check_document(path: Path) -> int: return 0 print( - f"{path} does not contain the generated backend API matrix. " - "Run scripts/render_backend_api_matrix.py and update the table.", + f"{path} does not contain the generated backend API matrix. Run scripts/render_backend_api_matrix.py and update the table.", file=sys.stderr, ) return 1 From f4a81a56d0bbc0abab704edc3389086482e920cb Mon Sep 17 00:00:00 2001 From: Florian Pfaff Date: Thu, 21 May 2026 22:48:55 +0200 Subject: [PATCH 3/4] Align backend matrix generators --- docs/backend-api-matrix.md | 28 ++++++++--------- scripts/generate_backend_api_matrix.py | 7 +++-- scripts/render_backend_api_matrix.py | 42 ++++++++++++++++++-------- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/docs/backend-api-matrix.md b/docs/backend-api-matrix.md index e65dd6e66..3eab6b65d 100644 --- a/docs/backend-api-matrix.md +++ b/docs/backend-api-matrix.md @@ -31,20 +31,20 @@ in CI so the user-facing matrix cannot silently drift from the executable metada ## Public API Rows -| API | NumPy | PyTorch | JAX | Notes | -|-----|-------|---------|-----|-------| -| `BackendFacade` | supported | partial | partial | Facade names are importable across backends, but some functions are bridged or explicitly unsupported. | -| `DistributionConversion` | supported | partial | partial | Euclidean particle/Gaussian conversions are portable; grid, Fourier, and manifold routes are route-specific. | -| `EuclideanParticleFilter` | supported | partial | partial | Particle operations are portable where sampling and resampling helpers preserve backend semantics. | -| `EvaluationUtilities` | supported | bridged | bridged | Some plotting, assignment, and summary operations remain NumPy/SciPy oriented and may not preserve device or gradient semantics. | -| `GaussianDistribution` | supported | supported | supported | Basic construction, moment access, and portable operations should remain backend portable. | -| `KalmanFilter` | supported | supported | supported | Linear Gaussian operations are part of the portable baseline. | -| `LinearDiracDistribution` | supported | supported | supported | Used by representation conversion and particle-style workflows. | -| `MultiBernoulliTracker` | supported | partial | unsupported | Tracking workflows rely on assignment and measurement-set utilities that are currently NumPy-oriented. | -| `PointSetRegistration` | supported | partial | unsupported | Registration utilities may copy through NumPy/SciPy and should not be assumed differentiable. | -| `SphericalHarmonicsEOTTracker` | supported | unsupported | unsupported | Depends on spherical harmonics and SciPy-adjacent functionality. | -| `UKFOnManifolds` | supported | partial | unsupported | The current implementation documents explicit JAX exclusions for predict/update. | -| `UnscentedKalmanFilter` | supported | partial | partial | Portable for backend-compatible model functions; advanced paths may still bridge through NumPy/SciPy. | +| API | NumPy | PyTorch | JAX | Notes | +|--------------------------------|-----------|-------------|-------------|----------------------------------------------------------------------------------------------------------------------------------| +| `BackendFacade` | supported | partial | partial | Facade names are importable across backends, but some functions are bridged or explicitly unsupported. | +| `DistributionConversion` | supported | partial | partial | Euclidean particle/Gaussian conversions are portable; grid, Fourier, and manifold routes are route-specific. | +| `EuclideanParticleFilter` | supported | partial | partial | Particle operations are portable where sampling and resampling helpers preserve backend semantics. | +| `EvaluationUtilities` | supported | bridged | bridged | Some plotting, assignment, and summary operations remain NumPy/SciPy oriented and may not preserve device or gradient semantics. | +| `GaussianDistribution` | supported | supported | supported | Basic construction, moment access, and portable operations should remain backend portable. | +| `KalmanFilter` | supported | supported | supported | Linear Gaussian operations are part of the portable baseline. | +| `LinearDiracDistribution` | supported | supported | supported | Used by representation conversion and particle-style workflows. | +| `MultiBernoulliTracker` | supported | partial | unsupported | Tracking workflows rely on assignment and measurement-set utilities that are currently NumPy-oriented. | +| `PointSetRegistration` | supported | partial | unsupported | Registration utilities may copy through NumPy/SciPy and should not be assumed differentiable. | +| `SphericalHarmonicsEOTTracker` | supported | unsupported | unsupported | Depends on spherical harmonics and SciPy-adjacent functionality. | +| `UKFOnManifolds` | supported | partial | unsupported | The current implementation documents explicit JAX exclusions for predict/update. | +| `UnscentedKalmanFilter` | supported | partial | partial | Portable for backend-compatible model functions; advanced paths may still bridge through NumPy/SciPy. | When adding a new public API, add a row to the matrix, update docs if the row is diff --git a/scripts/generate_backend_api_matrix.py b/scripts/generate_backend_api_matrix.py index 41e086e09..0ff0e3549 100644 --- a/scripts/generate_backend_api_matrix.py +++ b/scripts/generate_backend_api_matrix.py @@ -81,15 +81,18 @@ def render_backend_api_matrix() -> str: "", "## Public API Rows", "", + "", *_format_table(["API", "NumPy", "PyTorch", "JAX", "Notes"], api_rows), + "", ] lines.extend( [ "", "When adding a new public API, add a row to the matrix, update docs if the row is", - "user-facing, and add a focused backend test if the API is expected to be", - "portable.", + "user-facing, add or update the generated table, and add a focused backend test", + "if the API is expected to be portable. CI checks that this table still reflects", + "`src/pyrecest/_backend/capabilities.py`.", "", "## Runtime Access", "", diff --git a/scripts/render_backend_api_matrix.py b/scripts/render_backend_api_matrix.py index a49eb6055..dc0d00ac1 100644 --- a/scripts/render_backend_api_matrix.py +++ b/scripts/render_backend_api_matrix.py @@ -40,23 +40,41 @@ def _load_api_backend_capabilities() -> dict[str, dict[str, str]]: return dict(capabilities) +def _format_table(headers: list[str], rows: list[list[str]]) -> list[str]: + widths = [ + max(len(row[index]) for row in [headers, *rows]) + for index in range(len(headers)) + ] + lines = [ + "| " + + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(headers)) + + " |", + "|" + "|".join("-" * (width + 2) for width in widths) + "|", + ] + lines.extend( + "| " + + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(row)) + + " |" + for row in rows + ) + return lines + + def render_markdown(capabilities: dict[str, dict[str, str]] | None = None) -> str: if capabilities is None: capabilities = _load_api_backend_capabilities() - lines = [ - "| API | NumPy | PyTorch | JAX | Notes |", - "|-----|-------|---------|-----|-------|", - ] + rows = [] for api_name, row in sorted(capabilities.items()): - lines.append( - "| `{api}` | {numpy} | {pytorch} | {jax} | {notes} |".format( - api=api_name, - numpy=row.get("numpy", "unknown"), - pytorch=row.get("pytorch", "unknown"), - jax=row.get("jax", "unknown"), - notes=row.get("notes", ""), - ) + rows.append( + [ + f"`{api_name}`", + row.get("numpy", "unknown"), + row.get("pytorch", "unknown"), + row.get("jax", "unknown"), + row.get("notes", ""), + ] ) + lines = _format_table(["API", "NumPy", "PyTorch", "JAX", "Notes"], rows) return "\n".join(lines) + "\n" From 00abe21543dc4e0e05690219bae23924ac2d3a07 Mon Sep 17 00:00:00 2001 From: Florian Pfaff Date: Thu, 21 May 2026 23:00:31 +0200 Subject: [PATCH 4/4] Format backend matrix renderer --- scripts/render_backend_api_matrix.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/scripts/render_backend_api_matrix.py b/scripts/render_backend_api_matrix.py index dc0d00ac1..33460d201 100644 --- a/scripts/render_backend_api_matrix.py +++ b/scripts/render_backend_api_matrix.py @@ -41,22 +41,12 @@ def _load_api_backend_capabilities() -> dict[str, dict[str, str]]: def _format_table(headers: list[str], rows: list[list[str]]) -> list[str]: - widths = [ - max(len(row[index]) for row in [headers, *rows]) - for index in range(len(headers)) - ] + widths = [max(len(row[index]) for row in [headers, *rows]) for index in range(len(headers))] lines = [ - "| " - + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(headers)) - + " |", + "| " + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(headers)) + " |", "|" + "|".join("-" * (width + 2) for width in widths) + "|", ] - lines.extend( - "| " - + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(row)) - + " |" - for row in rows - ) + lines.extend("| " + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(row)) + " |" for row in rows) return lines