diff --git a/.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json b/.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json new file mode 100644 index 0000000..48e0393 --- /dev/null +++ b/.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json @@ -0,0 +1,380 @@ +{ + "machine_info": { + "node": "runnervm7b5n9", + "processor": "x86_64", + "machine": "x86_64", + "python_compiler": "GCC 13.3.0", + "python_implementation": "CPython", + "python_implementation_version": "3.14.6", + "python_version": "3.14.6", + "python_build": [ + "main", + "Jun 10 2026 14:29:35" + ], + "release": "6.17.0-1018-azure", + "system": "Linux", + "cpu": { + "python_version": "3.14.6.final.0 (64 bit)", + "cpuinfo_version": [ + 9, + 0, + 0 + ], + "cpuinfo_version_string": "9.0.0", + "arch": "X86_64", + "bits": 64, + "count": 4, + "arch_string_raw": "x86_64", + "vendor_id_raw": "AuthenticAMD", + "brand_raw": "AMD EPYC 7763 64-Core Processor", + "hz_advertised_friendly": "3.2406 GHz", + "hz_actual_friendly": "3.2406 GHz", + "hz_advertised": [ + 3240557000, + 0 + ], + "hz_actual": [ + 3240557000, + 0 + ], + "stepping": 1, + "model": 1, + "family": 25, + "flags": [ + "3dnowext", + "3dnowprefetch", + "abm", + "adx", + "aes", + "aperfmperf", + "apic", + "arat", + "avx", + "avx2", + "bmi1", + "bmi2", + "clflush", + "clflushopt", + "clwb", + "clzero", + "cmov", + "cmp_legacy", + "constant_tsc", + "cpuid", + "cr8_legacy", + "cx16", + "cx8", + "de", + "decodeassists", + "erms", + "extd_apicid", + "f16c", + "flushbyasid", + "fma", + "fpu", + "fsgsbase", + "fsrm", + "fxsr", + "fxsr_opt", + "ht", + "hypervisor", + "invpcid", + "lahf_lm", + "lm", + "mca", + "mce", + "misalignsse", + "mmx", + "mmxext", + "movbe", + "msr", + "mtrr", + "nonstop_tsc", + "nopl", + "npt", + "nrip_save", + "nx", + "osvw", + "osxsave", + "pae", + "pat", + "pausefilter", + "pcid", + "pclmulqdq", + "pdpe1gb", + "pfthreshold", + "pge", + "pni", + "popcnt", + "pse", + "pse36", + "rdpid", + "rdpru", + "rdrand", + "rdrnd", + "rdseed", + "rdtscp", + "rep_good", + "sep", + "sha", + "sha_ni", + "smap", + "smep", + "sse", + "sse2", + "sse4_1", + "sse4_2", + "sse4a", + "ssse3", + "svm", + "syscall", + "topoext", + "tsc", + "tsc_known_freq", + "tsc_reliable", + "tsc_scale", + "umip", + "user_shstk", + "v_vmsave_vmload", + "vaes", + "vmcb_clean", + "vme", + "vmmcall", + "vpclmulqdq", + "xgetbv1", + "xsave", + "xsavec", + "xsaveerptr", + "xsaveopt", + "xsaves" + ], + "l3_cache_size": 524288, + "l2_cache_size": 1048576, + "l1_data_cache_size": 65536, + "l1_instruction_cache_size": 65536, + "l2_cache_line_size": 512, + "l2_cache_associativity": 6 + } + }, + "commit_info": { + "id": "b0b0cffe4be5f1e77e33a742951f54e21140c35f", + "time": "2026-06-17T18:41:35-04:00", + "author_time": "2026-06-17T18:41:35-04:00", + "dirty": false, + "project": "cppa-weblate-plugin", + "branch": "(detached head)" + }, + "benchmarks": [ + { + "group": null, + "name": "test_benchmark_parse_qbk[100]", + "fullname": "tests/utils/test_quickbook.py::test_benchmark_parse_qbk[100]", + "params": { + "target_kb": 100 + }, + "param": "100", + "extra_info": { + "target_kb": 100, + "byte_len": 100985, + "segment_count": 1196 + }, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.041659632999994756, + "max": 0.042719768999972985, + "mean": 0.04211979870832986, + "stddev": 0.00025405583701836786, + "rounds": 24, + "median": 0.042130201000020406, + "iqr": 0.0002444385000046623, + "q1": 0.04197960599998396, + "q3": 0.04222404449998862, + "iqr_outliers": 2, + "stddev_outliers": 6, + "outliers": "6;2", + "ld15iqr": 0.041659632999994756, + "hd15iqr": 0.04264519999998129, + "ops": 23.74180387054495, + "total": 1.0108751689999167, + "data": [ + 0.041983963999996377, + 0.04178534599998329, + 0.041659632999994756, + 0.041672327000014775, + 0.04194475200000625, + 0.0419754579999676, + 0.04222623300000805, + 0.0420838389999858, + 0.04241104700003007, + 0.04216483900000867, + 0.04264519999998129, + 0.042124115000035545, + 0.04228840900003661, + 0.04213628700000527, + 0.04198375400000032, + 0.042719768999972985, + 0.04208120499998813, + 0.042175068999995347, + 0.04192686800001866, + 0.04201994099997819, + 0.042326388999981646, + 0.04216499999995449, + 0.0421538690000034, + 0.04222185599996919 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_benchmark_parse_qbk[500]", + "fullname": "tests/utils/test_quickbook.py::test_benchmark_parse_qbk[500]", + "params": { + "target_kb": 500 + }, + "param": "500", + "extra_info": { + "target_kb": 500, + "byte_len": 517037, + "segment_count": 6095 + }, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.2291080480000005, + "max": 0.23140663099997028, + "mean": 0.23019730579998168, + "stddev": 0.0008821685544740119, + "rounds": 5, + "median": 0.23025139800000716, + "iqr": 0.001282864749981627, + "q1": 0.22950557199997945, + "q3": 0.23078843674996108, + "iqr_outliers": 0, + "stddev_outliers": 2, + "outliers": "2;0", + "ld15iqr": 0.2291080480000005, + "hd15iqr": 0.23140663099997028, + "ops": 4.344099495538403, + "total": 1.1509865289999084, + "data": [ + 0.23025139800000716, + 0.23140663099997028, + 0.230582371999958, + 0.2291080480000005, + 0.22963807999997243 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_benchmark_parse_qbk[1000]", + "fullname": "tests/utils/test_quickbook.py::test_benchmark_parse_qbk[1000]", + "params": { + "target_kb": 1000 + }, + "param": "1000", + "extra_info": { + "target_kb": 1000, + "byte_len": 1035377, + "segment_count": 12190 + }, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.4775930939999853, + "max": 0.4950671309999848, + "mean": 0.48422370899999123, + "stddev": 0.006721239962039176, + "rounds": 5, + "median": 0.48135373299999173, + "iqr": 0.0077254694999595586, + "q1": 0.48038401350001436, + "q3": 0.4881094829999739, + "iqr_outliers": 0, + "stddev_outliers": 1, + "outliers": "1;0", + "ld15iqr": 0.4775930939999853, + "hd15iqr": 0.4950671309999848, + "ops": 2.0651611670671377, + "total": 2.421118544999956, + "data": [ + 0.4950671309999848, + 0.4775930939999853, + 0.48135373299999173, + 0.4857902669999703, + 0.48131432000002405 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_benchmark_quickbook_file_parse", + "fullname": "tests/utils/test_quickbook.py::test_benchmark_quickbook_file_parse", + "params": null, + "param": null, + "extra_info": { + "target_kb": 1024, + "byte_len": 1058849, + "segment_count": 12466 + }, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.5200788729999886, + "max": 0.6418512999999848, + "mean": 0.5456951891999893, + "stddev": 0.053778190923926994, + "rounds": 5, + "median": 0.521640079000008, + "iqr": 0.03327344175001201, + "q1": 0.5204441859999775, + "q3": 0.5537176277499896, + "iqr_outliers": 1, + "stddev_outliers": 1, + "outliers": "1;1", + "ld15iqr": 0.5200788729999886, + "hd15iqr": 0.6418512999999848, + "ops": 1.8325248596492842, + "total": 2.7284759459999464, + "data": [ + 0.5200788729999886, + 0.6418512999999848, + 0.5205659569999739, + 0.521640079000008, + 0.5243397369999911 + ], + "iterations": 1 + } + } + ], + "datetime": "2026-06-17T22:43:36.399500+00:00", + "version": "5.2.3" +} diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index f12cce2..0464415 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -18,6 +18,7 @@ GitHub Actions and CI/CD helpers for this repository (see [`.github/`](../.githu | [`workflows/release.yml`](workflows/release.yml) | **Release** — manual `workflow_dispatch` only; tags `main` from `pyproject.toml` (`v`) and creates a GitHub Release with Weblate compatibility metadata | | [`workflows/ci-lint.yml`](workflows/ci-lint.yml) | Lint and format (prek) | | [`workflows/ci-test.yml`](workflows/ci-test.yml) | Unit tests and coverage | +| [`workflows/ci-benchmark.yml`](workflows/ci-benchmark.yml) | QuickBook parser benchmarks (`pytest-benchmark`; JSON artifact; regression gate vs `.benchmarks/`) | | [`workflows/ci-package.yml`](workflows/ci-package.yml) | Build and package checks | | [`workflows/ci-dependencies.yml`](workflows/ci-dependencies.yml) | Dependency and license audit | | [`workflows/ci-weblate-pin.yml`](workflows/ci-weblate-pin.yml) | **Weblate version sync** — callable from CI; runs [`scripts/check-weblate-pin-sync.sh`](../scripts/check-weblate-pin-sync.sh) so `pyproject.toml` and `Dockerfile.weblate-plugin` pins match | @@ -57,6 +58,28 @@ GH_TEST_REPO_TOKEN= bash scripts/plugin-functional.sh Skip slow plugin tests during local iteration: add `-m "not slow"` to the pytest invocation in the script, or set `PYTEST_PLUGIN_OPTS` accordingly. +## QuickBook parser benchmarks + +[`ci-benchmark.yml`](workflows/ci-benchmark.yml) runs `pytest-benchmark` against synthetic `.qbk` documents (100 KB, 500 KB, 1 MB) in [`tests/utils/test_quickbook.py`](../tests/utils/test_quickbook.py). Results are written to `benchmark-results.json` and uploaded as a workflow artifact. By default the job compares against the committed baseline at [`.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json`](../.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json) and fails when mean time regresses beyond the configured threshold. + +| Variable | Where | Purpose | +|----------|-------|---------| +| `BENCHMARK_COMPARE_FAIL` | Repository variable / workflow env (default `mean:30%`) | Passed to `pytest --benchmark-compare-fail` | +| `BENCHMARK_COMPARE_ENABLED` | Repository variable / workflow env (default `true`) | Set to `false` to skip comparison (record-only mode) | + +**Refresh baseline** after an intentional parser performance change. Capture on **`ubuntu-latest` (GitHub Actions)** — the committed baseline must match CI hardware (local VMs/desktops are often ~2× faster and will cause false regressions). Download the `benchmark-*` artifact from a green run, or on a GitHub-hosted runner: + +```bash +uv run --group dev pytest -m benchmark --benchmark-only \ + -k "not peak_memory" \ + --benchmark-save=baseline tests/utils/test_quickbook.py +git add .benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json +``` + +Peak-memory bounds are checked separately (`test_parse_1mb_peak_memory`); they are not part of the timing baseline compare. + +If CI Python version changes, the `.benchmarks/Linux-CPython-*` directory name changes — regenerate and commit the new baseline path (update `.gitignore` exceptions if needed). + ## Other paths | Path | Role | diff --git a/.github/workflows/ci-benchmark.yml b/.github/workflows/ci-benchmark.yml new file mode 100644 index 0000000..f54d2e8 --- /dev/null +++ b/.github/workflows/ci-benchmark.yml @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 Andrew Zhang +# +# SPDX-License-Identifier: BSL-1.0 + +name: QuickBook parser benchmarks + +on: + workflow_call: + +permissions: + contents: read + +jobs: + quickbook-benchmarks: + name: QuickBook parser benchmarks + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BENCHMARK_COMPARE_ENABLED: ${{ vars.BENCHMARK_COMPARE_ENABLED || 'true' }} + BENCHMARK_COMPARE_FAIL: ${{ vars.BENCHMARK_COMPARE_FAIL || 'mean:30%' }} + steps: + # actions/checkout v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + persist-credentials: false + # actions/setup-python v6.2.0 + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: '3.14' + # astral-sh/setup-uv v8.1.0 + - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 + with: + version: 0.11.12 + - name: Install apt dependencies (Weblate venv) + run: sudo ./.github/ci/apt-install + - name: Install dependencies (incl. dev) + run: uv sync --frozen --group dev + - name: Run QuickBook parser benchmarks + run: | + set -euo pipefail + compare_args=() + if [ "${BENCHMARK_COMPARE_ENABLED}" != "false" ]; then + compare_args+=(--benchmark-compare=0001) + compare_args+=(--benchmark-compare-fail="${BENCHMARK_COMPARE_FAIL}") + fi + uv run --group dev pytest -m benchmark --benchmark-only -v \ + --benchmark-json=benchmark-results.json \ + -k "not peak_memory" \ + "${compare_args[@]}" \ + tests/utils/test_quickbook.py + uv run --group dev pytest -m benchmark -v \ + -k "peak_memory" \ + tests/utils/test_quickbook.py 2>&1 | tee peak-memory.log + - name: Benchmark summary + if: always() + run: | + uv run --group dev python - <<'PY' + import json + from pathlib import Path + + path = Path("benchmark-results.json") + if not path.is_file(): + print("benchmark-results.json not found; skipping summary") + raise SystemExit(0) + + data = json.loads(path.read_text(encoding="utf-8")) + print("## QuickBook parser benchmark summary") + print() + print("| Benchmark | Mean (s) | Min (s) | Max (s) |") + print("|-----------|----------|---------|---------|") + for bench in data.get("benchmarks", []): + name = bench.get("name", "?") + stats = bench.get("stats", {}) + mean_s = stats.get("mean", 0.0) + min_s = stats.get("min", 0.0) + max_s = stats.get("max", 0.0) + extra = bench.get("extra_info", {}) + if extra.get("peak_mib") is not None: + print( + f"| {name} (peak memory) | {extra['peak_mib']} MiB | — | — |" + ) + continue + print( + f"| {name} | {mean_s:.4f} | {min_s:.4f} | {max_s:.4f} |" + ) + peak_log = Path("peak-memory.log") + if peak_log.is_file(): + print() + print("Peak memory test log:") + print(peak_log.read_text(encoding="utf-8")) + PY + # actions/upload-artifact v4.6.2 + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: benchmark-${{ github.event.pull_request.number || github.run_id }} + path: benchmark-results.json + if-no-files-found: warn diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c5b7c87..95b8cc3 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -33,8 +33,11 @@ jobs: - name: Install dependencies (incl. dev) run: uv sync --frozen --group dev --group pre-commit - name: Pytest with coverage + env: + HYPOTHESIS_PROFILE: ci run: > uv run --group dev --group pre-commit pytest -v --tb=short + -m "not plugin and not benchmark" --cov=boost_weblate --cov-report=term-missing --cov-report=xml:coverage.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 393c511..91d956b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,9 @@ jobs: uses: ./.github/workflows/ci-lint.yml test: uses: ./.github/workflows/ci-test.yml + benchmark: + needs: [test] + uses: ./.github/workflows/ci-benchmark.yml package: uses: ./.github/workflows/ci-package.yml dependencies: diff --git a/.gitignore b/.gitignore index b378239..d8d2955 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ dist/ # Testing / coverage .pytest_cache/ +.benchmarks/* +!.benchmarks/Linux-CPython-3.14-64bit/ +!.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json +benchmark-results.json .coverage coverage.xml htmlcov/ diff --git a/README.md b/README.md index 9c7528b..4a904dc 100644 --- a/README.md +++ b/README.md @@ -279,12 +279,13 @@ The service has no plugin-owned models; it operates entirely through Weblate's D ### CI (`ci.yml`) -Triggered on push and PR to `main` and `develop`. Calls eight reusable sub-workflows: +Triggered on push and PR to `main` and `develop`. Calls nine reusable sub-workflows: | Job | Workflow | What it checks | |-----|----------|----------------| | `lint` | [`.github/workflows/ci-lint.yml`](.github/workflows/ci-lint.yml) | prek (Ruff, YAML/TOML, REUSE, actionlint, pytest) | | `test` | [`.github/workflows/ci-test.yml`](.github/workflows/ci-test.yml) | pytest + 90% coverage gate (`--cov-fail-under=90`) | +| `benchmark` | [`.github/workflows/ci-benchmark.yml`](.github/workflows/ci-benchmark.yml) | QuickBook parser benchmarks (`pytest-benchmark`); JSON artifact; optional regression gate vs `.benchmarks/` baseline | | `package` | [`.github/workflows/ci-package.yml`](.github/workflows/ci-package.yml) | `uv build`, twine, pydistcheck, pyroma, check-wheel-contents, check-manifest | | `dependencies` | [`.github/workflows/ci-dependencies.yml`](.github/workflows/ci-dependencies.yml) | pip-audit, liccheck, dependency review (on PRs) | | `weblate-pin` | [`.github/workflows/ci-weblate-pin.yml`](.github/workflows/ci-weblate-pin.yml) | PyPI `Weblate[all]==…` in `pyproject.toml` matches Docker `FROM weblate/weblate:…` (`scripts/check-weblate-pin-sync.sh`) | @@ -376,6 +377,44 @@ pytest -v --tb=short \ (`coverage.xml`, `htmlcov/`, and `.coverage` are gitignored; open `htmlcov/index.html` locally to browse line coverage.) +- **Parser benchmarks:** QuickBook parse-time and memory baselines live in [`tests/utils/test_quickbook.py`](tests/utils/test_quickbook.py) (`@pytest.mark.benchmark`). They are excluded from default pytest runs and pre-commit. Run locally: + +```bash +uv run --group dev pytest -m benchmark --benchmark-only -v tests/utils/test_quickbook.py +``` + +Compare against the committed baseline and fail on regression (default 30% mean): + +```bash +uv run --group dev pytest -m benchmark --benchmark-only -v \ + -k "not peak_memory" \ + --benchmark-compare=0001 \ + --benchmark-compare-fail=mean:30% \ + tests/utils/test_quickbook.py +``` + +Refresh the baseline after an intentional parser change. **Capture on `ubuntu-latest` (GitHub Actions)** — local machines are often faster and will not match CI. Easiest path: download the `benchmark-*` artifact from a green `ci-benchmark` run, copy its `benchmark-results.json` over `0001_baseline.json` (parse-timing benchmarks only), or re-run on a GitHub-hosted runner: + +```bash +uv run --group dev pytest -m benchmark --benchmark-only \ + -k "not peak_memory" \ + --benchmark-save=baseline tests/utils/test_quickbook.py +``` + +Commit the updated `.benchmarks/Linux-CPython-3.14-64bit/0001_baseline.json`. If CI Python changes, regenerate the baseline (the platform directory name changes). + +**Observed baselines** (`ubuntu-latest`, Python 3.14, synthetic documents): + +| Size | Mean `_parse_qbk` time | Segments | +|------|------------------------|----------| +| 100 KB | ~0.042 s | ~1,200 | +| 500 KB | ~0.23 s | ~6,100 | +| 1 MB | ~0.48 s | ~12,200 | +| 1 MB `QuickBookFile.parse` | ~0.55 s | (units ≈ segments) | +| 1 MB peak memory (`tracemalloc`) | ~5.3 MiB | — | + +CI sets `BENCHMARK_COMPARE_FAIL` (default `mean:30%`) and `BENCHMARK_COMPARE_ENABLED` (default `true`; set `false` for record-only runs). See [`.github/WORKFLOWS.md`](.github/WORKFLOWS.md). + - **Pull requests:** open PRs against the default branch on GitHub. Keep changes focused; ensure CI is green. Respond to review feedback on the PR thread; for design questions or bug reports, use [Issues](https://github.com/cppalliance/cppa-weblate-plugin/issues). ## License diff --git a/pyproject.toml b/pyproject.toml index 37bf84d..695cec5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,8 @@ requires = ["uv_build>=0.11.19,<0.12.0"] [dependency-groups] dev = [ "coverage[toml]==7.14.1", + "hypothesis==6.155.3", + "pytest-benchmark==5.2.3", "pytest-cov==7.1.0" ] lint = [ @@ -20,6 +22,7 @@ plugin = [ "pytest-timeout==2.4.0" ] pre-commit = [ + "hypothesis==6.155.3", "prek==0.4.4", "pytest==9.0.3" ] @@ -68,6 +71,7 @@ version = "1.0.0" [project.optional-dependencies] dev = [ "coverage[toml]==7.14.1", + "hypothesis==6.155.3", "prek==0.4.4", "pytest-cov==7.1.0", "pytest==9.0.3" @@ -82,7 +86,8 @@ Repository = "https://github.com/cppalliance/cppa-weblate-plugin" ignore = [ "*.yaml", "*.yml", - ".editorconfig" + ".editorconfig", + ".benchmarks/**" ] [tool.coverage] @@ -132,16 +137,22 @@ level = "cautious" unauthorized_licenses = [] [tool.pytest.ini_options] -addopts = ["-m", "not plugin"] +addopts = ["-m", "not plugin and not benchmark and not fuzz"] markers = [ + "benchmark: parser performance benchmarks (slow; excluded from default test runs)", "plugin: requires live Weblate stack (Docker Compose) and optional WEBLATE_API_TOKEN", - "slow: long-running plugin integration test" + "slow: long-running plugin integration test", + "fuzz: property-based / fuzz tests (excluded from default test runs; pytest -m fuzz)" ] python_classes = ["Test*"] python_files = ["test_*.py", "*_test.py"] pythonpath = ["src", "."] testpaths = ["tests"] +[tool.pytest-benchmark] +min_rounds = 5 +warmup = true + [tool.ruff] line-length = 88 target-version = "py312" diff --git a/src/boost_weblate/utils/quickbook.py b/src/boost_weblate/utils/quickbook.py index 40bdd93..e618d29 100644 --- a/src/boost_weblate/utils/quickbook.py +++ b/src/boost_weblate/utils/quickbook.py @@ -31,8 +31,9 @@ inside a paragraph) are treated as part of the surrounding paragraph. Sections whose body contains further translatable blocks are parsed -recursively (depth-limited) so nested paragraphs and headings are also -extracted. +recursively (depth-limited to 10) so nested paragraphs and headings are also +extracted. Beyond that depth, nested translatable content is silently skipped +rather than raising an error. Reconstruction: :func:`_apply_translations` replaces each translatable span in the @@ -172,7 +173,7 @@ def _find_bracket_end(text: str, start: int) -> int: i += 3 continue if text[i] == "\\": - i += 2 + i += min(2, n - i) continue if text[i] == "[": depth += 1 diff --git a/tests/conftest.py b/tests/conftest.py index 2154c87..4bb482f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,27 @@ def pytest_configure() -> None: if src not in sys.path: sys.path.insert(0, src) + from datetime import timedelta + + from hypothesis import HealthCheck + from hypothesis import settings as hypothesis_settings + + hypothesis_settings.register_profile( + "ci", + max_examples=50, + deadline=timedelta(milliseconds=500), + suppress_health_check=[HealthCheck.too_slow], + derandomize=True, + ) + hypothesis_settings.register_profile( + "dev", + max_examples=200, + deadline=timedelta(seconds=2), + suppress_health_check=[HealthCheck.too_slow], + ) + profile = os.environ.get("HYPOTHESIS_PROFILE", "ci") + hypothesis_settings.load_profile(profile) + # Always use bundled settings so a host ``DJANGO_SETTINGS_MODULE`` does not # break collection or ``python tests/formats/test_quickbook.py``. os.environ["DJANGO_SETTINGS_MODULE"] = "tests.django_qbk_format_settings" diff --git a/tests/utils/test_quickbook.py b/tests/utils/test_quickbook.py index 6e32255..3266b04 100644 --- a/tests/utils/test_quickbook.py +++ b/tests/utils/test_quickbook.py @@ -15,22 +15,33 @@ from __future__ import annotations +import functools +import os import sys +import tracemalloc from io import BytesIO from pathlib import Path import pytest +from hypothesis import given +from hypothesis import strategies as st _REPO_ROOT = Path(__file__).resolve().parents[2] +# Needed for standalone execution (python tests/utils/test_quickbook.py); +# pytest picks this up via conftest.pytest_configure instead. sys.path.insert(0, str(_REPO_ROOT / "src")) from translate.storage.pypo import pofile # noqa: E402 from boost_weblate.utils import quickbook as quickbook_mod # noqa: E402 from boost_weblate.utils.quickbook import ( # noqa: E402 + _ADMONITION_KEYWORDS, + _HEADING_KEYWORDS, QuickBookFile, QuickBookTranslator, QuickBookUnit, + _apply_translations, + _clean_cell_text, _find_bracket_end, _has_prose, _parse_bracket_keyword, @@ -365,6 +376,346 @@ def test_parse_table_inner_cell_bracket_extends_past_row() -> None: assert any(s.msgid == "c" for s in segs) +# --------------------------------------------------------------------------- +# Hypothesis fuzz strategies and property tests +# --------------------------------------------------------------------------- + +_UNICODE_EDGE_CHARS = ( + "\u200f\u202b\u202c" # RTL marks + "\u200d\u200c" # ZWJ / ZWNJ + "\u0301" # combining acute + "\U0001f600" # emoji +) + +_qbk_safe_line = st.text( + alphabet=st.characters(codec="utf-8", blacklist_categories=("Cs",)), + min_size=0, + max_size=40, +) + +qbk_arbitrary = st.text(min_size=0, max_size=1024) + +qbk_unicode_edge = st.text( + alphabet=st.characters(codec="utf-8", blacklist_categories=("Cs",)) + | st.sampled_from(list(_UNICODE_EDGE_CHARS)), + min_size=0, + max_size=512, +) + + +@st.composite +def _qbk_structured_leaf(draw: st.DrawFn) -> str: + text = draw(_qbk_safe_line) + kind = draw( + st.sampled_from( + ["plain", "heading", "template", "raw", "code", "list", "table"] + ) + ) + if kind == "plain": + return (text or "prose") + "\n" + if kind == "heading": + kw = draw(st.sampled_from(sorted(_HEADING_KEYWORDS))) + return f"[{kw} {text or 'Title'}]\n" + if kind == "template": + return f"[template {text or 'a'} {text or 'b'}]\n" + if kind == "raw": + return f"'''{text or 'raw'}'''\n" + if kind == "code": + return " " + (text or "code") + "\n" + if kind == "list": + marker = draw(st.sampled_from(["*", "#"])) + return f"{marker} {text or 'item'}\n" + return f"[table\n{(text or 'Title')}\n[[a][{text or 'cell'}]]]\n" + + +qbk_structured = st.recursive( + _qbk_structured_leaf(), + lambda children: st.one_of( + st.builds( + lambda body, title: f"[section {title}\n{body}]\n", + children, + _qbk_safe_line, + ), + st.builds( + lambda body, kw: f"[{kw} {body}]\n", + children, + st.sampled_from(sorted(_ADMONITION_KEYWORDS)), + ), + st.builds(lambda a, b: a + b, children, children), + ), + max_leaves=20, +) + +_qbk_fuzz_inputs = qbk_arbitrary | qbk_structured | qbk_unicode_edge + + +def _assert_segment_offsets(data: str, segs: list[_Seg]) -> None: + for seg in segs: + assert 0 <= seg.text_start <= seg.text_end <= len(data) + if seg.msgid: + raw = data[seg.text_start : seg.text_end] + assert raw.strip(), "non-empty msgid must map to non-whitespace span" + if seg.no_wrap: + if seg.seg_type in {"table", "variablelist"}: + assert _clean_cell_text(raw) == seg.msgid + else: + assert seg.msgid == raw.strip() + elif seg.msgid: + # msgid normalises soft-wrapped lines; at minimum every word in msgid + # must appear in the raw span. + assert all(word in raw for word in seg.msgid.split()), ( + f"paragraph msgid word not found in span: {seg.msgid!r} vs {raw!r}" + ) + + +@pytest.mark.fuzz +@given(data=_qbk_fuzz_inputs) +def test_parse_qbk_fuzz_properties(data: str) -> None: + """Parser safety, offset invariants, identity round-trip, and bounded output.""" + segs = _parse_qbk(data) + if data.startswith("["): + _find_bracket_end(data, 0) + + assert len(segs) <= len(data) + _assert_segment_offsets(data, segs) + + result = _apply_translations(data, lambda s: s) + assert result == data + + store = QuickBookFile() + store.parse(data) + assert store.filesrc == data + + +@pytest.mark.fuzz +def test_fuzz_corpus_empty_input() -> None: + assert _parse_qbk("") == [] + assert _apply_translations("", lambda s: s) == "" + + +@pytest.mark.fuzz +def test_fuzz_corpus_unclosed_brackets() -> None: + data = "[" * 5000 + _parse_qbk(data) + assert _apply_translations(data, lambda s: s) == data + + +@pytest.mark.fuzz +def test_fuzz_corpus_section_depth_beyond_cap() -> None: + data = "[section " + "[nested\n" * 15 + "body\n" * 15 + _parse_qbk(data) + assert _apply_translations(data, lambda s: s) == data + + +@pytest.mark.fuzz +def test_fuzz_corpus_rtl_wrapped_heading() -> None: + data = "\u200f[h2 Title\u200f]\n" + segs = _parse_qbk(data) + assert segs + assert _apply_translations(data, lambda s: s) == data + + +@pytest.mark.fuzz +def test_fuzz_corpus_invalid_utf8_raises_decode_error() -> None: + store = QuickBookFile() + with pytest.raises(UnicodeDecodeError): + store.parse(b"\xff\xfe") + + +@pytest.mark.fuzz +def test_fuzz_corpus_long_line() -> None: + data = "x" * 16384 + "\n" + assert _apply_translations(data, lambda s: s) == data + + +@pytest.mark.fuzz +def test_fuzz_corpus_large_input() -> None: + data = "[section title\n" + "paragraph line.\n" * 4000 + "]\n" + _parse_qbk(data) + assert _apply_translations(data, lambda s: s) == data + + +# --- Benchmarks --- + +_SIZE_TOLERANCE = 0.02 +# Set after first CI measurement (2x observed peak on ubuntu-latest / Python 3.14). +_PEAK_MEMORY_LIMIT_BYTES = int( + os.environ.get("QBK_PEAK_MEMORY_LIMIT_BYTES", 12 * 1024 * 1024) +) + + +def _synthetic_block(n: int) -> str: + return f"""[template api_{n} [link beast.ref.boost__beast__http__message `message`]] + +[section:sec_{n} Section title {n}] + +Opening paragraph with [@https://example.com/doc/rfc{n} RFC-style link] and +[link beast.ref.boost__beast__http__request `request`] in prose. + +[h2 Section headings and lists] + +* First bullet names [link beast.ref.boost__beast__http__response `response`]. +* Second bullet continues with plain prose. + +[#anchor_{n}] + +[heading:custom_{n} Custom heading with id] + +[:This is a single-line blockquote for translation.] + +[note +Multi-line admonition body for section {n}. +A second paragraph inside the same note uses +[@https://tools.ietf.org/html/rfc6455 WebSocket] markup. +] + +[section:nested_{n} Nested section title here] + +Inner section prose explains that `template` parameters accept any +[link beast.ref.boost__beast__http__fields `fields`] type meeting requirements. + + // Indented code block (non-translatable). + // template + // class message; + +After the code block, prose resumes with a dollar image that is skipped: +[$beast/images/message.png [width 100px] [height 50px]] + +[funcref boost::beast::http::message Reference to message type] + +[endsect] + +[table Message patterns {n} +[[Name][Description]] +[[ + __message__ +][ + ``` + /// Class template overview + template + class message; + ``` +]] +[[ + [link beast.ref.boost__beast__http__request `request`] +][ + ``` + /// HTTP request alias + template + using request = message; + ``` +]] +[[Plain prose cell][ + This cell has human-readable text only, without a code fence. +]] +] + +[variablelist FAQ-style entries {n} +[[ + "Does section {n} include a variablelist?" +][ + Yes. This pair mimics patterns from the FAQ chapter. + + Second paragraph in the same answer cell. +]] +] + +[warning This is a one-line warning about edge cases in section {n}.] + +[endsect] +""" + + +@functools.lru_cache(maxsize=8) +def generate_synthetic_qbk(target_bytes: int) -> str: + if target_bytes <= 0: + raise ValueError("target_bytes must be positive") + low = int(target_bytes * (1 - _SIZE_TOLERANCE)) + high = int(target_bytes * (1 + _SIZE_TOLERANCE)) + header = "[quickbook 1.7]\n\n" + header_len = len(header.encode("utf-8")) + block_size = len(_synthetic_block(0).encode("utf-8")) + num_blocks = max(1, (target_bytes - header_len) // block_size) + filler = "[/ sizing filler]\n" + filler_size = len(filler.encode("utf-8")) + + while num_blocks >= 1: + parts = [header] + for i in range(num_blocks): + parts.append(_synthetic_block(i)) + actual = len("".join(parts).encode("utf-8")) + while actual < low: + parts.append(filler) + actual += filler_size + text = "".join(parts) + actual = len(text.encode("utf-8")) + if actual <= high: + return text + num_blocks -= 1 + + raise RuntimeError( + f"could not generate synthetic qbk within " + f"±{_SIZE_TOLERANCE:.0%} of {target_bytes}" + ) + + +def _assert_synthetic_qbk_valid(text: str, target_bytes: int) -> list[_Seg]: + actual = len(text.encode("utf-8")) + low = int(target_bytes * (1 - _SIZE_TOLERANCE)) + high = int(target_bytes * (1 + _SIZE_TOLERANCE)) + assert low <= actual <= high, f"size {actual} not within [{low}, {high}]" + segs = _parse_qbk(text) + assert segs, "synthetic qbk must yield translatable segments" + return segs + + +@pytest.mark.benchmark +@pytest.mark.parametrize("target_kb", [100, 500, 1000]) +def test_benchmark_parse_qbk(benchmark, target_kb: int) -> None: + target_bytes = target_kb * 1024 + text = generate_synthetic_qbk(target_bytes) + segs = _assert_synthetic_qbk_valid(text, target_bytes) + benchmark.extra_info["target_kb"] = target_kb + benchmark.extra_info["byte_len"] = len(text.encode("utf-8")) + benchmark.extra_info["segment_count"] = len(segs) + result = benchmark(_parse_qbk, text) + assert result + + +@pytest.mark.benchmark +def test_benchmark_quickbook_file_parse(benchmark) -> None: + target_bytes = 1024 * 1024 + text = generate_synthetic_qbk(target_bytes) + segs = _assert_synthetic_qbk_valid(text, target_bytes) + + def _run() -> int: + store = QuickBookFile() + store.parse(text) + return len(store.units) + + benchmark.extra_info["target_kb"] = 1024 + benchmark.extra_info["byte_len"] = len(text.encode("utf-8")) + benchmark.extra_info["segment_count"] = len(segs) + unit_count = benchmark(_run) + assert unit_count > 0 + + +@pytest.mark.benchmark +def test_parse_1mb_peak_memory() -> None: + target_bytes = 1024 * 1024 + text = generate_synthetic_qbk(target_bytes) + _assert_synthetic_qbk_valid(text, target_bytes) + tracemalloc.start() + try: + _parse_qbk(text) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < _PEAK_MEMORY_LIMIT_BYTES, ( + f"peak={peak} ({peak / (1024 * 1024):.2f} MiB)" + ) + + def main(argv: list[str]) -> int: path = Path(argv[1]).resolve() if len(argv) > 1 else DEFAULT_FIXTURE if not path.is_file(): diff --git a/uv.lock b/uv.lock index e691db4..557c28e 100644 --- a/uv.lock +++ b/uv.lock @@ -707,9 +707,12 @@ version = "1.0.0" [package.dev-dependencies] dev = [ {name = "coverage"}, + {name = "hypothesis"}, + {name = "pytest-benchmark"}, {name = "pytest-cov"} ] lint = [ + {name = "hypothesis"}, {name = "prek"}, {name = "pytest"} ] @@ -719,10 +722,12 @@ plugin = [ {name = "pytest-timeout"} ] pre-commit = [ + {name = "hypothesis"}, {name = "prek"}, {name = "pytest"} ] tooling = [ + {name = "hypothesis"}, {name = "prek"}, {name = "pytest"} ] @@ -731,6 +736,7 @@ tooling = [ provides-extras = ["dev"] requires-dist = [ {name = "coverage", extras = ["toml"], marker = "extra == 'dev'", specifier = "==7.14.1"}, + {name = "hypothesis", marker = "extra == 'dev'", specifier = "==6.155.3"}, {name = "packaging", specifier = "==26.2"}, {name = "prek", marker = "extra == 'dev'", specifier = "==0.4.4"}, {name = "pytest", marker = "extra == 'dev'", specifier = "==9.0.3"}, @@ -741,9 +747,12 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ {name = "coverage", extras = ["toml"], specifier = "==7.14.1"}, + {name = "hypothesis", specifier = "==6.155.3"}, + {name = "pytest-benchmark", specifier = "==5.2.3"}, {name = "pytest-cov", specifier = "==7.1.0"} ] lint = [ + {name = "hypothesis", specifier = "==6.155.3"}, {name = "prek", specifier = "==0.4.4"}, {name = "pytest", specifier = "==9.0.3"} ] @@ -753,10 +762,12 @@ plugin = [ {name = "pytest-timeout", specifier = "==2.4.0"} ] pre-commit = [ + {name = "hypothesis", specifier = "==6.155.3"}, {name = "prek", specifier = "==0.4.4"}, {name = "pytest", specifier = "==9.0.3"} ] tooling = [ + {name = "hypothesis", specifier = "==6.155.3"}, {name = "prek", specifier = "==0.4.4"}, {name = "pytest", specifier = "==9.0.3"} ] @@ -764,6 +775,7 @@ tooling = [ [package.optional-dependencies] dev = [ {name = "coverage"}, + {name = "hypothesis"}, {name = "prek"}, {name = "pytest"}, {name = "pytest-cov"} @@ -1590,6 +1602,18 @@ wheels = [ {url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z"} ] +[[package]] +dependencies = [ + {name = "sortedcontainers"} +] +name = "hypothesis" +sdist = {url = "https://files.pythonhosted.org/packages/36/77/13ec9b6390bce44f5badab39837dd6789bbfe6342a2ac611a71537a7756f/hypothesis-6.155.3.tar.gz", hash = "sha256:1e34b17ae9873515384312cb7640abd773eb096c7eef8c0d9c614fa2c306e9bb", size = 477961, upload-time = "2026-06-16T00:33:23.273Z"} +source = {registry = "https://pypi.org/simple"} +version = "6.155.3" +wheels = [ + {url = "https://files.pythonhosted.org/packages/a2/23/ce3a543935a01e478349e82f6c1440776f92d4cb346662c4d81574878fed/hypothesis-6.155.3-py3-none-any.whl", hash = "sha256:ede5a3d142d9c5c9f70cb3075541905b228d6c3a682bcec3d4fe0722e9eda127", size = 544401, upload-time = "2026-06-16T00:33:20.497Z"} +] + [[package]] name = "idna" sdist = {url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z"} @@ -2297,6 +2321,15 @@ wheels = [ {url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z"} ] +[[package]] +name = "py-cpuinfo" +sdist = {url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z"} +source = {registry = "https://pypi.org/simple"} +version = "9.0.0" +wheels = [ + {url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z"} +] + [[package]] name = "pyaskalono" sdist = {url = "https://files.pythonhosted.org/packages/e3/f5/7f9b6cc9944a91ef293e3e741c4452f0e79066924c30ff4bea39a206bc69/pyaskalono-0.2.0.tar.gz", hash = "sha256:00563a0ea333c4124226418b26e46bd3e929cde2ce9f60dea2916f738fc7a190", size = 2040010, upload-time = "2025-11-21T21:01:34.376Z"} @@ -2448,6 +2481,19 @@ wheels = [ {url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z"} ] +[[package]] +dependencies = [ + {name = "py-cpuinfo"}, + {name = "pytest"} +] +name = "pytest-benchmark" +sdist = {url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z"} +source = {registry = "https://pypi.org/simple"} +version = "5.2.3" +wheels = [ + {url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z"} +] + [[package]] dependencies = [ {name = "coverage"}, @@ -3172,6 +3218,15 @@ wheels = [ {url = "https://files.pythonhosted.org/packages/ff/f2/0cb91ff294282800cc5ec7cefdfa3d87068c805d30920e3842fc859853ac/social_auth_core-4.9.1-py3-none-any.whl", hash = "sha256:2eca8a6ae678bae7e24e5aa3c72f99c77983012a09b8916ce9791dcfccade725", size = 457143, upload-time = "2026-04-30T07:16:17.235Z"} ] +[[package]] +name = "sortedcontainers" +sdist = {url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z"} +source = {registry = "https://pypi.org/simple"} +version = "2.4.0" +wheels = [ + {url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z"} +] + [[package]] dependencies = [ {name = "alabaster"},