Create pyo3 runner

.github/workflows/dependencies.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: Install tools
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-deny,cargo-unmaintained,mise@2026.6.5,osv-scanner
+          tool: cargo-deny,cargo-unmaintained,coreutils,mise@2026.6.5,osv-scanner,ripgrep
 
       - name: Trust mise config
         run: mise trust

.mise/config.dart.toml

@@ -3,8 +3,8 @@
 [vars]
 # Google dart-archive base URLs — the release tree and the bucket listing —
 # referenced by [tools.dart] below so its url/version_list_url stay one line.
-dart_release = "https://storage.googleapis.com/dart-archive/channels/stable/release"
 dart_bucket = "https://storage.googleapis.com/storage/v1/b/dart-archive/o"
+dart_release = "https://storage.googleapis.com/dart-archive/channels/stable/release"
 
 [tools]
 # cargo: backend -- dart-typegen has no prebuilt binary (crates.io only).
@@ -14,14 +14,14 @@ dart_bucket = "https://storage.googleapis.com/storage/v1/b/dart-archive/o"
 # archive, discovering the latest stable from the bucket listing. `{{ version }}`
 # etc. resolve in the tool-url context; the static host/path prefix is a var.
 [tools.dart]
-version = "latest"
 url = "{{ vars.dart_release }}/{{ version }}/sdk/dartsdk-{{ os() }}-{{ arch() }}-release.zip"
-version_list_url = "{{ vars.dart_bucket }}?prefix=channels/stable/release/&delimiter=/"
+version = "latest"
 version_expr = '''
 fromJSON(body).prefixes
 | filter({ # matches "^channels/stable/release/(\\d+\\.\\d+\\.\\d+)/$" })
 | map({split(#, "/")[3]})
 | sortVersions()'''
+version_list_url = "{{ vars.dart_bucket }}?prefix=channels/stable/release/&delimiter=/"
 
 [tasks.dart-pub-get]
 # `dart format` and `dart analyze` walk up to find pubspec.yaml *and* read

.mise/config.linux.toml

@@ -1,6 +1,6 @@
 # Linux-only mise config. Shared vars (rpath_flag, pylib_flag, conda_openssl,
-# py313_unix, …) come from config.toml, which loads first; PYO3_PYTHON keeps its
-# config.toml default (py313_unix).
+# py3_unix, …) come from config.toml, which loads first; PYO3_PYTHON keeps its
+# config.toml default (py3_unix).
 
 [tools]
 # Ships the C `libclang.so` (+ clang resource headers) bindgen needs to build the
@@ -44,19 +44,39 @@ LIBCLANG_PATH = "{{ vars.a_conda_clangxx }}/lib"
 
 [tasks.preinstall]
 # Preinstall: the shared cross-platform base (via _setup_all), then verify the
-# system build prerequisites mise can't supply -- the C/C++ toolchain, gpg and
-# archive tools the Dockerfile installs via apt. A workstation missing any is
-# told to install them with its package manager (CI/Docker already have them).
+# build prerequisites mise can't supply. The list has a single source of truth --
+# the Dockerfile's APT_PACKAGES: the Docker build exposes it as an env var (the
+# ARG is in scope for the preinstall RUN), and a workstation reads it back from
+# the Dockerfile in the checkout. Each package is probed for presence -- a command
+# for the CLI tools (gpg for gnupg, xz for xz-utils, else the package name), a key
+# file for ca-certificates and libc6-dev, and the versioned shared library on disk
+# for libicu (a file check, not ldconfig, so it doesn't hinge on the ld.so cache
+# being refreshed). An empty list is a hard error, not a silent skip.
 depends = ["_setup_all"]
 description = "Preinstall: shared base + verify system build prerequisites"
 run = """
-pkgs="bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils"
+pkgs="${APT_PACKAGES:-}"
+if [ -z "$pkgs" ]; then
+  dockerfile="{{ config_root }}/Dockerfile"
+  pkgs="$(rg '^ARG APT_PACKAGES=' "$dockerfile" 2>/dev/null | coreutils cut -d'"' -f2 || true)"
+fi
+if [ -z "$pkgs" ]; then
+  echo "preinstall: could not read the APT_PACKAGES prerequisite list" >&2
+  exit 1
+fi
 missing=""
-for cmd in gcc g++ make git curl gpg unzip bzip2 xz; do
-  command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd"
+for pkg in $pkgs; do
+  case "$pkg" in
+    gnupg) command -v gpg >/dev/null 2>&1 ;;
+    xz-utils) command -v xz >/dev/null 2>&1 ;;
+    ca-certificates) [ -r /etc/ssl/certs/ca-certificates.crt ] ;;
+    libc6-dev) [ -r /usr/include/stdio.h ] ;;
+    libicu*) coreutils ls /usr/lib/*/libicui18n.so.${pkg#libicu} >/dev/null 2>&1 ;;
+    *) command -v "$pkg" >/dev/null 2>&1 ;;
+  esac || missing="$missing $pkg"
 done
 if [ -n "$missing" ]; then
-  echo "preinstall: missing required system tools:$missing" >&2
+  echo "preinstall: missing required build prerequisites:$missing" >&2
   echo "Install them with your package manager. On Debian/Ubuntu:" >&2
   echo "  sudo apt-get install -y $pkgs" >&2
   exit 1

.mise/config.macos.toml

@@ -1,6 +1,6 @@
-# macOS-only mise config. Shared vars (rpath_flag, pylib_flag, py313_unix, …)
+# macOS-only mise config. Shared vars (rpath_flag, pylib_flag, py3_unix, …)
 # come from config.toml, which loads first; PYO3_PYTHON keeps its config.toml
-# default (py313_unix).
+# default (py3_unix).
 
 [tools]
 # The LLD linker (ships ld64.lld) the RUSTFLAGS below use; Apple's /usr/bin/clang

.mise/config.python.toml

@@ -7,6 +7,10 @@
 "pipx:datamodel-code-generator" = { version = "latest", extras = "ruff" }
 "pipx:openapi-python-client" = "0.29.0"
 "pipx:pytest" = "latest"
+# PyTorch, for the et-ws-pyo3-runner `torch_inference` test. Python-only (not in
+# the always-loaded config) -- it's a large optional dependency, and the test
+# skips itself when torch isn't among the mise-installed packages on sys.path.
+"pipx:torch" = "latest"
 ruff = "latest"
 
 # Use the GitHub release tarball, not `npm:pyodide`. The npm package is only
@@ -18,13 +22,10 @@ ruff = "latest"
 # top-level `pyodide/` directory the modules service picks up — see
 # `default_modules_folders` in libs/edge-toolkit/src/config.rs.
 [tools."http:pyodide"]
-version = "0.29.3"
 url = "https://github.com/pyodide/pyodide/releases/download/{{ version }}/pyodide-{{ version }}.tar.bz2"
+version = "0.29.3"
 
 [vars]
-# Directories the ruff tasks lint/format (the generated componentize-py dirs
-# under them are excluded by ruff.toml's `extend-exclude`).
-py_dirs = "services/ws-modules/ generated/python-ws/ generated/python-rest/"
 # Shared WIT dir for the wasi-graphics-info componentize-py build (relative to
 # that module's dir); used by both the bindings and componentize steps.
 wit_dir = "../../../generated/specs/wit"
@@ -34,9 +35,13 @@ wit_dir = "../../../generated/specs/wit"
 RUFF_CACHE_DIR = "{{ config_root }}/target/ruff-cache"
 
 [tasks]
-ruff-check = "ruff check {{ vars.py_dirs }}"
-ruff-fmt = "ruff format {{ vars.py_dirs }}"
-ruff-fmt-check = "ruff format --check {{ vars.py_dirs }}"
+# No path args: ruff walks from the repo root, honouring .gitignore (so
+# target/ is skipped) and ruff.toml's `exclude`/`extend-exclude`. Every
+# tracked *.py already lives under a dir we want linted, so discovery covers
+# them without an explicit include list -- scope is narrowed by excludes only.
+ruff-check = "ruff check"
+ruff-fmt = "ruff format"
+ruff-fmt-check = "ruff format --check"
 
 # Namespaced aggregators picked up by the default config's globbed
 # `check`/`fmt`/`test`.
@@ -57,15 +62,8 @@ dir = "services/ws-modules/pyface1"
 run = "uv run pytest"
 
 [tasks.test-pyface1.env]
-# See the long-form comment in `[tasks."prefetch:python".env]` for why this is
-# a per-OS tera template: `mise which python3.13` works on Linux/macOS and
-# disambiguates from pyodide's `python` wrapper; on Windows mise's
-# python-build-standalone install only exposes `python.exe`, so we fall
-# back to `mise which python` (pyodide doesn't shim on Windows, so
-# there's no ambiguity to resolve).
-UV_PYTHON = """\
-{% if os() == 'windows' %}{{ exec(command='mise where python') }}/python.exe\
-{% else %}{{ exec(command='mise which python3.13') }}{% endif %}"""
+# Same uv-interpreter pin as `[tasks."prefetch:python".env]`; see there.
+UV_PYTHON = "{% if os() == 'windows' %}{{ vars.py3_win }}{% else %}{{ vars.py3_unix }}{% endif %}"
 
 [tasks.build-et-ws-wheel]
 depends = ["build-et-cli", "gen:python-ws"]
@@ -152,39 +150,33 @@ uv sync --directory services/ws-modules/pyface1
 shell = "bash -euo pipefail -c"
 
 [tasks."prefetch:python".env]
-# UV_PYTHON pins uv to the mise-managed regular CPython, not Pyodide's
-# emscripten-wasm32 interpreter. mise's `http-pyodide` install ships a
-# `python` wrapper script in its install dir that uv's auto-discovery
-# finds and prefers — it reports CPython 3.13.2, so bare version requests
-# (`3.13`, `python3.13`, `cpython@3.13`) don't disambiguate. Resolve the
-# absolute path via mise's `exec()` template rather than a shell `export` in
-# the `run` body: the template is evaluated by mise (shell-agnostic), so it
-# works under `cmd` on Windows, which has no `export`. Matches the same idiom
-# used by `[tasks.test-pyface1.env]` above.
-#
-# Windows quirk: mise's python-build-standalone install on Windows only
-# exposes `python.exe` (no `python3.13.exe`), so `mise which python3.13`
-# errors with "not a mise bin". And `mise which python` resolves to
-# pyodide's bundled `python.exe` (uv fails to inspect it — the pyodide
-# shim's `sys.path` injection is not a valid uv interpreter). Use
-# `mise where python` instead — that returns the python plugin's
-# install dir specifically — and append `python.exe`, the bin name
-# python-build-standalone places at the install root on Windows.
-UV_PYTHON = """\
-{% if os() == 'windows' %}{{ exec(command='mise where python') }}/python.exe\
-{% else %}{{ exec(command='mise which python3.13') }}{% endif %}"""
+# Pin uv to the mise-managed regular CPython, not Pyodide's emscripten-wasm32
+# interpreter. uv's auto-discovery otherwise finds and prefers the `python`
+# wrapper mise's `http:pyodide` install ships (it reports CPython 3.13.2, so
+# bare version requests like `3.13` / `python3.13` don't disambiguate). Point
+# uv straight at the absolute interpreter path via the py3_unix / py3_win
+# vars -- the same interpreter PYO3_PYTHON resolves to -- so the pyodide wrapper
+# never enters discovery and no `mise` subprocess runs per task. Windows needs
+# its own var because python-build-standalone lays the binary out at the install
+# root (`python.exe`), not under `bin/`.
+UV_PYTHON = "{% if os() == 'windows' %}{{ vars.py3_win }}{% else %}{{ vars.py3_unix }}{% endif %}"
 
 [tasks."gen:python-rest"]
 depends = ["gen:ws-spec"]
 description = "Emit the typed Python REST client via openapi-python-client (consumes generated/specs/rest.yaml)"
 run = """
+# Codegen; committed and regenerated/verified on Linux. Skip on Windows, where
+# the workspace `ruff` (a mise tool) isn't on the busybox-ash PATH -- same as
+# gen:python-ws. Regenerating there would only risk clobbering committed source.
+[ "${OS:-}" = "Windows_NT" ] && { echo "gen:python-rest: skipped on Windows (regenerated on Linux)"; exit 0; }
 mkdir -p generated/python-rest/et_rest_client
-openapi-python-client generate \
-  --config config/openapi-python-client.yaml \
-  --path generated/specs/rest.yaml \
-  --meta none \
-  --overwrite \
-  --output-path generated/python-rest/et_rest_client
+# None of these args move into config/openapi-python-client.yaml -- path / meta
+# / overwrite / output-path are CLI-only `generate` options. Build them up in a
+# shell var (split over two lines) so neither line needs a backslash
+# continuation or runs past 120 cols; unquoted $args word-splits into argv.
+args="--config config/openapi-python-client.yaml --path generated/specs/rest.yaml"
+args="$args --meta none --overwrite --output-path generated/python-rest/et_rest_client"
+openapi-python-client generate $args
 # openapi-python-client / our follow-up ruff drop a .ruff_cache next to
 # the generated source; we don't want that committed.
 rm -rf generated/python-rest/et_rest_client/.ruff_cache
@@ -203,38 +195,26 @@ shell = "bash -euo pipefail -c"
 [tasks."gen:python-ws"]
 depends = ["gen:ws-spec"]
 description = "Emit Pydantic models for the WS protocol via datamodel-code-generator"
+# Runs from generated/python-ws/ so datamodel-codegen discovers that dir's
+# pyproject.toml [tool.datamodel-codegen] (model-shape + style flags); only the
+# input/output paths are passed on the CLI.
+dir = "generated/python-ws"
 run = """
-mkdir -p generated/python-ws/et_ws
-# `--custom-file-header "#"` is the only way to suppress datamodel-codegen's
-# "generated by datamodel-codegen / filename: …" banner (an empty string is
-# treated as "no override" and the default banner returns). Strip the
-# remaining lone `#` line below so the file starts cleanly.
-# `--formatters ruff-format ruff-check` opts in to the new ruff-based
-# formatter stack — silences the FutureWarning about black/isort being
-# replaced and gives us a single formatter (ruff) end-to-end. The `[ruff]`
-# extra on the pipx install (in [tools]) puts ruff in datamodel-codegen's
-# venv so it can find it.
-datamodel-codegen \
-  --input target/int-gen/ws.schema.json \
-  --input-file-type jsonschema \
-  --output generated/python-ws/et_ws/messages.py \
-  --output-model-type pydantic_v2.BaseModel \
-  --target-python-version 3.10 \
-  --use-schema-description \
-  --use-title-as-name \
-  --use-double-quotes \
-  --use-union-operator \
-  --field-constraints \
-  --disable-timestamp \
-  --formatters ruff-format ruff-check \
-  --custom-file-header '#'
-sed -i.bak -e '1{/^#$/d;}' -e '2{/^$/d;}' generated/python-ws/et_ws/messages.py
-rm generated/python-ws/et_ws/messages.py.bak
-# datamodel-codegen's ruff formatter runs without our repo ruff.toml, so
-# line-length defaults differ and imports aren't isort-grouped. Re-run
-# ruff from the repo to apply both — `check --fix` for import sorting
-# ("I" rules in ruff.toml), then `format` for whitespace.
-ruff check --fix generated/python-ws/et_ws/messages.py
-ruff format generated/python-ws/et_ws/messages.py
+# Codegen; the output is committed and regenerated/verified on Linux. Skip on
+# Windows, where this task's tools (coreutils/ruff) aren't on the busybox-ash
+# PATH -- regenerating there would only risk clobbering the committed source.
+[ "${OS:-}" = "Windows_NT" ] && { echo "gen:python-ws: skipped on Windows (regenerated on Linux)"; exit 0; }
+mkdir -p et_ws
+datamodel-codegen --input ../../target/int-gen/ws.schema.json --output et_ws/messages.py
+# Drop the 2-line header (a lone `#` + the blank after it) that
+# custom-file-header = "#" emits, so the file starts cleanly. tail, not sed:
+# datamodel-codegen always emits exactly those two lines, so a fixed offset is safe.
+coreutils tail -n +3 et_ws/messages.py > et_ws/messages.py.tmp
+coreutils mv et_ws/messages.py.tmp et_ws/messages.py
+# datamodel-codegen's bundled ruff runs with its own defaults, not our repo
+# ruff.toml, so re-run repo ruff: `check --fix` for import sorting ("I" rules),
+# then `format` for whitespace/line-length.
+ruff check --fix et_ws/messages.py
+ruff format et_ws/messages.py
 """
 shell = "bash -euo pipefail -c"