From 7d9f34e477a6777a66ced1246a1acc362faa7802 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sat, 7 Feb 2026 23:05:01 -0800 Subject: [PATCH 01/27] Add Deno/Pyodide executor skeleton --- src/py_code_mode/execution/__init__.py | 13 + .../execution/deno_pyodide/__init__.py | 10 + .../execution/deno_pyodide/config.py | 37 ++ .../execution/deno_pyodide/executor.py | 548 ++++++++++++++++++ .../execution/deno_pyodide/runner/main.ts | 223 +++++++ .../execution/deno_pyodide/runner/worker.ts | 257 ++++++++ .../execution/resource_provider.py | 288 +++++++++ .../execution/subprocess/executor.py | 319 +--------- tests/test_deno_pyodide_executor.py | 40 ++ 9 files changed, 1417 insertions(+), 318 deletions(-) create mode 100644 src/py_code_mode/execution/deno_pyodide/__init__.py create mode 100644 src/py_code_mode/execution/deno_pyodide/config.py create mode 100644 src/py_code_mode/execution/deno_pyodide/executor.py create mode 100644 src/py_code_mode/execution/deno_pyodide/runner/main.ts create mode 100644 src/py_code_mode/execution/deno_pyodide/runner/worker.ts create mode 100644 src/py_code_mode/execution/resource_provider.py create mode 100644 tests/test_deno_pyodide_executor.py diff --git a/src/py_code_mode/execution/__init__.py b/src/py_code_mode/execution/__init__.py index 0eae687..c5ca16c 100644 --- a/src/py_code_mode/execution/__init__.py +++ b/src/py_code_mode/execution/__init__.py @@ -35,6 +35,16 @@ SubprocessConfig = None # type: ignore SubprocessExecutor = None # type: ignore +# Deno/Pyodide is optional at runtime (requires deno + pyodide assets). +try: + from py_code_mode.execution.deno_pyodide import DenoPyodideConfig, DenoPyodideExecutor + + DENO_PYODIDE_AVAILABLE = True +except Exception: + DENO_PYODIDE_AVAILABLE = False + DenoPyodideConfig = None # type: ignore + DenoPyodideExecutor = None # type: ignore + __all__ = [ "Capability", "Executor", @@ -52,4 +62,7 @@ "SubprocessExecutor", "SubprocessConfig", "SUBPROCESS_AVAILABLE", + "DenoPyodideExecutor", + "DenoPyodideConfig", + "DENO_PYODIDE_AVAILABLE", ] diff --git a/src/py_code_mode/execution/deno_pyodide/__init__.py b/src/py_code_mode/execution/deno_pyodide/__init__.py new file mode 100644 index 0000000..126365e --- /dev/null +++ b/src/py_code_mode/execution/deno_pyodide/__init__.py @@ -0,0 +1,10 @@ +"""Deno + Pyodide execution backend. + +This backend runs Python inside Pyodide (WASM) hosted by a Deno subprocess. +It is intended to provide a sandboxed runtime without requiring Docker. +""" + +from py_code_mode.execution.deno_pyodide.config import DenoPyodideConfig +from py_code_mode.execution.deno_pyodide.executor import DenoPyodideExecutor + +__all__ = ["DenoPyodideConfig", "DenoPyodideExecutor"] diff --git a/src/py_code_mode/execution/deno_pyodide/config.py b/src/py_code_mode/execution/deno_pyodide/config.py new file mode 100644 index 0000000..6b47011 --- /dev/null +++ b/src/py_code_mode/execution/deno_pyodide/config.py @@ -0,0 +1,37 @@ +"""Configuration for DenoPyodideExecutor.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class DenoPyodideConfig: + """Configuration for DenoPyodideExecutor. + + Notes: + - This executor expects Pyodide runtime assets (WASM + stdlib files) to be + present on disk and readable by the Deno subprocess. + - Dependency installs are best-effort via Pyodide; many wheels will not work. + """ + + default_timeout: float | None = 60.0 + allow_runtime_deps: bool = True + tools_path: Path | None = None + deps: tuple[str, ...] | None = None + deps_file: Path | None = None + ipc_timeout: float = 30.0 + + deno_executable: str = "deno" + # If None, the executor uses the packaged runner script adjacent to this module. + runner_path: Path | None = None + + # Directory used for Deno's module/npm cache (DENO_DIR). If None, a default + # per-user cache directory is used. This cache is prepared outside the + # sandbox (host) via `deno cache`, then the sandbox runs with --cached-only. + deno_dir: Path | None = None + + # Deno network mode: keep sandbox `--deny-net` by default. + # Future: allowlist support for deps/networking use cases. + network_mode: str = "deny" # "deny" | "allow" | "allowlist" diff --git a/src/py_code_mode/execution/deno_pyodide/executor.py b/src/py_code_mode/execution/deno_pyodide/executor.py new file mode 100644 index 0000000..2066cd1 --- /dev/null +++ b/src/py_code_mode/execution/deno_pyodide/executor.py @@ -0,0 +1,548 @@ +"""Deno + Pyodide execution backend. + +This executor runs Python inside Pyodide (WASM) in a Deno subprocess. + +It uses an NDJSON IPC protocol and a host-side provider to service sandbox +namespace proxy operations (tools/workflows/artifacts/deps persistence). +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import shutil +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from py_code_mode.deps import ( + FileDepsStore, + MemoryDepsStore, + RuntimeDepsDisabledError, + collect_configured_deps, +) +from py_code_mode.deps.store import DepsStore +from py_code_mode.execution.deno_pyodide.config import DenoPyodideConfig +from py_code_mode.execution.protocol import Capability, validate_storage_not_access +from py_code_mode.execution.registry import register_backend +from py_code_mode.execution.resource_provider import StorageResourceProvider +from py_code_mode.tools import ToolRegistry, load_tools_from_path +from py_code_mode.types import ExecutionResult + +if TYPE_CHECKING: + from asyncio.subprocess import Process + + from py_code_mode.storage.backends import StorageBackend + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _Pending: + fut: asyncio.Future[dict[str, Any]] + + +class DenoPyodideExecutor: + """Execute code in Pyodide hosted by Deno. + + Capabilities: + - TIMEOUT: Yes (soft timeout in host wait; does not interrupt sandbox run) + - PROCESS_ISOLATION: Yes (Deno subprocess) + - NETWORK_ISOLATION: Yes (Deno permissions; default deny) + - FILESYSTEM_ISOLATION: Partial (Deno permissions; allow-read to Pyodide assets) + - RESET: Yes (restart Deno subprocess) + - DEPS_INSTALL / DEPS_UNINSTALL: Best-effort (Pyodide/micropip) + """ + + _CAPABILITIES = frozenset( + { + Capability.TIMEOUT, + Capability.PROCESS_ISOLATION, + Capability.NETWORK_ISOLATION, + Capability.FILESYSTEM_ISOLATION, + Capability.RESET, + Capability.DEPS_INSTALL, + Capability.DEPS_UNINSTALL, + } + ) + + def __init__(self, config: DenoPyodideConfig | None = None) -> None: + self._config = config or DenoPyodideConfig() + self._proc: Process | None = None + self._stdout_task: asyncio.Task[None] | None = None + self._pending: dict[str, _Pending] = {} + self._closed = False + self._wedged = False # indicates a soft-timeout run may still be executing + self._deno_dir: Path | None = None + + self._storage: StorageBackend | None = None + self._provider: StorageResourceProvider | None = None + self._tool_registry: ToolRegistry | None = None + self._deps_store: DepsStore | None = None + + def supports(self, capability: str) -> bool: + return capability in self._CAPABILITIES + + def supported_capabilities(self) -> set[str]: + return set(self._CAPABILITIES) + + def get_configured_deps(self) -> list[str]: + return collect_configured_deps(self._config.deps, self._config.deps_file) + + async def __aenter__(self) -> DenoPyodideExecutor: + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + await self.close() + + async def start(self, storage: StorageBackend | None = None) -> None: + validate_storage_not_access(storage, "DenoPyodideExecutor") + if self._proc is not None: + return + + if shutil.which(self._config.deno_executable) is None: + raise RuntimeError(f"deno not found: {self._config.deno_executable!r}") + + self._storage = storage + self._deno_dir = (self._config.deno_dir or self._default_deno_dir()).expanduser().resolve() + self._deno_dir.mkdir(parents=True, exist_ok=True) + + # Tools from executor config + if self._config.tools_path is not None: + self._tool_registry = await load_tools_from_path(self._config.tools_path) + else: + self._tool_registry = ToolRegistry() + + # Deps store from executor config (persistence only; installs happen in sandbox) + initial_deps = collect_configured_deps(self._config.deps, self._config.deps_file) + if self._config.deps_file: + self._deps_store = FileDepsStore(self._config.deps_file.parent) + for dep in initial_deps: + if not self._deps_store.exists(dep): + self._deps_store.add(dep) + else: + self._deps_store = MemoryDepsStore() + for dep in initial_deps: + self._deps_store.add(dep) + + if storage is not None: + self._provider = StorageResourceProvider( + storage=storage, + tool_registry=self._tool_registry, + deps_store=self._deps_store, + allow_runtime_deps=self._config.allow_runtime_deps, + ) + + await self._ensure_deno_cache() + await self._spawn_runner() + + def _default_deno_dir(self) -> Path: + return Path.home() / ".cache" / "py-code-mode" / "deno-pyodide" + + async def _ensure_deno_cache(self) -> None: + """Cache runner modules + npm:pyodide outside the sandbox.""" + runner_path = self._config.runner_path or ( + Path(__file__).resolve().parent / "runner" / "main.ts" + ) + worker_path = runner_path.parent / "worker.ts" + deno_dir = self._deno_dir + assert deno_dir is not None + + cmd = [ + self._config.deno_executable, + "cache", + "--quiet", + "--no-check", + str(runner_path), + str(worker_path), + ] + proc = await asyncio.create_subprocess_exec( + *cmd, + env={**os.environ, "DENO_DIR": str(deno_dir)}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _out, err = await proc.communicate() + if proc.returncode != 0: + raise RuntimeError(f"deno cache failed: {err.decode('utf-8', errors='replace')}") + + async def _spawn_runner(self) -> None: + runner_path = self._config.runner_path or ( + Path(__file__).resolve().parent / "runner" / "main.ts" + ) + runner_dir = runner_path.parent.resolve() + deno_dir = self._deno_dir + assert deno_dir is not None + + # Sandbox defaults: cached-only + deny-net. + allow_reads = [runner_dir, deno_dir] + + deno_args: list[str] = [ + self._config.deno_executable, + "run", + "--quiet", + "--no-check", + "--no-prompt", + "--cached-only", + "--location=https://pyodide.invalid/", + "--deny-write", + "--deny-env", + "--deny-run", + ] + if self._config.network_mode == "allow": + deno_args.append("--allow-net") + else: + deno_args.append("--deny-net") + + deno_args.append(f"--allow-read={','.join(str(p) for p in allow_reads)}") + deno_args.append(str(runner_path)) + + self._proc = await asyncio.create_subprocess_exec( + *deno_args, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env={**os.environ, "DENO_DIR": str(deno_dir)}, + ) + + assert self._proc.stdin is not None + assert self._proc.stdout is not None + + self._stdout_task = asyncio.create_task(self._stdout_loop()) + + # Initialize runner with runtime directory and wait for ready. + init_id = uuid.uuid4().hex + ready = await self._request( + { + "id": init_id, + "type": "init", + }, + timeout=self._config.ipc_timeout, + ) + if ready.get("type") != "ready": + raise RuntimeError(f"Runner init failed: {ready!r}") + + self._wedged = False + + async def _stdout_loop(self) -> None: + assert self._proc is not None and self._proc.stdout is not None + reader = self._proc.stdout + while True: + line = await reader.readline() + if not line: + return + try: + msg = json.loads(line.decode("utf-8", errors="replace")) + except Exception: + continue + if not isinstance(msg, dict): + continue + + msg_type = msg.get("type") + msg_id = msg.get("id") + + if msg_type == "rpc_request": + await self._handle_rpc_request(msg) + continue + + if isinstance(msg_id, str) and msg_id in self._pending: + pending = self._pending.pop(msg_id) + if not pending.fut.done(): + pending.fut.set_result(msg) + + async def _handle_rpc_request(self, msg: dict[str, Any]) -> None: + if self._proc is None or self._proc.stdin is None: + return + + req_id = msg.get("id") + namespace = msg.get("namespace") + op = msg.get("op") + args = msg.get("args") or {} + + if not isinstance(req_id, str) or not isinstance(namespace, str) or not isinstance(op, str): + await self._send( + { + "id": str(req_id), + "type": "rpc_response", + "ok": False, + "error": "bad rpc request", + } + ) + return + + try: + result = await self._dispatch_rpc(namespace, op, args) + await self._send({"id": req_id, "type": "rpc_response", "ok": True, "result": result}) + except Exception as e: + await self._send( + { + "id": req_id, + "type": "rpc_response", + "ok": False, + "error": {"type": type(e).__name__, "message": str(e)}, + } + ) + + async def _dispatch_rpc(self, namespace: str, op: str, args: dict[str, Any]) -> Any: + if namespace in ("tools", "workflows", "artifacts"): + if self._provider is None: + raise RuntimeError("No storage/provider configured") + + if namespace == "tools": + if op == "list_tools": + return await self._provider.list_tools() + if op == "search_tools": + return await self._provider.search_tools(args["query"], int(args.get("limit", 10))) + if op == "list_tool_recipes": + return await self._provider.list_tool_recipes(args["name"]) + if op == "call_tool": + return await self._provider.call_tool(args["name"], args.get("args") or {}) + raise ValueError(f"unknown tools op: {op}") + + if namespace == "workflows": + if op == "list_workflows": + return await self._provider.list_workflows() + if op == "search_workflows": + return await self._provider.search_workflows( + args["query"], int(args.get("limit", 5)) + ) + if op == "get_workflow": + return await self._provider.get_workflow(args["name"]) + if op == "create_workflow": + return await self._provider.create_workflow( + args["name"], args["source"], args.get("description", "") + ) + if op == "delete_workflow": + return await self._provider.delete_workflow(args["name"]) + raise ValueError(f"unknown workflows op: {op}") + + if namespace == "artifacts": + if op == "list_artifacts": + return await self._provider.list_artifacts() + if op == "get_artifact": + return await self._provider.get_artifact(args["name"]) + if op == "artifact_exists": + return await self._provider.artifact_exists(args["name"]) + if op == "load_artifact": + return await self._provider.load_artifact(args["name"]) + if op == "save_artifact": + return await self._provider.save_artifact( + args["name"], args.get("data"), args.get("description", "") + ) + if op == "delete_artifact": + await self._provider.delete_artifact(args["name"]) + return None + raise ValueError(f"unknown artifacts op: {op}") + + if namespace == "deps": + # Persistence only; sandbox performs actual installs best-effort. + if self._deps_store is None: + return [] if op == "list_deps" else False + + if op == "list_deps": + return self._deps_store.list() + + if op == "persist_add": + if not self._config.allow_runtime_deps: + raise RuntimeDepsDisabledError( + "Runtime dependency installation is disabled. " + "Dependencies must be pre-configured before session start." + ) + spec = args["spec"] + self._deps_store.add(spec) + return True + + if op == "persist_remove": + if not self._config.allow_runtime_deps: + raise RuntimeDepsDisabledError( + "Runtime dependency modification is disabled. " + "Dependencies must be pre-configured before session start." + ) + name = args["spec_or_name"] + return self._deps_store.remove(name) + + raise ValueError(f"unknown deps op: {op}") + + raise ValueError(f"unknown rpc namespace: {namespace}") + + async def _send(self, msg: dict[str, Any]) -> None: + if self._proc is None or self._proc.stdin is None: + raise RuntimeError("Runner not started") + data = (json.dumps(msg, ensure_ascii=True) + "\n").encode("utf-8") + self._proc.stdin.write(data) + await self._proc.stdin.drain() + + async def _request(self, msg: dict[str, Any], timeout: float | None) -> dict[str, Any]: + req_id = msg.get("id") + if not isinstance(req_id, str): + raise TypeError("request message must include string id") + fut: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() + self._pending[req_id] = _Pending(fut=fut) + await self._send(msg) + + if timeout is None: + return await fut + return await asyncio.wait_for(fut, timeout=timeout) + + async def run(self, code: str, timeout: float | None = None) -> ExecutionResult: + if self._closed: + return ExecutionResult(value=None, stdout="", error="Executor is closed") + if self._proc is None: + await self.start(storage=self._storage) + + if self._wedged: + return ExecutionResult( + value=None, + stdout="", + error=( + "Previous execution timed out; sandbox may still be running. " + "Call session.reset()." + ), + ) + + effective_timeout = timeout if timeout is not None else self._config.default_timeout + req_id = uuid.uuid4().hex + + try: + res = await self._request( + {"id": req_id, "type": "exec", "code": code}, + timeout=effective_timeout, + ) + except TimeoutError: + self._wedged = True + return ExecutionResult( + value=None, + stdout="", + error=( + f"Execution timeout after {effective_timeout} seconds " + "(soft timeout; state preserved if it finishes)." + ), + ) + + if res.get("type") != "exec_result": + return ExecutionResult( + value=None, stdout="", error=f"Unexpected runner response: {res!r}" + ) + + return ExecutionResult( + value=res.get("value"), + stdout=res.get("stdout") or "", + error=res.get("error"), + ) + + async def reset(self) -> None: + # Hard reset: restart the Deno subprocess to guarantee recovery. + await self._restart() + + async def _restart(self) -> None: + await self.close() + self._closed = False + self._proc = None + self._stdout_task = None + self._pending.clear() + self._wedged = False + await self.start(storage=self._storage) + + async def close(self) -> None: + if self._proc is not None: + try: + await self._send({"id": uuid.uuid4().hex, "type": "close"}) + except Exception: + pass + try: + self._proc.terminate() + except ProcessLookupError: + pass + try: + await asyncio.wait_for(self._proc.wait(), timeout=2.0) + except Exception: + pass + if self._stdout_task is not None: + self._stdout_task.cancel() + if self._tool_registry is not None: + await self._tool_registry.close() + self._proc = None + self._stdout_task = None + self._closed = True + self._pending.clear() + self._wedged = False + + # ------------------------------------------------------------------------- + # Tools facade (host-side) + # ------------------------------------------------------------------------- + + async def list_tools(self) -> list[dict[str, Any]]: + if self._tool_registry is None: + return [] + return [tool.to_dict() for tool in self._tool_registry.list_tools()] + + async def search_tools(self, query: str, limit: int = 10) -> list[dict[str, Any]]: + if self._tool_registry is None: + return [] + return [tool.to_dict() for tool in self._tool_registry.search(query, limit=limit)] + + # ------------------------------------------------------------------------- + # Deps facade (host persistence + sandbox best-effort install via exec) + # ------------------------------------------------------------------------- + + async def install_deps(self, packages: list[str]) -> dict[str, Any]: + # Best-effort install is not implemented yet (no micropip integration). + return {"installed": [], "already_present": [], "failed": list(packages)} + + async def uninstall_deps(self, packages: list[str]) -> dict[str, Any]: + # Pyodide does not reliably support uninstall. We treat this as config removal only. + removed: list[str] = [] + failed: list[str] = [] + for pkg in packages: + try: + await self.remove_dep(pkg) + removed.append(pkg) + except Exception: + failed.append(pkg) + return {"removed": removed, "not_found": [], "failed": failed} + + async def add_dep(self, package: str) -> dict[str, Any]: + if not self._config.allow_runtime_deps: + raise RuntimeDepsDisabledError( + "Runtime dependency installation is disabled. " + "Dependencies must be pre-configured before session start." + ) + if self._deps_store is not None: + self._deps_store.add(package) + return {"installed": [], "already_present": [], "failed": [package]} + + async def remove_dep(self, package: str) -> dict[str, Any]: + if not self._config.allow_runtime_deps: + raise RuntimeDepsDisabledError( + "Runtime dependency modification is disabled. " + "Dependencies must be pre-configured before session start." + ) + removed_from_config = False + if self._deps_store is not None: + removed_from_config = self._deps_store.remove(package) + return { + "removed": [package] if removed_from_config else [], + "not_found": [] if removed_from_config else [package], + "failed": [], + "removed_from_config": removed_from_config, + } + + async def list_deps(self) -> list[str]: + if self._deps_store is None: + return [] + return self._deps_store.list() + + async def sync_deps(self) -> dict[str, Any]: + if self._deps_store is None: + return {"installed": [], "already_present": [], "failed": []} + return { + "installed": [], + "already_present": [], + "failed": [f"Deps sync not implemented (configured: {len(self._deps_store.list())})"], + } + + +register_backend("deno-pyodide", DenoPyodideExecutor) diff --git a/src/py_code_mode/execution/deno_pyodide/runner/main.ts b/src/py_code_mode/execution/deno_pyodide/runner/main.ts new file mode 100644 index 0000000..0f9173f --- /dev/null +++ b/src/py_code_mode/execution/deno_pyodide/runner/main.ts @@ -0,0 +1,223 @@ +// NDJSON protocol runner: host (Python) <-> Deno main <-> Pyodide worker. +// +// This is dependency-light. The only external dependency should come from +// worker.ts (npm:pyodide). We rely on `deno cache` being run outside the +// sandbox so the sandboxed runner can use `--cached-only` and `--deny-net`. + +type Req = + | { id: string; type: "init" } + | { id: string; type: "exec"; code: string } + | { id: string; type: "reset" } + | { id: string; type: "close" } + | { id: string; type: "rpc_response"; ok: boolean; result?: unknown; error?: unknown }; + +type Resp = + | { id: string; type: "ready" } + | { id: string; type: "exec_result"; stdout: string; value: unknown; error: string | null } + | { id: string; type: "rpc_request"; namespace: string; op: string; args: Record } + | { id: string; type: "error"; message: string }; + +function writeMsg(msg: Resp) { + Deno.stdout.writeSync(new TextEncoder().encode(JSON.stringify(msg) + "\n")); +} + +async function* readNdjsonLines(): AsyncGenerator { + const decoder = new TextDecoder(); + let buf = ""; + for await (const chunk of Deno.stdin.readable) { + buf += decoder.decode(chunk, { stream: true }); + while (true) { + const idx = buf.indexOf("\n"); + if (idx < 0) break; + const line = buf.slice(0, idx); + buf = buf.slice(idx + 1); + yield line; + } + } + if (buf) yield buf; +} + +// Shared buffers for synchronous RPC from the Pyodide worker. +// Single-flight: one outstanding RPC at a time. +const rpcState = new Int32Array(new SharedArrayBuffer(8)); // [status, length] +const rpcBuf = new Uint8Array(new SharedArrayBuffer(1024 * 1024)); // 1MB payload +// status: 0 = idle, 1 = response ready +let currentRpcId: string | null = null; + +function rpcWriteResponse(rpcId: string, payload: unknown) { + if (currentRpcId !== rpcId) { + writeMsg({ id: rpcId, type: "error", message: `rpc id mismatch: expected ${currentRpcId}, got ${rpcId}` }); + return; + } + const data = new TextEncoder().encode(JSON.stringify(payload ?? null)); + const out = data.length > rpcBuf.byteLength + ? new TextEncoder().encode(JSON.stringify({ ok: false, error: { type: "RPCTransportError", message: "rpc payload too large" } })) + : data; + + rpcBuf.set(out.subarray(0, rpcBuf.byteLength)); + rpcState[1] = out.length; + rpcState[0] = 1; + Atomics.notify(rpcState, 0, 1); + currentRpcId = null; +} + +function newWorker(): Worker { + const worker = new Worker(new URL("./worker.ts", import.meta.url), { + type: "module", + deno: { namespace: true }, + }); + worker.postMessage({ + type: "boot", + rpcState, + rpcBuf, + }); + return worker; +} + +let worker: Worker | null = null; +let workerBootError: string | null = null; +let workerReady = false; +let bootWait: Promise | null = null; +let bootResolve: (() => void) | null = null; +let bootReject: ((e: unknown) => void) | null = null; + +const execPending = new Map void; reject: (e: any) => void }>(); + +function ensureWorker() { + if (!worker) worker = newWorker(); +} + +function resetWorker() { + if (worker) worker.terminate(); + worker = newWorker(); + workerBootError = null; + workerReady = false; + bootWait = new Promise((resolve, reject) => { + bootResolve = resolve; + bootReject = reject; + }); +} + +function callExec(id: string, code: string): Promise { + ensureWorker(); + return new Promise((resolve, reject) => { + if (workerBootError) return reject(new Error(workerBootError)); + execPending.set(id, { resolve, reject }); + worker!.postMessage({ type: "exec", id, code }); + }); +} + +function attachWorkerHandler() { + if (!worker) return; + worker.onmessage = (ev: MessageEvent) => { + const msg = ev.data; + if (!msg || typeof msg !== "object") return; + + if (msg.type === "boot_ok") { + workerReady = true; + bootResolve?.(); + bootResolve = null; + bootReject = null; + return; + } + + if (msg.type === "boot_error") { + workerBootError = String(msg.error ?? "boot failed"); + workerReady = false; + bootReject?.(new Error(workerBootError)); + bootResolve = null; + bootReject = null; + for (const [id, pending] of execPending) { + pending.reject(new Error(workerBootError)); + execPending.delete(id); + } + writeMsg({ id: "worker", type: "error", message: workerBootError }); + return; + } + + if (msg.type === "exec_result") { + const pending = execPending.get(msg.id); + if (pending) { + execPending.delete(msg.id); + pending.resolve(msg); + } + return; + } + + if (msg.type === "rpc_request") { + currentRpcId = msg.id; + writeMsg({ id: msg.id, type: "rpc_request", namespace: msg.namespace, op: msg.op, args: msg.args }); + return; + } + }; + + worker.onerror = (e) => { + const m = String((e as any).message ?? e); + workerBootError = m; + workerReady = false; + bootReject?.(new Error(m)); + bootResolve = null; + bootReject = null; + for (const [id, pending] of execPending) { + pending.reject(new Error(m)); + execPending.delete(id); + } + writeMsg({ id: "worker", type: "error", message: m }); + }; +} + +for await (const line of readNdjsonLines()) { + if (!line.trim()) continue; + let req: Req; + try { + req = JSON.parse(line); + } catch (e) { + writeMsg({ id: "parse", type: "error", message: `invalid json: ${String(e)}` }); + continue; + } + + try { + if (req.type === "init") { + resetWorker(); + attachWorkerHandler(); + if (bootWait) await bootWait; + writeMsg({ id: req.id, type: "ready" }); + continue; + } + + if (req.type === "reset") { + resetWorker(); + attachWorkerHandler(); + if (bootWait) await bootWait; + writeMsg({ id: req.id, type: "ready" }); + continue; + } + + if (req.type === "close") { + if (worker) worker.terminate(); + worker = null; + writeMsg({ id: req.id, type: "ready" }); + break; + } + + if (req.type === "rpc_response") { + rpcWriteResponse(req.id, req.ok ? { ok: true, result: req.result } : { ok: false, error: req.error }); + continue; + } + + if (req.type === "exec") { + if (bootWait) await bootWait; + const res = await callExec(req.id, req.code); + writeMsg({ + id: req.id, + type: "exec_result", + stdout: res.stdout ?? "", + value: res.value ?? null, + error: res.error ?? null, + }); + continue; + } + } catch (e) { + writeMsg({ id: req.id, type: "error", message: String((e as any)?.stack ?? e) }); + } +} diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts new file mode 100644 index 0000000..42f3412 --- /dev/null +++ b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts @@ -0,0 +1,257 @@ +// Pyodide worker. Executes Python code and provides synchronous RPC calls +// back to the Deno main thread (which forwards to the Python host). +// +// Single-flight RPC: one outstanding call at a time. + +import { loadPyodide } from "npm:pyodide@0.29.3"; + +type BootMsg = { + type: "boot"; + rpcState: Int32Array; + rpcBuf: Uint8Array; +}; + +type ExecMsg = { type: "exec"; id: string; code: string }; + +let rpcState: Int32Array | null = null; +let rpcBuf: Uint8Array | null = null; + +function rpcCallSync(namespace: string, op: string, args: Record): string { + if (!rpcState || !rpcBuf) throw new Error("rpc not initialized"); + const id = crypto.randomUUID(); + (self as any).postMessage({ type: "rpc_request", id, namespace, op, args }); + + while (Atomics.load(rpcState, 0) === 0) { + Atomics.wait(rpcState, 0, 0, 60_000); + } + + const len = Atomics.load(rpcState, 1); + const bytes = rpcBuf.subarray(0, len); + const txt = new TextDecoder().decode(bytes); + Atomics.store(rpcState, 0, 0); + Atomics.store(rpcState, 1, 0); + return txt; +} + +function pyodidePackageDir(): string { + // Resolve a file inside the npm package, then derive its directory path. + const resolved = import.meta.resolve("npm:pyodide@0.29.3/pyodide.asm.js"); + if (resolved.startsWith("file:")) { + const u = new URL(resolved); + // Deno uses POSIX paths on macOS/Linux. + const path = decodeURIComponent(u.pathname); + return path.slice(0, path.lastIndexOf("/") + 1); + } + // Fallback: treat as path-like. + const s = String(resolved); + return s.slice(0, s.lastIndexOf("/") + 1); +} + +let pyodide: any = null; +let booted = false; + +async function ensureBooted() { + if (booted) return; + + // Use a filesystem path for indexURL to avoid "file:/..." pseudo-path issues. + const indexURL = pyodidePackageDir(); + pyodide = await loadPyodide({ indexURL }); + + (self as any).rpc_call_sync = rpcCallSync; + + const bootstrap = ` +import ast, io, json, sys + +class _RPC: + @staticmethod + def call(namespace: str, op: str, args: dict): + import js + payload = js.rpc_call_sync(namespace, op, args) + obj = json.loads(str(payload)) + if not obj.get("ok", False): + err = obj.get("error") or {} + raise RuntimeError(f"RPCError: {err.get('message','unknown')}") + return obj.get("result") + +class _ToolCallable: + def __init__(self, name: str, recipe: str | None = None): + self._name = name + self._recipe = recipe + def __call__(self, **kwargs): + tool = self._name if self._recipe is None else f"{self._name}.{self._recipe}" + return _RPC.call("tools", "call_tool", {"name": tool, "args": kwargs}) + +class _Tool: + def __init__(self, name: str): + self._name = name + def __call__(self, **kwargs): + return _RPC.call("tools", "call_tool", {"name": self._name, "args": kwargs}) + def __getattr__(self, recipe: str): + if recipe.startswith("_"): + raise AttributeError(recipe) + return _ToolCallable(self._name, recipe) + def recipes(self): + return _RPC.call("tools", "list_tool_recipes", {"name": self._name}) + +class tools: + def __getattr__(self, name: str): + if name.startswith("_"): + raise AttributeError(name) + return _Tool(name) + @staticmethod + def list(): + return _RPC.call("tools", "list_tools", {}) + @staticmethod + def search(query: str, limit: int = 5): + return _RPC.call("tools", "search_tools", {"query": query, "limit": limit}) + +class workflows: + @staticmethod + def list(): + return _RPC.call("workflows", "list_workflows", {}) + @staticmethod + def search(query: str, limit: int = 5): + return _RPC.call("workflows", "search_workflows", {"query": query, "limit": limit}) + @staticmethod + def get(name: str): + return _RPC.call("workflows", "get_workflow", {"name": name}) + @staticmethod + def create(name: str, source: str, description: str): + return _RPC.call("workflows", "create_workflow", {"name": name, "source": source, "description": description}) + @staticmethod + def delete(name: str): + return _RPC.call("workflows", "delete_workflow", {"name": name}) + +class artifacts: + @staticmethod + def list(): + return _RPC.call("artifacts", "list_artifacts", {}) + @staticmethod + def get(name: str): + return _RPC.call("artifacts", "get_artifact", {"name": name}) + @staticmethod + def exists(name: str): + return _RPC.call("artifacts", "artifact_exists", {"name": name}) + @staticmethod + def load(name: str): + return _RPC.call("artifacts", "load_artifact", {"name": name}) + @staticmethod + def save(name: str, data, description: str = ""): + return _RPC.call("artifacts", "save_artifact", {"name": name, "data": data, "description": description}) + @staticmethod + def delete(name: str): + return _RPC.call("artifacts", "delete_artifact", {"name": name}) + +class deps: + @staticmethod + def list(): + return _RPC.call("deps", "list_deps", {}) + @staticmethod + def add(spec: str): + _RPC.call("deps", "persist_add", {"spec": spec}) + return {"installed": [], "already_present": [], "failed": ["deps.add not implemented (micropip TBD)"]} + @staticmethod + def remove(spec_or_name: str): + removed = _RPC.call("deps", "persist_remove", {"spec_or_name": spec_or_name}) + return { + "removed": [spec_or_name] if removed else [], + "not_found": [] if removed else [spec_or_name], + "failed": [], + "removed_from_config": bool(removed), + } + @staticmethod + def sync(): + specs = _RPC.call("deps", "list_deps", {}) or [] + return {"installed": [], "already_present": [], "failed": [f"deps.sync not implemented (configured: {len(specs)})"]} +`; + + pyodide.runPython(bootstrap); + booted = true; +} + +function runWithLastExpr(code: string): { stdout: string; value: any; error: string | null } { + const wrapper = ` +import ast, io, sys, traceback +_stdout = io.StringIO() +_value = None +_error = None +try: + _tree = ast.parse(_CODE) + if _tree.body and isinstance(_tree.body[-1], ast.Expr): + _stmts = _tree.body[:-1] + _expr = _tree.body[-1] + if _stmts: + _m = ast.Module(body=_stmts, type_ignores=[]) + _c = compile(_m, "", "exec") + _old = sys.stdout + sys.stdout = _stdout + try: + exec(_c, globals()) + finally: + sys.stdout = _old + _ec = compile(ast.Expression(body=_expr.value), "", "eval") + _old = sys.stdout + sys.stdout = _stdout + try: + _value = eval(_ec, globals()) + finally: + sys.stdout = _old + else: + _old = sys.stdout + sys.stdout = _stdout + try: + exec(_CODE, globals()) + finally: + sys.stdout = _old + _value = None +except Exception: + _error = traceback.format_exc() +`; + + if (!pyodide) throw new Error("pyodide not initialized"); + pyodide.globals.set("_CODE", code); + pyodide.runPython(wrapper); + const stdout = pyodide.globals.get("_stdout").getvalue(); + const error = pyodide.globals.get("_error"); + + let outValue: any; + try { + const jsonTxt = pyodide.runPython("import json; json.dumps(_value)") as string; + outValue = JSON.parse(String(jsonTxt)); + } catch { + const repr = pyodide.runPython("repr(_value)") as string; + outValue = { __py_repr__: String(repr) }; + } + + return { stdout: String(stdout ?? ""), value: outValue ?? null, error: error ? String(error) : null }; +} + +self.onmessage = async (ev: MessageEvent) => { + const msg = ev.data; + if (msg.type === "boot") { + rpcState = msg.rpcState; + rpcBuf = msg.rpcBuf; + try { + await ensureBooted(); + (self as any).postMessage({ type: "boot_ok" }); + } catch (e) { + (self as any).postMessage({ type: "boot_error", error: String((e as any)?.stack ?? e) }); + } + return; + } + + if (msg.type === "exec") { + try { + const res = runWithLastExpr(msg.code); + (self as any).postMessage({ type: "exec_result", id: msg.id, ...res }); + } catch (e) { + (self as any).postMessage({ + type: "exec_result", + id: msg.id, + stdout: "", + value: null, + error: String((e as any)?.stack ?? e), + }); + } + } +}; diff --git a/src/py_code_mode/execution/resource_provider.py b/src/py_code_mode/execution/resource_provider.py new file mode 100644 index 0000000..68f2eaa --- /dev/null +++ b/src/py_code_mode/execution/resource_provider.py @@ -0,0 +1,288 @@ +"""Host-side ResourceProvider for sandboxed executors. + +This module contains the provider used by isolated runtimes to access +tools/workflows/artifacts/deps via RPC. + +Today SubprocessExecutor uses this to service kernel-side proxy namespaces. +The Deno/Pyodide executor will reuse the same provider to avoid duplicating +behavior across backends. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from py_code_mode.deps import DepsStore +from py_code_mode.tools import ToolRegistry + +if TYPE_CHECKING: + from py_code_mode.execution.subprocess.venv import KernelVenv, VenvManager + from py_code_mode.storage.backends import StorageBackend + +logger = logging.getLogger(__name__) + + +class StorageResourceProvider: + """ResourceProvider that bridges RPC to storage backend. + + Delegates to: + - tool_registry (executor-owned, from config.tools_path) + - storage backend (workflows + artifacts) + - deps_store (executor-owned, from config.deps/deps_file) + + Notes: + - For SubprocessExecutor, deps installation uses VenvManager. + - For other executors (e.g., Deno/Pyodide), deps ops may be mediated + differently; those executors can ignore the venv fields or provide their own. + """ + + def __init__( + self, + storage: StorageBackend, + tool_registry: ToolRegistry | None = None, + deps_store: DepsStore | None = None, + allow_runtime_deps: bool = True, + venv_manager: VenvManager | None = None, + venv: KernelVenv | None = None, + ) -> None: + self._storage = storage + self._tool_registry = tool_registry + self._deps_store = deps_store + self._allow_runtime_deps = allow_runtime_deps + self._venv_manager = venv_manager + self._venv = venv + self._workflow_library = None # lazy + + def _get_tool_registry(self) -> ToolRegistry | None: + return self._tool_registry + + def _get_workflow_library(self): + if self._workflow_library is None: + self._workflow_library = self._storage.get_workflow_library() + return self._workflow_library + + # ------------------------------------------------------------------------- + # Tool methods + # ------------------------------------------------------------------------- + + async def call_tool(self, name: str, args: dict[str, Any]) -> Any: + registry = self._get_tool_registry() + if registry is None: + raise ValueError("No tools configured") + + if "." in name: + tool_name, recipe_name = name.split(".", 1) + else: + tool_name = name + recipe_name = None + + adapter = registry.find_adapter_for_tool(tool_name) + if adapter is None: + raise ValueError(f"Unknown tool: {tool_name}") + + return await adapter.call_tool(tool_name, recipe_name, args) + + async def list_tools(self) -> list[dict[str, Any]]: + registry = self._get_tool_registry() + if registry is None: + return [] + return [tool.to_dict() for tool in registry.list_tools()] + + async def search_tools(self, query: str, limit: int) -> list[dict[str, Any]]: + registry = self._get_tool_registry() + if registry is None: + return [] + return [tool.to_dict() for tool in registry.search(query, limit=limit)] + + async def list_tool_recipes(self, name: str) -> list[dict[str, Any]]: + registry = self._get_tool_registry() + if registry is None: + raise ValueError("No tools configured") + + all_tools = registry.get_all_tools() + tool = next((t for t in all_tools if t.name == name), None) + if tool is None: + raise ValueError(f"Unknown tool: {name}") + + return [{"name": c.name, "description": c.description or ""} for c in tool.callables] + + # ------------------------------------------------------------------------- + # Workflow methods + # ------------------------------------------------------------------------- + + async def search_workflows(self, query: str, limit: int) -> list[dict[str, Any]]: + library = self._get_workflow_library() + library.refresh() + workflows = library.search(query, limit=limit) + return [ + { + "name": w.name, + "description": w.description, + "params": {p.name: p.description or p.type for p in w.parameters}, + } + for w in workflows + ] + + async def list_workflows(self) -> list[dict[str, Any]]: + library = self._get_workflow_library() + library.refresh() + workflows = library.list() + return [ + { + "name": w.name, + "description": w.description, + "params": {p.name: p.description or p.type for p in w.parameters}, + } + for w in workflows + ] + + async def get_workflow(self, name: str) -> dict[str, Any] | None: + library = self._get_workflow_library() + library.refresh() + workflow = library.get(name) + if workflow is None: + return None + return { + "name": workflow.name, + "description": workflow.description, + "source": workflow.source, + "params": {p.name: p.description or p.type for p in workflow.parameters}, + } + + async def create_workflow(self, name: str, source: str, description: str) -> dict[str, Any]: + from py_code_mode.workflows import PythonWorkflow + + workflow = PythonWorkflow.from_source( + name=name, + source=source, + description=description, + ) + + library = self._get_workflow_library() + library.add(workflow) + + return { + "name": workflow.name, + "description": workflow.description, + "params": {p.name: p.description or p.type for p in workflow.parameters}, + } + + async def delete_workflow(self, name: str) -> bool: + library = self._get_workflow_library() + return library.remove(name) + + # ------------------------------------------------------------------------- + # Artifact methods + # ------------------------------------------------------------------------- + + async def load_artifact(self, name: str) -> Any: + store = self._storage.get_artifact_store() + return store.load(name) + + async def save_artifact(self, name: str, data: Any, description: str) -> dict[str, Any]: + store = self._storage.get_artifact_store() + artifact = store.save(name, data, description=description) + return { + "name": artifact.name, + "path": artifact.path, + "description": artifact.description, + "created_at": artifact.created_at.isoformat(), + } + + async def list_artifacts(self) -> list[dict[str, Any]]: + store = self._storage.get_artifact_store() + artifacts = store.list() + return [ + { + "name": a.name, + "path": a.path, + "description": a.description, + "created_at": a.created_at.isoformat(), + } + for a in artifacts + ] + + async def delete_artifact(self, name: str) -> None: + store = self._storage.get_artifact_store() + store.delete(name) + + async def artifact_exists(self, name: str) -> bool: + store = self._storage.get_artifact_store() + return store.exists(name) + + async def get_artifact(self, name: str) -> dict[str, Any] | None: + store = self._storage.get_artifact_store() + artifact = store.get(name) + if artifact is None: + return None + return { + "name": artifact.name, + "path": artifact.path, + "description": artifact.description, + "created_at": artifact.created_at.isoformat(), + } + + # ------------------------------------------------------------------------- + # Deps methods (host-side persistence + optional venv install hook) + # ------------------------------------------------------------------------- + + async def add_dep(self, package: str) -> dict[str, Any]: + if not self._allow_runtime_deps: + raise RuntimeError( + "RuntimeDepsDisabledError: Runtime dependency installation is disabled. " + "Dependencies must be pre-configured before session start." + ) + + if self._deps_store is not None: + self._deps_store.add(package) + + # SubprocessExecutor path: also install into venv if available. + if self._venv_manager is not None and self._venv is not None: + try: + await self._venv_manager.add_package(self._venv, package) + return {"installed": [package], "already_present": [], "failed": []} + except Exception as e: + logger.warning("Failed to install %s: %s", package, e) + return {"installed": [], "already_present": [], "failed": [package]} + + return {"installed": [package], "already_present": [], "failed": []} + + async def remove_dep(self, package: str) -> bool: + if not self._allow_runtime_deps: + raise RuntimeError( + "RuntimeDepsDisabledError: Runtime dependency modification is disabled. " + "Dependencies must be pre-configured before session start." + ) + + if self._deps_store is None: + return False + return self._deps_store.remove(package) + + async def list_deps(self) -> list[str]: + if self._deps_store is None: + return [] + return self._deps_store.list() + + async def sync_deps(self) -> dict[str, Any]: + if self._deps_store is None: + return {"installed": [], "already_present": [], "failed": []} + + packages = self._deps_store.list() + if not packages: + return {"installed": [], "already_present": [], "failed": []} + + if self._venv_manager is None or self._venv is None: + return {"installed": packages, "already_present": [], "failed": []} + + installed: list[str] = [] + failed: list[str] = [] + for pkg in packages: + try: + await self._venv_manager.add_package(self._venv, pkg) + installed.append(pkg) + except Exception as e: + logger.warning("Failed to install %s: %s", pkg, e) + failed.append(pkg) + + return {"installed": installed, "already_present": [], "failed": failed} diff --git a/src/py_code_mode/execution/subprocess/executor.py b/src/py_code_mode/execution/subprocess/executor.py index f3c234d..aaad9d2 100644 --- a/src/py_code_mode/execution/subprocess/executor.py +++ b/src/py_code_mode/execution/subprocess/executor.py @@ -31,6 +31,7 @@ validate_storage_not_access, ) from py_code_mode.execution.registry import register_backend +from py_code_mode.execution.resource_provider import StorageResourceProvider from py_code_mode.execution.subprocess.config import SubprocessConfig from py_code_mode.execution.subprocess.host import KernelHost from py_code_mode.execution.subprocess.venv import KernelVenv, VenvManager @@ -64,324 +65,6 @@ def _deserialize_value(text_repr: str | None) -> Any: return text_repr -class StorageResourceProvider: - """ResourceProvider that bridges RPC to storage backend. - - This class implements the ResourceProvider protocol by delegating - to the storage backend for workflows and artifacts, and using - executor-provided tool registry and deps store. - """ - - def __init__( - self, - storage: StorageBackend, - tool_registry: ToolRegistry | None = None, - deps_store: DepsStore | None = None, - allow_runtime_deps: bool = True, - venv_manager: VenvManager | None = None, - venv: KernelVenv | None = None, - ) -> None: - """Initialize provider. - - Args: - storage: Storage backend for workflows and artifacts. - tool_registry: Tool registry loaded from executor config. - deps_store: Deps store for dependency management. - allow_runtime_deps: Whether to allow deps.add() and deps.remove(). - venv_manager: VenvManager for package installation. - venv: KernelVenv for the current subprocess. - """ - self._storage = storage - self._tool_registry = tool_registry - self._deps_store = deps_store - self._allow_runtime_deps = allow_runtime_deps - self._venv_manager = venv_manager - self._venv = venv - # Cached workflow library (lazy initialized) - self._workflow_library = None - - def _get_tool_registry(self) -> ToolRegistry | None: - """Get tool registry. Already loaded at construction time.""" - return self._tool_registry - - def _get_workflow_library(self): - """Get workflow library, caching the result.""" - if self._workflow_library is None: - self._workflow_library = self._storage.get_workflow_library() - return self._workflow_library - - # ------------------------------------------------------------------------- - # Tool methods - # ------------------------------------------------------------------------- - - async def call_tool(self, name: str, args: dict[str, Any]) -> Any: - """Call a tool by name with given arguments. - - Supports both: - - Direct tool invocation: name="curl", args={...} - - Recipe invocation: name="curl.get", args={...} - """ - registry = self._get_tool_registry() - if registry is None: - raise ValueError("No tools configured") - - # Parse name for recipe syntax (e.g., "curl.get") - if "." in name: - tool_name, recipe_name = name.split(".", 1) - else: - tool_name = name - recipe_name = None - - # Find the tool's adapter - adapter = registry.find_adapter_for_tool(tool_name) - if adapter is None: - raise ValueError(f"Unknown tool: {tool_name}") - - # Call the tool - return await adapter.call_tool(tool_name, recipe_name, args) - - async def list_tools(self) -> list[dict[str, Any]]: - """List all available tools.""" - registry = self._get_tool_registry() - if registry is None: - return [] - return [tool.to_dict() for tool in registry.list_tools()] - - async def search_tools(self, query: str, limit: int) -> list[dict[str, Any]]: - """Search tools by query.""" - registry = self._get_tool_registry() - if registry is None: - return [] - return [tool.to_dict() for tool in registry.search(query, limit=limit)] - - async def list_tool_recipes(self, name: str) -> list[dict[str, Any]]: - """List recipes for a specific tool.""" - registry = self._get_tool_registry() - if registry is None: - raise ValueError("No tools configured") - all_tools = registry.get_all_tools() - tool = next((t for t in all_tools if t.name == name), None) - if tool is None: - raise ValueError(f"Unknown tool: {name}") - - return [ - { - "name": c.name, - "description": c.description or "", - } - for c in tool.callables - ] - - # ------------------------------------------------------------------------- - # Workflow methods - # ------------------------------------------------------------------------- - - async def search_workflows(self, query: str, limit: int) -> list[dict[str, Any]]: - """Search for workflows matching query.""" - library = self._get_workflow_library() - library.refresh() - workflows = library.search(query, limit=limit) - return [ - { - "name": w.name, - "description": w.description, - "params": {p.name: p.description or p.type for p in w.parameters}, - } - for w in workflows - ] - - async def list_workflows(self) -> list[dict[str, Any]]: - """List all available workflows.""" - library = self._get_workflow_library() - library.refresh() - workflows = library.list() - return [ - { - "name": w.name, - "description": w.description, - "params": {p.name: p.description or p.type for p in w.parameters}, - } - for w in workflows - ] - - async def get_workflow(self, name: str) -> dict[str, Any] | None: - """Get a workflow by name.""" - library = self._get_workflow_library() - library.refresh() - workflow = library.get(name) - if workflow is None: - return None - return { - "name": workflow.name, - "description": workflow.description, - "source": workflow.source, - "params": {p.name: p.description or p.type for p in workflow.parameters}, - } - - async def create_workflow(self, name: str, source: str, description: str) -> dict[str, Any]: - """Create and save a new workflow.""" - from py_code_mode.workflows import PythonWorkflow - - workflow = PythonWorkflow.from_source( - name=name, - source=source, - description=description, - ) - - library = self._get_workflow_library() - library.add(workflow) - - return { - "name": workflow.name, - "description": workflow.description, - "params": {p.name: p.description or p.type for p in workflow.parameters}, - } - - async def delete_workflow(self, name: str) -> bool: - """Delete a workflow.""" - library = self._get_workflow_library() - return library.remove(name) - - # ------------------------------------------------------------------------- - # Artifact methods - # ------------------------------------------------------------------------- - - async def load_artifact(self, name: str) -> Any: - """Load an artifact by name.""" - store = self._storage.get_artifact_store() - return store.load(name) - - async def save_artifact(self, name: str, data: Any, description: str) -> dict[str, Any]: - """Save an artifact.""" - store = self._storage.get_artifact_store() - artifact = store.save(name, data, description=description) - return { - "name": artifact.name, - "path": artifact.path, - "description": artifact.description, - "created_at": artifact.created_at.isoformat(), - } - - async def list_artifacts(self) -> list[dict[str, Any]]: - """List all artifacts.""" - store = self._storage.get_artifact_store() - artifacts = store.list() - return [ - { - "name": a.name, - "path": a.path, - "description": a.description, - "created_at": a.created_at.isoformat(), - } - for a in artifacts - ] - - async def delete_artifact(self, name: str) -> None: - """Delete an artifact.""" - store = self._storage.get_artifact_store() - store.delete(name) - - async def artifact_exists(self, name: str) -> bool: - """Check if an artifact exists.""" - store = self._storage.get_artifact_store() - return store.exists(name) - - async def get_artifact(self, name: str) -> dict[str, Any] | None: - """Get artifact metadata.""" - store = self._storage.get_artifact_store() - artifact = store.get(name) - if artifact is None: - return None - return { - "name": artifact.name, - "path": artifact.path, - "description": artifact.description, - "created_at": artifact.created_at.isoformat(), - } - - # ------------------------------------------------------------------------- - # Deps methods - # ------------------------------------------------------------------------- - - async def add_dep(self, package: str) -> dict[str, Any]: - """Add and install a package. - - When allow_runtime_deps=False, raises RuntimeError. - """ - if not self._allow_runtime_deps: - raise RuntimeError( - "RuntimeDepsDisabledError: Runtime dependency installation is disabled. " - "Dependencies must be pre-configured before session start." - ) - - # Add to deps store if configured - if self._deps_store is not None: - self._deps_store.add(package) - - # Install via venv manager if available - if self._venv_manager is not None and self._venv is not None: - try: - await self._venv_manager.add_package(self._venv, package) - return {"installed": [package], "already_present": [], "failed": []} - except Exception as e: - logger.warning("Failed to install %s: %s", package, e) - return {"installed": [], "already_present": [], "failed": [package]} - else: - # No venv manager - just report what we would install - return {"installed": [package], "already_present": [], "failed": []} - - async def remove_dep(self, package: str) -> bool: - """Remove a package from configuration. - - When allow_runtime_deps=False, raises RuntimeError. - """ - if not self._allow_runtime_deps: - raise RuntimeError( - "RuntimeDepsDisabledError: Runtime dependency modification is disabled. " - "Dependencies must be pre-configured before session start." - ) - - if self._deps_store is None: - return False - return self._deps_store.remove(package) - - async def list_deps(self) -> list[str]: - """List configured packages.""" - if self._deps_store is None: - return [] - return self._deps_store.list() - - async def sync_deps(self) -> dict[str, Any]: - """Install all configured packages. - - This is always allowed, even when allow_runtime_deps=False, - because it only installs pre-configured packages. - """ - if self._deps_store is None: - return {"installed": [], "already_present": [], "failed": []} - - packages = self._deps_store.list() - - if not packages: - return {"installed": [], "already_present": [], "failed": []} - - if self._venv_manager is None or self._venv is None: - return {"installed": packages, "already_present": [], "failed": []} - - installed: list[str] = [] - failed: list[str] = [] - - for pkg in packages: - try: - await self._venv_manager.add_package(self._venv, pkg) - installed.append(pkg) - except Exception as e: - logger.warning("Failed to install %s: %s", pkg, e) - failed.append(pkg) - - return {"installed": installed, "already_present": [], "failed": failed} - - class SubprocessExecutor: """Execute code in an isolated subprocess with its own venv and IPython kernel. diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py new file mode 100644 index 0000000..8c551c2 --- /dev/null +++ b/tests/test_deno_pyodide_executor.py @@ -0,0 +1,40 @@ +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + os.environ.get("PY_CODE_MODE_TEST_DENO") != "1", + reason="Set PY_CODE_MODE_TEST_DENO=1 to run Deno/Pyodide integration tests.", +) + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r1 = await session.run("1 + 1") + assert r1.error is None + assert r1.value == 2 + + r2 = await session.run("x = 40") + assert r2.error is None + + r3 = await session.run("x + 2") + assert r3.error is None + assert r3.value == 42 From a3879676e8cfbab51a7b72743455fdda4d535c88 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 01:15:45 -0800 Subject: [PATCH 02/27] Implement micropip deps install + network presets for Deno/Pyodide --- .../execution/deno_pyodide/config.py | 16 +- .../execution/deno_pyodide/executor.py | 66 +++++- .../execution/deno_pyodide/runner/main.ts | 114 +++++++--- .../execution/deno_pyodide/runner/worker.ts | 200 +++++++++++++++++- tests/test_deno_pyodide_executor.py | 81 +++++++ 5 files changed, 426 insertions(+), 51 deletions(-) diff --git a/src/py_code_mode/execution/deno_pyodide/config.py b/src/py_code_mode/execution/deno_pyodide/config.py index 6b47011..5a661bd 100644 --- a/src/py_code_mode/execution/deno_pyodide/config.py +++ b/src/py_code_mode/execution/deno_pyodide/config.py @@ -22,6 +22,7 @@ class DenoPyodideConfig: deps: tuple[str, ...] | None = None deps_file: Path | None = None ipc_timeout: float = 30.0 + deps_timeout: float | None = 300.0 deno_executable: str = "deno" # If None, the executor uses the packaged runner script adjacent to this module. @@ -32,6 +33,15 @@ class DenoPyodideConfig: # sandbox (host) via `deno cache`, then the sandbox runs with --cached-only. deno_dir: Path | None = None - # Deno network mode: keep sandbox `--deny-net` by default. - # Future: allowlist support for deps/networking use cases. - network_mode: str = "deny" # "deny" | "allow" | "allowlist" + # Network profile: + # - "none": deny all network access (no runtime dep installs) + # - "deps-only": allow just enough for micropip / pyodide package fetches + # - "full": allow all network access + network_profile: str = "full" # "none" | "deps-only" | "full" + + # Used when network_profile="deps-only". + deps_net_allowlist: tuple[str, ...] = ( + "pypi.org", + "files.pythonhosted.org", + "cdn.jsdelivr.net", + ) diff --git a/src/py_code_mode/execution/deno_pyodide/executor.py b/src/py_code_mode/execution/deno_pyodide/executor.py index 2066cd1..37d64ad 100644 --- a/src/py_code_mode/execution/deno_pyodide/executor.py +++ b/src/py_code_mode/execution/deno_pyodide/executor.py @@ -62,6 +62,7 @@ class DenoPyodideExecutor: Capability.TIMEOUT, Capability.PROCESS_ISOLATION, Capability.NETWORK_ISOLATION, + Capability.NETWORK_FILTERING, Capability.FILESYSTEM_ISOLATION, Capability.RESET, Capability.DEPS_INSTALL, @@ -73,6 +74,7 @@ def __init__(self, config: DenoPyodideConfig | None = None) -> None: self._config = config or DenoPyodideConfig() self._proc: Process | None = None self._stdout_task: asyncio.Task[None] | None = None + self._stderr_task: asyncio.Task[None] | None = None self._pending: dict[str, _Pending] = {} self._closed = False self._wedged = False # indicates a soft-timeout run may still be executing @@ -188,14 +190,22 @@ async def _spawn_runner(self) -> None: "--no-prompt", "--cached-only", "--location=https://pyodide.invalid/", - "--deny-write", "--deny-env", "--deny-run", ] - if self._config.network_mode == "allow": + # Pyodide caches wheels into the Deno npm cache under DENO_DIR. + # Allow writes only to that cache directory. + deno_args.append(f"--allow-write={deno_dir}") + profile = self._config.network_profile + if profile == "full": deno_args.append("--allow-net") - else: + elif profile == "deps-only": + allow = ",".join(self._config.deps_net_allowlist) + deno_args.append(f"--allow-net={allow}") + elif profile == "none": deno_args.append("--deny-net") + else: + raise ValueError(f"unknown network_profile: {profile!r}") deno_args.append(f"--allow-read={','.join(str(p) for p in allow_reads)}") deno_args.append(str(runner_path)) @@ -212,6 +222,7 @@ async def _spawn_runner(self) -> None: assert self._proc.stdout is not None self._stdout_task = asyncio.create_task(self._stdout_loop()) + self._stderr_task = asyncio.create_task(self._stderr_loop()) # Initialize runner with runtime directory and wait for ready. init_id = uuid.uuid4().hex @@ -253,6 +264,40 @@ async def _stdout_loop(self) -> None: if not pending.fut.done(): pending.fut.set_result(msg) + async def _stderr_loop(self) -> None: + assert self._proc is not None and self._proc.stderr is not None + reader = self._proc.stderr + while True: + line = await reader.readline() + if not line: + return + # Keep stderr drained to avoid deadlocks; log at debug to aid diagnosis. + logger.debug("deno stderr: %s", line.decode("utf-8", errors="replace").rstrip()) + + async def _deps_install(self, packages: list[str]) -> dict[str, Any]: + if not packages: + return {"installed": [], "already_present": [], "failed": []} + if self._proc is None: + await self.start(storage=self._storage) + + req_id = uuid.uuid4().hex + timeout = self._config.deps_timeout + res = await self._request( + {"id": req_id, "type": "deps_install", "packages": list(packages)}, + timeout=timeout, + ) + if res.get("type") != "deps_install_result": + return { + "installed": [], + "already_present": [], + "failed": [f"Unexpected runner response: {res!r}"], + } + return { + "installed": list(res.get("installed") or []), + "already_present": list(res.get("already_present") or []), + "failed": list(res.get("failed") or []), + } + async def _handle_rpc_request(self, msg: dict[str, Any]) -> None: if self._proc is None or self._proc.stdin is None: return @@ -462,10 +507,13 @@ async def close(self) -> None: pass if self._stdout_task is not None: self._stdout_task.cancel() + if self._stderr_task is not None: + self._stderr_task.cancel() if self._tool_registry is not None: await self._tool_registry.close() self._proc = None self._stdout_task = None + self._stderr_task = None self._closed = True self._pending.clear() self._wedged = False @@ -489,8 +537,8 @@ async def search_tools(self, query: str, limit: int = 10) -> list[dict[str, Any] # ------------------------------------------------------------------------- async def install_deps(self, packages: list[str]) -> dict[str, Any]: - # Best-effort install is not implemented yet (no micropip integration). - return {"installed": [], "already_present": [], "failed": list(packages)} + # System-level API (used by Session._sync_deps()). + return await self._deps_install(packages) async def uninstall_deps(self, packages: list[str]) -> dict[str, Any]: # Pyodide does not reliably support uninstall. We treat this as config removal only. @@ -512,7 +560,7 @@ async def add_dep(self, package: str) -> dict[str, Any]: ) if self._deps_store is not None: self._deps_store.add(package) - return {"installed": [], "already_present": [], "failed": [package]} + return await self._deps_install([package]) async def remove_dep(self, package: str) -> dict[str, Any]: if not self._config.allow_runtime_deps: @@ -538,11 +586,7 @@ async def list_deps(self) -> list[str]: async def sync_deps(self) -> dict[str, Any]: if self._deps_store is None: return {"installed": [], "already_present": [], "failed": []} - return { - "installed": [], - "already_present": [], - "failed": [f"Deps sync not implemented (configured: {len(self._deps_store.list())})"], - } + return await self._deps_install(self._deps_store.list()) register_backend("deno-pyodide", DenoPyodideExecutor) diff --git a/src/py_code_mode/execution/deno_pyodide/runner/main.ts b/src/py_code_mode/execution/deno_pyodide/runner/main.ts index 0f9173f..ddd6b7b 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/main.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/main.ts @@ -7,6 +7,7 @@ type Req = | { id: string; type: "init" } | { id: string; type: "exec"; code: string } + | { id: string; type: "deps_install"; packages: string[] } | { id: string; type: "reset" } | { id: string; type: "close" } | { id: string; type: "rpc_response"; ok: boolean; result?: unknown; error?: unknown }; @@ -14,6 +15,7 @@ type Req = type Resp = | { id: string; type: "ready" } | { id: string; type: "exec_result"; stdout: string; value: unknown; error: string | null } + | { id: string; type: "deps_install_result"; installed: string[]; already_present: string[]; failed: string[] } | { id: string; type: "rpc_request"; namespace: string; op: string; args: Record } | { id: string; type: "error"; message: string }; @@ -107,6 +109,15 @@ function callExec(id: string, code: string): Promise { }); } +function callDepsInstall(id: string, packages: string[]): Promise { + ensureWorker(); + return new Promise((resolve, reject) => { + if (workerBootError) return reject(new Error(workerBootError)); + execPending.set(id, { resolve, reject }); + worker!.postMessage({ type: "deps_install", id, packages }); + }); +} + function attachWorkerHandler() { if (!worker) return; worker.onmessage = (ev: MessageEvent) => { @@ -144,6 +155,15 @@ function attachWorkerHandler() { return; } + if (msg.type === "deps_install_result") { + const pending = execPending.get(msg.id); + if (pending) { + execPending.delete(msg.id); + pending.resolve(msg); + } + return; + } + if (msg.type === "rpc_request") { currentRpcId = msg.id; writeMsg({ id: msg.id, type: "rpc_request", namespace: msg.namespace, op: msg.op, args: msg.args }); @@ -166,23 +186,27 @@ function attachWorkerHandler() { }; } -for await (const line of readNdjsonLines()) { - if (!line.trim()) continue; - let req: Req; - try { - req = JSON.parse(line); - } catch (e) { - writeMsg({ id: "parse", type: "error", message: `invalid json: ${String(e)}` }); - continue; - } +// Ensure exec/deps_install are single-flight, but don't block stdin processing: +// the worker needs rpc_response messages to arrive while code is running. +let runChain: Promise = Promise.resolve(); +function enqueueRun(fn: () => Promise) { + const p = runChain.then(fn, fn); + runChain = p.catch(() => {}); +} +async function handleReq(req: Req) { try { + if (req.type === "rpc_response") { + rpcWriteResponse(req.id, req.ok ? { ok: true, result: req.result } : { ok: false, error: req.error }); + return; + } + if (req.type === "init") { resetWorker(); attachWorkerHandler(); if (bootWait) await bootWait; writeMsg({ id: req.id, type: "ready" }); - continue; + return; } if (req.type === "reset") { @@ -190,34 +214,62 @@ for await (const line of readNdjsonLines()) { attachWorkerHandler(); if (bootWait) await bootWait; writeMsg({ id: req.id, type: "ready" }); - continue; - } - - if (req.type === "close") { - if (worker) worker.terminate(); - worker = null; - writeMsg({ id: req.id, type: "ready" }); - break; + return; } - if (req.type === "rpc_response") { - rpcWriteResponse(req.id, req.ok ? { ok: true, result: req.result } : { ok: false, error: req.error }); - continue; + if (req.type === "exec") { + enqueueRun(async () => { + if (bootWait) await bootWait; + const res = await callExec(req.id, req.code); + writeMsg({ + id: req.id, + type: "exec_result", + stdout: res.stdout ?? "", + value: res.value ?? null, + error: res.error ?? null, + }); + }); + return; } - if (req.type === "exec") { - if (bootWait) await bootWait; - const res = await callExec(req.id, req.code); - writeMsg({ - id: req.id, - type: "exec_result", - stdout: res.stdout ?? "", - value: res.value ?? null, - error: res.error ?? null, + if (req.type === "deps_install") { + enqueueRun(async () => { + if (bootWait) await bootWait; + const res = await callDepsInstall(req.id, req.packages ?? []); + writeMsg({ + id: req.id, + type: "deps_install_result", + installed: res.installed ?? [], + already_present: res.already_present ?? [], + failed: res.failed ?? [], + }); }); - continue; + return; } } catch (e) { writeMsg({ id: req.id, type: "error", message: String((e as any)?.stack ?? e) }); } } + +for await (const line of readNdjsonLines()) { + if (!line.trim()) continue; + let req: Req; + try { + req = JSON.parse(line); + } catch (e) { + writeMsg({ id: "parse", type: "error", message: `invalid json: ${String(e)}` }); + continue; + } + + if (req.type === "close") { + try { + if (worker) worker.terminate(); + worker = null; + writeMsg({ id: req.id, type: "ready" }); + } finally { + break; + } + } + + void handleReq(req); +} diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts index 42f3412..20c806e 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts @@ -5,6 +5,15 @@ import { loadPyodide } from "npm:pyodide@0.29.3"; +// Keep stdout/stderr reserved for the NDJSON protocol in main.ts. +// Pyodide and micropip can be chatty, and interleaved output would corrupt NDJSON. +const _noop = () => {}; +console.log = _noop; +console.info = _noop; +console.warn = _noop; +console.debug = _noop; +console.error = _noop; + type BootMsg = { type: "boot"; rpcState: Int32Array; @@ -12,6 +21,7 @@ type BootMsg = { }; type ExecMsg = { type: "exec"; id: string; code: string }; +type DepsInstallMsg = { type: "deps_install"; id: string; packages: string[] }; let rpcState: Int32Array | null = null; let rpcBuf: Uint8Array | null = null; @@ -33,6 +43,32 @@ function rpcCallSync(namespace: string, op: string, args: Record = {}; + try { + const parsed = JSON.parse(String(argsJson ?? "{}")); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + args = parsed as Record; + } + } catch { + // fall back to empty args + } + return rpcCallSync(namespace, op, args); +} + +function rpcCallSyncResult(namespace: string, op: string, args: Record): any { + const txt = rpcCallSync(namespace, op, args); + const obj = JSON.parse(txt); + if (!obj || typeof obj !== "object") { + throw new Error(`RPCTransportError: invalid rpc payload: ${txt.slice(0, 200)}`); + } + if (!obj.ok) { + const err = obj.error ?? {}; + throw new Error(`RPCError: ${String(err.message ?? err.type ?? "unknown")}`); + } + return obj.result; +} + function pyodidePackageDir(): string { // Resolve a file inside the npm package, then derive its directory path. const resolved = import.meta.resolve("npm:pyodide@0.29.3/pyodide.asm.js"); @@ -49,15 +85,21 @@ function pyodidePackageDir(): string { let pyodide: any = null; let booted = false; +let micropipLoaded = false; +const attemptedInstalls = new Set(); async function ensureBooted() { if (booted) return; // Use a filesystem path for indexURL to avoid "file:/..." pseudo-path issues. const indexURL = pyodidePackageDir(); - pyodide = await loadPyodide({ indexURL }); + // Silence Pyodide's own package-loader chatter; stdout for user code is captured + // explicitly in runWithLastExpr() by redirecting sys.stdout. + pyodide = await loadPyodide({ indexURL, stdout: _noop, stderr: _noop }); - (self as any).rpc_call_sync = rpcCallSync; + // Expose a JSON-args RPC to Python; Pyodide dicts become non-cloneable proxies + // if we try to postMessage them directly. + (self as any).rpc_call_sync = rpcCallSyncJsonArgs; const bootstrap = ` import ast, io, json, sys @@ -66,7 +108,7 @@ class _RPC: @staticmethod def call(namespace: str, op: str, args: dict): import js - payload = js.rpc_call_sync(namespace, op, args) + payload = js.rpc_call_sync(namespace, op, json.dumps(args)) obj = json.loads(str(payload)) if not obj.get("ok", False): err = obj.get("error") or {} @@ -149,7 +191,7 @@ class deps: @staticmethod def add(spec: str): _RPC.call("deps", "persist_add", {"spec": spec}) - return {"installed": [], "already_present": [], "failed": ["deps.add not implemented (micropip TBD)"]} + return {"installed": [spec], "already_present": [], "failed": []} @staticmethod def remove(spec_or_name: str): removed = _RPC.call("deps", "persist_remove", {"spec_or_name": spec_or_name}) @@ -162,13 +204,142 @@ class deps: @staticmethod def sync(): specs = _RPC.call("deps", "list_deps", {}) or [] - return {"installed": [], "already_present": [], "failed": [f"deps.sync not implemented (configured: {len(specs)})"]} + return {"installed": list(specs), "already_present": [], "failed": []} `; pyodide.runPython(bootstrap); booted = true; } +async function ensureMicropipLoaded(): Promise { + if (micropipLoaded) return; + if (!pyodide) throw new Error("pyodide not initialized"); + await pyodide.loadPackage("micropip"); + micropipLoaded = true; +} + +async function micropipInstallOne(spec: string): Promise { + if (!pyodide) throw new Error("pyodide not initialized"); + pyodide.globals.set("_PYCM_DEP_SPEC", spec); + // runPythonAsync supports top-level await. + await pyodide.runPythonAsync(` +import micropip +await micropip.install(_PYCM_DEP_SPEC) +`); +} + +async function installPackages(packages: string[]): Promise<{ + installed: string[]; + already_present: string[]; + failed: string[]; +}> { + const installed: string[] = []; + const already_present: string[] = []; + const failed: string[] = []; + + const unique: string[] = []; + for (const p of packages) { + const spec = String(p ?? "").trim(); + if (!spec) continue; + if (attemptedInstalls.has(spec)) { + already_present.push(spec); + continue; + } + unique.push(spec); + } + + if (!unique.length) return { installed, already_present, failed }; + + await ensureMicropipLoaded(); + + for (const spec of unique) { + try { + await micropipInstallOne(spec); + attemptedInstalls.add(spec); + installed.push(spec); + } catch (e) { + failed.push(spec); + // Keep going; this is best-effort. + // We intentionally do not mark attemptedInstalls on failure so a later retry + // (e.g., after network policy changes) can re-attempt. + } + } + + return { installed, already_present, failed }; +} + +function extractDepsRequests(code: string): { specs: string[]; wants_sync: boolean } { + if (!pyodide) throw new Error("pyodide not initialized"); + pyodide.globals.set("_PYCM_SCAN_CODE", code); + const txt = pyodide.runPython(` +import ast, json +try: + _tree = ast.parse(_PYCM_SCAN_CODE) +except Exception: + _out = json.dumps({"specs": [], "wants_sync": False}) +else: + class _V(ast.NodeVisitor): + def __init__(self): + self.specs = set() + self.wants_sync = False + # Skip descending into defs/classes entirely; those bodies aren't executed + # just by being present in a cell. + def visit_FunctionDef(self, node): return + def visit_AsyncFunctionDef(self, node): return + def visit_ClassDef(self, node): return + def visit_Lambda(self, node): return + + def visit_Call(self, node): + try: + f = node.func + if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name) and f.value.id == "deps": + if f.attr == "add" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): + self.specs.add(node.args[0].value) + elif f.attr == "sync" and not node.args and not node.keywords: + self.wants_sync = True + except Exception: + pass + self.generic_visit(node) + + _v = _V() + # Only scan top-level statements (and their non-def bodies). + for _stmt in getattr(_tree, "body", []): + _v.visit(_stmt) + _out = json.dumps({"specs": sorted(_v.specs), "wants_sync": bool(_v.wants_sync)}) + +_out +`); + const obj = JSON.parse(String(txt)); + return { specs: (obj.specs ?? []) as string[], wants_sync: Boolean(obj.wants_sync) }; +} + +async function maybeInstallDepsForCode(code: string): Promise { + const info = extractDepsRequests(code); + const specsToAdd = info.specs ?? []; + + for (const spec of specsToAdd) { + rpcCallSyncResult("deps", "persist_add", { spec }); + } + + let specsToInstall = specsToAdd; + if (info.wants_sync) { + const all = rpcCallSyncResult("deps", "list_deps", {}) ?? []; + specsToInstall = Array.isArray(all) ? all.map((s) => String(s)) : []; + } + + if (!specsToInstall.length) return; + + let res; + try { + res = await installPackages(specsToInstall); + } catch (e) { + throw new Error(`Failed to load micropip (network permission?): ${String((e as any)?.stack ?? e)}`); + } + if (res.failed.length) { + throw new Error(`Dependency install failed: ${res.failed.join(", ")}`); + } +} + function runWithLastExpr(code: string): { stdout: string; value: any; error: string | null } { const wrapper = ` import ast, io, sys, traceback @@ -226,7 +397,7 @@ except Exception: return { stdout: String(stdout ?? ""), value: outValue ?? null, error: error ? String(error) : null }; } -self.onmessage = async (ev: MessageEvent) => { +self.onmessage = async (ev: MessageEvent) => { const msg = ev.data; if (msg.type === "boot") { rpcState = msg.rpcState; @@ -242,6 +413,7 @@ self.onmessage = async (ev: MessageEvent) => { if (msg.type === "exec") { try { + await maybeInstallDepsForCode(msg.code); const res = runWithLastExpr(msg.code); (self as any).postMessage({ type: "exec_result", id: msg.id, ...res }); } catch (e) { @@ -254,4 +426,20 @@ self.onmessage = async (ev: MessageEvent) => { }); } } + + if (msg.type === "deps_install") { + try { + const res = await installPackages(msg.packages ?? []); + (self as any).postMessage({ type: "deps_install_result", id: msg.id, ...res }); + } catch (e) { + (self as any).postMessage({ + type: "deps_install_result", + id: msg.id, + installed: [], + already_present: [], + failed: (msg.packages ?? []).map((p) => String(p)), + error: String((e as any)?.stack ?? e), + }); + } + } }; diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index 8c551c2..d1e9ea3 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -24,6 +24,7 @@ async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, + network_profile="none", ) ) @@ -38,3 +39,83 @@ async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: r3 = await session.run("x + 2") assert r3.error is None assert r3.value == 42 + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=300.0, + deps_timeout=300.0, + ipc_timeout=120.0, + network_profile="deps-only", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run("deps.add('packaging')\nimport packaging\npackaging.__version__") + assert r.error is None + assert isinstance(r.value, str) + assert r.value + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + deps_timeout=300.0, + ipc_timeout=120.0, + deps=("packaging",), + network_profile="deps-only", + ) + ) + + async with Session(storage=storage, executor=executor, sync_deps_on_start=True) as session: + r = await session.run("import packaging\npackaging.__version__") + assert r.error is None + assert isinstance(r.value, str) + assert r.value + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=120.0, + deps_timeout=120.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run("deps.add('packaging')\nimport packaging\npackaging.__version__") + assert r.error is not None From 0a1f6719188e7b75aafa8ae65e03688ce3e087a7 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 01:16:23 -0800 Subject: [PATCH 03/27] Type network_profile as Literal --- src/py_code_mode/execution/deno_pyodide/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/py_code_mode/execution/deno_pyodide/config.py b/src/py_code_mode/execution/deno_pyodide/config.py index 5a661bd..d1d76d5 100644 --- a/src/py_code_mode/execution/deno_pyodide/config.py +++ b/src/py_code_mode/execution/deno_pyodide/config.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from pathlib import Path +from typing import Literal @dataclass(frozen=True) @@ -37,7 +38,7 @@ class DenoPyodideConfig: # - "none": deny all network access (no runtime dep installs) # - "deps-only": allow just enough for micropip / pyodide package fetches # - "full": allow all network access - network_profile: str = "full" # "none" | "deps-only" | "full" + network_profile: Literal["none", "deps-only", "full"] = "full" # Used when network_profile="deps-only". deps_net_allowlist: tuple[str, ...] = ( From 602f940ae240e96fc7aa36bea5b643a20f07ed33 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 01:58:43 -0800 Subject: [PATCH 04/27] Fix tools proxy + expand Deno/Pyodide executor integration tests --- .../execution/deno_pyodide/runner/worker.ts | 12 +- tests/test_deno_pyodide_executor.py | 211 ++++++++++++++++++ 2 files changed, 217 insertions(+), 6 deletions(-) diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts index 20c806e..9de6ea1 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts @@ -132,21 +132,21 @@ class _Tool: if recipe.startswith("_"): raise AttributeError(recipe) return _ToolCallable(self._name, recipe) - def recipes(self): + def list(self): return _RPC.call("tools", "list_tool_recipes", {"name": self._name}) -class tools: +class _Tools: def __getattr__(self, name: str): if name.startswith("_"): raise AttributeError(name) return _Tool(name) - @staticmethod - def list(): + def list(self): return _RPC.call("tools", "list_tools", {}) - @staticmethod - def search(query: str, limit: int = 5): + def search(self, query: str, limit: int = 5): return _RPC.call("tools", "search_tools", {"query": query, "limit": limit}) +tools = _Tools() + class workflows: @staticmethod def list(): diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index d1e9ea3..d495b76 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -119,3 +119,214 @@ async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path async with Session(storage=storage, executor=executor) as session: r = await session.run("deps.add('packaging')\nimport packaging\npackaging.__version__") assert r.error is not None + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run( + "artifacts.save('obj', {'a': 1, 'b': [2, 3]}, description='t')\n" + "artifacts.load('obj')['b'][1]" + ) + assert r.error is None + assert r.value == 3 + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + source = "async def run(name: str) -> str:\n return 'hi ' + name\n" + + async with Session(storage=storage, executor=executor) as session: + r = await session.run( + "workflows.create('hello', " + f"{source!r}, " + "'test wf')\n" + "workflows.get('hello')['source']" + ) + assert r.error is None + assert r.value == source + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + (tools_dir / "echo.yaml").write_text( + "\n".join( + [ + "name: echo", + "description: Echo text", + "command: /bin/echo", + "timeout: 5", + "schema:", + " positional:", + " - name: message", + " type: string", + " required: true", + " description: message", + "recipes:", + " say:", + " description: Echo message", + " params:", + " message: {}", + "", + ] + ), + encoding="utf-8", + ) + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + tools_path=tools_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r_list = await session.run("sorted([t['name'] for t in tools.list()])") + assert r_list.error is None + assert "echo" in r_list.value + + r = await session.run("tools.echo.say(message='hi').strip()") + assert r.error is None + assert r.value == "hi" + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=15.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run( + "\n".join( + [ + "artifacts.save('x', {'n': 1}, description='')", + "s = 0", + "for _ in range(50):", + " if artifacts.exists('x'):", + " s += artifacts.load('x')['n']", + "s", + ] + ) + ) + assert r.error is None + assert r.value == 50 + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r1 = await session.run("x = 1\nx") + assert r1.error is None + assert r1.value == 1 + + await session.reset() + + r2 = await session.run("x") + assert r2.error is not None + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=300.0, + deps_timeout=300.0, + ipc_timeout=120.0, + network_profile="deps-only", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + out = await session.add_dep("packaging") + assert out["failed"] == [] + + r = await session.run("import packaging\npackaging.__version__") + assert r.error is None + assert isinstance(r.value, str) From 3dd8722267d6811e0e5d38a30300e5ec8854ee9f Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 02:27:28 -0800 Subject: [PATCH 05/27] Add deeper Deno/Pyodide executor validation tests --- tests/test_deno_pyodide_executor.py | 246 ++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index d495b76..365072a 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -1,4 +1,7 @@ +import hashlib +import math import os +from dataclasses import dataclass from pathlib import Path import pytest @@ -9,6 +12,72 @@ ) +class _StableEmbedder: + """Deterministic lightweight embedder for integration tests. + + Avoids pulling large sentence-transformers models while still exercising + semantic-search codepaths. + """ + + def __init__(self, dimension: int = 64) -> None: + self._dimension = dimension + + @property + def dimension(self) -> int: + return self._dimension + + def _embed_one(self, text: str) -> list[float]: + vec = [0.0] * self._dimension + for token in "".join(c.lower() if c.isalnum() else " " for c in text).split(): + h = hashlib.sha256(token.encode("utf-8")).digest() + idx = int.from_bytes(h[:4], "big") % self._dimension + vec[idx] += 1.0 + norm = math.sqrt(sum(v * v for v in vec)) or 1.0 + return [v / norm for v in vec] + + def embed(self, texts: list[str]) -> list[list[float]]: + return [self._embed_one(t) for t in texts] + + def embed_query(self, query: str) -> list[float]: + return self._embed_one(query) + + +@dataclass +class _TestStorage: + """Minimal StorageBackend for exercising workflows/artifacts via RPC.""" + + root: Path + + def __post_init__(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + + from py_code_mode.artifacts import FileArtifactStore + from py_code_mode.workflows import FileWorkflowStore, WorkflowLibrary + + self._artifact_store = FileArtifactStore(self.root / "artifacts") + self._workflow_store = FileWorkflowStore(self.root / "workflows") + self._workflow_library = WorkflowLibrary( + embedder=_StableEmbedder(), + store=self._workflow_store, + vector_store=None, + ) + + def get_serializable_access(self): + from py_code_mode.execution.protocol import FileStorageAccess + + return FileStorageAccess( + workflows_path=self.root / "workflows", + artifacts_path=self.root / "artifacts", + vectors_path=None, + ) + + def get_workflow_library(self): + return self._workflow_library + + def get_artifact_store(self): + return self._artifact_store + + @pytest.mark.asyncio async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor @@ -330,3 +399,180 @@ async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> r = await session.run("import packaging\npackaging.__version__") assert r.error is None assert isinstance(r.value, str) + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + + # Local stdio MCP server using fastmcp. + mcp_server = tmp_path / "mcp_server.py" + mcp_server.write_text( + "\n".join( + [ + "from fastmcp import FastMCP", + "", + "mcp = FastMCP('test')", + "", + "@mcp.tool", + "async def add(a: int, b: int) -> str:", + " return str(a + b)", + "", + "if __name__ == '__main__':", + " mcp.run()", + "", + ] + ), + encoding="utf-8", + ) + + (tools_dir / "math.yaml").write_text( + "\n".join( + [ + "name: math", + "type: mcp", + "transport: stdio", + "command: python", + f"args: [{mcp_server!s}]", + "description: Simple math MCP server", + "", + ] + ), + encoding="utf-8", + ) + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + tools_path=tools_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r_list = await session.run("sorted([t['name'] for t in tools.list()])") + assert r_list.error is None + assert "math" in r_list.value + + r = await session.run("tools.math.add(a=2, b=3).strip()") + assert r.error is None + assert r.value == "5" + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + + storage = _TestStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + src = "async def run() -> str:\n return 'hello world'\n" + + async with Session(storage=storage, executor=executor) as session: + r1 = await session.run("workflows.create('wf', " f"{src!r}, " "'greeting workflow')") + assert r1.error is None + + r2 = await session.run("workflows.search('greeting', limit=5)[0]['name']") + assert r2.error is None + assert r2.value == "wf" + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + store = storage.get_artifact_store() + store.save("small", "x" * (200 * 1024), description="") + store.save("big", "x" * (2 * 1024 * 1024), description="") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=180.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r_ok = await session.run( + "\n".join( + [ + "len(artifacts.load('small'))", + ] + ) + ) + assert r_ok.error is None + assert r_ok.value == 200 * 1024 + + r_big = await session.run( + "\n".join( + [ + "artifacts.load('big')", + ] + ) + ) + assert r_big.error is not None + assert "rpc payload too large" in r_big.error.lower() + + +@pytest.mark.asyncio +async def test_deno_pyodide_executor_soft_timeout_wedges_until_reset(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + default_timeout=0.05, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r1 = await session.run("import time\ntime.sleep(0.2)\n1") + assert r1.error is not None + assert "timeout" in r1.error.lower() + + r2 = await session.run("1 + 1") + assert r2.error is not None + assert "previous execution timed out" in r2.error.lower() + + await session.reset() + + r3 = await session.run("1 + 1") + assert r3.error is None + assert r3.value == 2 From dca25fa57f9fa69d519e4bea63ee03298269db66 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 03:40:36 -0800 Subject: [PATCH 06/27] DenoPyodide: async sandbox RPC + chunked responses --- .../execution/deno_pyodide/executor.py | 22 +- .../execution/deno_pyodide/runner/main.ts | 111 +++-- .../execution/deno_pyodide/runner/worker.ts | 454 ++++++++---------- tests/test_deno_pyodide_executor.py | 42 +- 4 files changed, 330 insertions(+), 299 deletions(-) diff --git a/src/py_code_mode/execution/deno_pyodide/executor.py b/src/py_code_mode/execution/deno_pyodide/executor.py index 37d64ad..52f4a77 100644 --- a/src/py_code_mode/execution/deno_pyodide/executor.py +++ b/src/py_code_mode/execution/deno_pyodide/executor.py @@ -305,7 +305,27 @@ async def _handle_rpc_request(self, msg: dict[str, Any]) -> None: req_id = msg.get("id") namespace = msg.get("namespace") op = msg.get("op") - args = msg.get("args") or {} + # Runner forwards rpc_request from the Pyodide worker. New protocol uses + # args_json (string) to avoid structured clone issues with Python proxy + # objects. Keep backwards-compat with args (dict) for older runners. + args: dict[str, Any] + if "args_json" in msg: + raw = msg.get("args_json") + if not isinstance(raw, str): + raw = "{}" + try: + parsed = json.loads(raw) + except Exception: + parsed = {} + if parsed is None: + args = {} + elif isinstance(parsed, dict): + args = parsed + else: + args = {} + else: + raw_args = msg.get("args") or {} + args = raw_args if isinstance(raw_args, dict) else {} if not isinstance(req_id, str) or not isinstance(namespace, str) or not isinstance(op, str): await self._send( diff --git a/src/py_code_mode/execution/deno_pyodide/runner/main.ts b/src/py_code_mode/execution/deno_pyodide/runner/main.ts index ddd6b7b..4e53bf4 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/main.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/main.ts @@ -10,13 +10,37 @@ type Req = | { id: string; type: "deps_install"; packages: string[] } | { id: string; type: "reset" } | { id: string; type: "close" } - | { id: string; type: "rpc_response"; ok: boolean; result?: unknown; error?: unknown }; + | { + id: string; + type: "rpc_response"; + ok: boolean; + result?: unknown; + error?: unknown; + }; type Resp = | { id: string; type: "ready" } - | { id: string; type: "exec_result"; stdout: string; value: unknown; error: string | null } - | { id: string; type: "deps_install_result"; installed: string[]; already_present: string[]; failed: string[] } - | { id: string; type: "rpc_request"; namespace: string; op: string; args: Record } + | { + id: string; + type: "exec_result"; + stdout: string; + value: unknown; + error: string | null; + } + | { + id: string; + type: "deps_install_result"; + installed: string[]; + already_present: string[]; + failed: string[]; + } + | { + id: string; + type: "rpc_request"; + namespace: string; + op: string; + args_json: string; + } | { id: string; type: "error"; message: string }; function writeMsg(msg: Resp) { @@ -39,30 +63,6 @@ async function* readNdjsonLines(): AsyncGenerator { if (buf) yield buf; } -// Shared buffers for synchronous RPC from the Pyodide worker. -// Single-flight: one outstanding RPC at a time. -const rpcState = new Int32Array(new SharedArrayBuffer(8)); // [status, length] -const rpcBuf = new Uint8Array(new SharedArrayBuffer(1024 * 1024)); // 1MB payload -// status: 0 = idle, 1 = response ready -let currentRpcId: string | null = null; - -function rpcWriteResponse(rpcId: string, payload: unknown) { - if (currentRpcId !== rpcId) { - writeMsg({ id: rpcId, type: "error", message: `rpc id mismatch: expected ${currentRpcId}, got ${rpcId}` }); - return; - } - const data = new TextEncoder().encode(JSON.stringify(payload ?? null)); - const out = data.length > rpcBuf.byteLength - ? new TextEncoder().encode(JSON.stringify({ ok: false, error: { type: "RPCTransportError", message: "rpc payload too large" } })) - : data; - - rpcBuf.set(out.subarray(0, rpcBuf.byteLength)); - rpcState[1] = out.length; - rpcState[0] = 1; - Atomics.notify(rpcState, 0, 1); - currentRpcId = null; -} - function newWorker(): Worker { const worker = new Worker(new URL("./worker.ts", import.meta.url), { type: "module", @@ -70,8 +70,6 @@ function newWorker(): Worker { }); worker.postMessage({ type: "boot", - rpcState, - rpcBuf, }); return worker; } @@ -83,7 +81,10 @@ let bootWait: Promise | null = null; let bootResolve: (() => void) | null = null; let bootReject: ((e: unknown) => void) | null = null; -const execPending = new Map void; reject: (e: any) => void }>(); +const execPending = new Map< + string, + { resolve: (v: any) => void; reject: (e: any) => void } +>(); function ensureWorker() { if (!worker) worker = newWorker(); @@ -165,10 +166,19 @@ function attachWorkerHandler() { } if (msg.type === "rpc_request") { - currentRpcId = msg.id; - writeMsg({ id: msg.id, type: "rpc_request", namespace: msg.namespace, op: msg.op, args: msg.args }); + // args_json is required to avoid DataCloneError (Pyodide dict proxies are not cloneable). + writeMsg({ + id: msg.id, + type: "rpc_request", + namespace: msg.namespace, + op: msg.op, + args_json: String(msg.args_json ?? "{}"), + }); return; } + + // Streamed RPC response from host -> runner -> worker. + // Note: worker only sends rpc_request messages. }; worker.onerror = (e) => { @@ -197,7 +207,28 @@ function enqueueRun(fn: () => Promise) { async function handleReq(req: Req) { try { if (req.type === "rpc_response") { - rpcWriteResponse(req.id, req.ok ? { ok: true, result: req.result } : { ok: false, error: req.error }); + // Forward as chunks to avoid fixed-size transport limits. + ensureWorker(); + const payload = JSON.stringify( + req.ok ? { ok: true, result: req.result } : { + ok: false, + error: req.error, + }, + ); + + const chunkSize = 64 * 1024; + let seq = 0; + for (let off = 0; off < payload.length; off += chunkSize) { + const chunk = payload.slice(off, off + chunkSize); + worker!.postMessage({ + type: "rpc_response_chunk", + id: req.id, + seq, + chunk, + }); + seq += 1; + } + worker!.postMessage({ type: "rpc_response_end", id: req.id, seq }); return; } @@ -247,7 +278,11 @@ async function handleReq(req: Req) { return; } } catch (e) { - writeMsg({ id: req.id, type: "error", message: String((e as any)?.stack ?? e) }); + writeMsg({ + id: req.id, + type: "error", + message: String((e as any)?.stack ?? e), + }); } } @@ -257,7 +292,11 @@ for await (const line of readNdjsonLines()) { try { req = JSON.parse(line); } catch (e) { - writeMsg({ id: "parse", type: "error", message: `invalid json: ${String(e)}` }); + writeMsg({ + id: "parse", + type: "error", + message: `invalid json: ${String(e)}`, + }); continue; } diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts index 9de6ea1..b57269e 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts @@ -1,7 +1,8 @@ -// Pyodide worker. Executes Python code and provides synchronous RPC calls +// Pyodide worker. Executes Python code and provides async RPC calls // back to the Deno main thread (which forwards to the Python host). // -// Single-flight RPC: one outstanding call at a time. +// RPC is promise-based and supports arbitrarily large payloads by streaming +// chunks over postMessage (runner -> worker). import { loadPyodide } from "npm:pyodide@0.29.3"; @@ -14,59 +15,56 @@ console.warn = _noop; console.debug = _noop; console.error = _noop; -type BootMsg = { - type: "boot"; - rpcState: Int32Array; - rpcBuf: Uint8Array; -}; - +type BootMsg = { type: "boot" }; type ExecMsg = { type: "exec"; id: string; code: string }; type DepsInstallMsg = { type: "deps_install"; id: string; packages: string[] }; -let rpcState: Int32Array | null = null; -let rpcBuf: Uint8Array | null = null; +type RpcResponseChunkMsg = { + type: "rpc_response_chunk"; + id: string; + seq: number; + chunk: string; +}; +type RpcResponseEndMsg = { type: "rpc_response_end"; id: string; seq: number }; + +type RpcReq = { + type: "rpc_request"; + id: string; + namespace: string; + op: string; + args_json: string; +}; -function rpcCallSync(namespace: string, op: string, args: Record): string { - if (!rpcState || !rpcBuf) throw new Error("rpc not initialized"); +type Msg = + | BootMsg + | ExecMsg + | DepsInstallMsg + | RpcResponseChunkMsg + | RpcResponseEndMsg; + +const rpcPending = new Map< + string, + { chunks: string[]; resolve: (v: string) => void; reject: (e: Error) => void } +>(); + +function rpcCallAsync( + namespace: string, + op: string, + argsJson: string, +): Promise { const id = crypto.randomUUID(); - (self as any).postMessage({ type: "rpc_request", id, namespace, op, args }); - - while (Atomics.load(rpcState, 0) === 0) { - Atomics.wait(rpcState, 0, 0, 60_000); - } - - const len = Atomics.load(rpcState, 1); - const bytes = rpcBuf.subarray(0, len); - const txt = new TextDecoder().decode(bytes); - Atomics.store(rpcState, 0, 0); - Atomics.store(rpcState, 1, 0); - return txt; -} - -function rpcCallSyncJsonArgs(namespace: string, op: string, argsJson: string): string { - let args: Record = {}; - try { - const parsed = JSON.parse(String(argsJson ?? "{}")); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - args = parsed as Record; - } - } catch { - // fall back to empty args - } - return rpcCallSync(namespace, op, args); -} - -function rpcCallSyncResult(namespace: string, op: string, args: Record): any { - const txt = rpcCallSync(namespace, op, args); - const obj = JSON.parse(txt); - if (!obj || typeof obj !== "object") { - throw new Error(`RPCTransportError: invalid rpc payload: ${txt.slice(0, 200)}`); - } - if (!obj.ok) { - const err = obj.error ?? {}; - throw new Error(`RPCError: ${String(err.message ?? err.type ?? "unknown")}`); - } - return obj.result; + const msg: RpcReq = { + type: "rpc_request", + id, + namespace, + op, + args_json: String(argsJson ?? "{}"), + }; + + return new Promise((resolve, reject) => { + rpcPending.set(id, { chunks: [], resolve, reject }); + (self as any).postMessage(msg); + }); } function pyodidePackageDir(): string { @@ -91,24 +89,24 @@ const attemptedInstalls = new Set(); async function ensureBooted() { if (booted) return; - // Use a filesystem path for indexURL to avoid "file:/..." pseudo-path issues. const indexURL = pyodidePackageDir(); - // Silence Pyodide's own package-loader chatter; stdout for user code is captured - // explicitly in runWithLastExpr() by redirecting sys.stdout. pyodide = await loadPyodide({ indexURL, stdout: _noop, stderr: _noop }); - // Expose a JSON-args RPC to Python; Pyodide dicts become non-cloneable proxies - // if we try to postMessage them directly. - (self as any).rpc_call_sync = rpcCallSyncJsonArgs; + // Expose async RPC to Python. + (self as any).rpc_call_async = rpcCallAsync; + + // Expose deps installers to Python (so deps.add/sync can just await these). + (self as any).pycm_install_package = installPackage; + (self as any).pycm_install_packages = installPackages; const bootstrap = ` -import ast, io, json, sys +import ast, json class _RPC: @staticmethod - def call(namespace: str, op: str, args: dict): + async def call(namespace: str, op: str, args: dict): import js - payload = js.rpc_call_sync(namespace, op, json.dumps(args)) + payload = await js.rpc_call_async(namespace, op, json.dumps(args)) obj = json.loads(str(payload)) if not obj.get("ok", False): err = obj.get("error") or {} @@ -119,82 +117,86 @@ class _ToolCallable: def __init__(self, name: str, recipe: str | None = None): self._name = name self._recipe = recipe - def __call__(self, **kwargs): + async def __call__(self, **kwargs): tool = self._name if self._recipe is None else f"{self._name}.{self._recipe}" - return _RPC.call("tools", "call_tool", {"name": tool, "args": kwargs}) + return await _RPC.call("tools", "call_tool", {"name": tool, "args": kwargs}) class _Tool: def __init__(self, name: str): self._name = name - def __call__(self, **kwargs): - return _RPC.call("tools", "call_tool", {"name": self._name, "args": kwargs}) + async def __call__(self, **kwargs): + return await _RPC.call("tools", "call_tool", {"name": self._name, "args": kwargs}) def __getattr__(self, recipe: str): if recipe.startswith("_"): raise AttributeError(recipe) return _ToolCallable(self._name, recipe) - def list(self): - return _RPC.call("tools", "list_tool_recipes", {"name": self._name}) + async def list(self): + return await _RPC.call("tools", "list_tool_recipes", {"name": self._name}) class _Tools: def __getattr__(self, name: str): if name.startswith("_"): raise AttributeError(name) return _Tool(name) - def list(self): - return _RPC.call("tools", "list_tools", {}) - def search(self, query: str, limit: int = 5): - return _RPC.call("tools", "search_tools", {"query": query, "limit": limit}) + async def list(self): + return await _RPC.call("tools", "list_tools", {}) + async def search(self, query: str, limit: int = 5): + return await _RPC.call("tools", "search_tools", {"query": query, "limit": limit}) tools = _Tools() class workflows: @staticmethod - def list(): - return _RPC.call("workflows", "list_workflows", {}) + async def list(): + return await _RPC.call("workflows", "list_workflows", {}) @staticmethod - def search(query: str, limit: int = 5): - return _RPC.call("workflows", "search_workflows", {"query": query, "limit": limit}) + async def search(query: str, limit: int = 5): + return await _RPC.call("workflows", "search_workflows", {"query": query, "limit": limit}) @staticmethod - def get(name: str): - return _RPC.call("workflows", "get_workflow", {"name": name}) + async def get(name: str): + return await _RPC.call("workflows", "get_workflow", {"name": name}) @staticmethod - def create(name: str, source: str, description: str): - return _RPC.call("workflows", "create_workflow", {"name": name, "source": source, "description": description}) + async def create(name: str, source: str, description: str): + return await _RPC.call("workflows", "create_workflow", {"name": name, "source": source, "description": description}) @staticmethod - def delete(name: str): - return _RPC.call("workflows", "delete_workflow", {"name": name}) + async def delete(name: str): + return await _RPC.call("workflows", "delete_workflow", {"name": name}) class artifacts: @staticmethod - def list(): - return _RPC.call("artifacts", "list_artifacts", {}) + async def list(): + return await _RPC.call("artifacts", "list_artifacts", {}) @staticmethod - def get(name: str): - return _RPC.call("artifacts", "get_artifact", {"name": name}) + async def get(name: str): + return await _RPC.call("artifacts", "get_artifact", {"name": name}) @staticmethod - def exists(name: str): - return _RPC.call("artifacts", "artifact_exists", {"name": name}) + async def exists(name: str): + return await _RPC.call("artifacts", "artifact_exists", {"name": name}) @staticmethod - def load(name: str): - return _RPC.call("artifacts", "load_artifact", {"name": name}) + async def load(name: str): + return await _RPC.call("artifacts", "load_artifact", {"name": name}) @staticmethod - def save(name: str, data, description: str = ""): - return _RPC.call("artifacts", "save_artifact", {"name": name, "data": data, "description": description}) + async def save(name: str, data, description: str = ""): + return await _RPC.call("artifacts", "save_artifact", {"name": name, "data": data, "description": description}) @staticmethod - def delete(name: str): - return _RPC.call("artifacts", "delete_artifact", {"name": name}) + async def delete(name: str): + return await _RPC.call("artifacts", "delete_artifact", {"name": name}) class deps: @staticmethod - def list(): - return _RPC.call("deps", "list_deps", {}) + async def list(): + return await _RPC.call("deps", "list_deps", {}) @staticmethod - def add(spec: str): - _RPC.call("deps", "persist_add", {"spec": spec}) - return {"installed": [spec], "already_present": [], "failed": []} + async def add(spec: str): + await _RPC.call("deps", "persist_add", {"spec": spec}) + import js + ok = await js.pycm_install_package(spec) + if ok: + return {"installed": [spec], "already_present": [], "failed": []} + return {"installed": [], "already_present": [], "failed": [spec]} @staticmethod - def remove(spec_or_name: str): - removed = _RPC.call("deps", "persist_remove", {"spec_or_name": spec_or_name}) + async def remove(spec_or_name: str): + removed = await _RPC.call("deps", "persist_remove", {"spec_or_name": spec_or_name}) return { "removed": [spec_or_name] if removed else [], "not_found": [] if removed else [spec_or_name], @@ -202,9 +204,11 @@ class deps: "removed_from_config": bool(removed), } @staticmethod - def sync(): - specs = _RPC.call("deps", "list_deps", {}) or [] - return {"installed": list(specs), "already_present": [], "failed": []} + async def sync(): + specs = await _RPC.call("deps", "list_deps", {}) or [] + import js + res = await js.pycm_install_packages(list(specs)) + return res `; pyodide.runPython(bootstrap); @@ -218,203 +222,162 @@ async function ensureMicropipLoaded(): Promise { micropipLoaded = true; } -async function micropipInstallOne(spec: string): Promise { - if (!pyodide) throw new Error("pyodide not initialized"); - pyodide.globals.set("_PYCM_DEP_SPEC", spec); - // runPythonAsync supports top-level await. - await pyodide.runPythonAsync(` +async function installPackage(spec: string): Promise { + const s = String(spec ?? "").trim(); + if (!s) return false; + if (attemptedInstalls.has(s)) return true; + + await ensureMicropipLoaded(); + try { + pyodide.globals.set("_PYCM_DEP_SPEC", s); + await pyodide.runPythonAsync(` import micropip await micropip.install(_PYCM_DEP_SPEC) `); + attemptedInstalls.add(s); + return true; + } catch { + return false; + } } -async function installPackages(packages: string[]): Promise<{ - installed: string[]; - already_present: string[]; - failed: string[]; -}> { +async function installPackages( + packages: string[], +): Promise< + { installed: string[]; already_present: string[]; failed: string[] } +> { const installed: string[] = []; const already_present: string[] = []; const failed: string[] = []; - const unique: string[] = []; - for (const p of packages) { + for (const p of packages ?? []) { const spec = String(p ?? "").trim(); if (!spec) continue; if (attemptedInstalls.has(spec)) { already_present.push(spec); continue; } - unique.push(spec); - } - - if (!unique.length) return { installed, already_present, failed }; - - await ensureMicropipLoaded(); - - for (const spec of unique) { - try { - await micropipInstallOne(spec); - attemptedInstalls.add(spec); - installed.push(spec); - } catch (e) { - failed.push(spec); - // Keep going; this is best-effort. - // We intentionally do not mark attemptedInstalls on failure so a later retry - // (e.g., after network policy changes) can re-attempt. - } + const ok = await installPackage(spec); + if (ok) installed.push(spec); + else failed.push(spec); } return { installed, already_present, failed }; } -function extractDepsRequests(code: string): { specs: string[]; wants_sync: boolean } { - if (!pyodide) throw new Error("pyodide not initialized"); - pyodide.globals.set("_PYCM_SCAN_CODE", code); - const txt = pyodide.runPython(` -import ast, json -try: - _tree = ast.parse(_PYCM_SCAN_CODE) -except Exception: - _out = json.dumps({"specs": [], "wants_sync": False}) -else: - class _V(ast.NodeVisitor): - def __init__(self): - self.specs = set() - self.wants_sync = False - # Skip descending into defs/classes entirely; those bodies aren't executed - # just by being present in a cell. - def visit_FunctionDef(self, node): return - def visit_AsyncFunctionDef(self, node): return - def visit_ClassDef(self, node): return - def visit_Lambda(self, node): return - - def visit_Call(self, node): - try: - f = node.func - if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name) and f.value.id == "deps": - if f.attr == "add" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): - self.specs.add(node.args[0].value) - elif f.attr == "sync" and not node.args and not node.keywords: - self.wants_sync = True - except Exception: - pass - self.generic_visit(node) - - _v = _V() - # Only scan top-level statements (and their non-def bodies). - for _stmt in getattr(_tree, "body", []): - _v.visit(_stmt) - _out = json.dumps({"specs": sorted(_v.specs), "wants_sync": bool(_v.wants_sync)}) - -_out -`); - const obj = JSON.parse(String(txt)); - return { specs: (obj.specs ?? []) as string[], wants_sync: Boolean(obj.wants_sync) }; -} - -async function maybeInstallDepsForCode(code: string): Promise { - const info = extractDepsRequests(code); - const specsToAdd = info.specs ?? []; - - for (const spec of specsToAdd) { - rpcCallSyncResult("deps", "persist_add", { spec }); - } - - let specsToInstall = specsToAdd; - if (info.wants_sync) { - const all = rpcCallSyncResult("deps", "list_deps", {}) ?? []; - specsToInstall = Array.isArray(all) ? all.map((s) => String(s)) : []; - } - - if (!specsToInstall.length) return; - - let res; - try { - res = await installPackages(specsToInstall); - } catch (e) { - throw new Error(`Failed to load micropip (network permission?): ${String((e as any)?.stack ?? e)}`); - } - if (res.failed.length) { - throw new Error(`Dependency install failed: ${res.failed.join(", ")}`); - } -} - -function runWithLastExpr(code: string): { stdout: string; value: any; error: string | null } { +async function runWithLastExprAsync( + code: string, +): Promise<{ stdout: string; value: any; error: string | null }> { const wrapper = ` -import ast, io, sys, traceback +import ast, asyncio, io, sys, traceback _stdout = io.StringIO() _value = None _error = None -try: - _tree = ast.parse(_CODE) - if _tree.body and isinstance(_tree.body[-1], ast.Expr): - _stmts = _tree.body[:-1] - _expr = _tree.body[-1] - if _stmts: - _m = ast.Module(body=_stmts, type_ignores=[]) - _c = compile(_m, "", "exec") + +async def _pycm_exec(_CODE: str): + global _value, _error + try: + _tree = ast.parse(_CODE) + if _tree.body and isinstance(_tree.body[-1], ast.Expr): + _stmts = _tree.body[:-1] + _expr = _tree.body[-1].value + if _stmts: + _m = ast.Module(body=_stmts, type_ignores=[]) + _c = compile(_m, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + _old = sys.stdout + sys.stdout = _stdout + try: + _r = eval(_c, globals(), globals()) + if asyncio.iscoroutine(_r): + await _r + finally: + sys.stdout = _old + _ec = compile(ast.Expression(body=_expr), "", "eval", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) _old = sys.stdout sys.stdout = _stdout try: - exec(_c, globals()) + _r2 = eval(_ec, globals(), globals()) + if asyncio.iscoroutine(_r2): + _r2 = await _r2 + _value = _r2 finally: sys.stdout = _old - _ec = compile(ast.Expression(body=_expr.value), "", "eval") - _old = sys.stdout - sys.stdout = _stdout - try: - _value = eval(_ec, globals()) - finally: - sys.stdout = _old - else: - _old = sys.stdout - sys.stdout = _stdout - try: - exec(_CODE, globals()) - finally: - sys.stdout = _old - _value = None -except Exception: - _error = traceback.format_exc() + else: + _c = compile(_tree, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + _old = sys.stdout + sys.stdout = _stdout + try: + _r = eval(_c, globals(), globals()) + if asyncio.iscoroutine(_r): + await _r + finally: + sys.stdout = _old + _value = None + except Exception: + _error = traceback.format_exc() + +await _pycm_exec(_CODE) `; if (!pyodide) throw new Error("pyodide not initialized"); pyodide.globals.set("_CODE", code); - pyodide.runPython(wrapper); + await pyodide.runPythonAsync(wrapper); + const stdout = pyodide.globals.get("_stdout").getvalue(); const error = pyodide.globals.get("_error"); let outValue: any; try { - const jsonTxt = pyodide.runPython("import json; json.dumps(_value)") as string; + const jsonTxt = pyodide.runPython( + "import json; json.dumps(_value)", + ) as string; outValue = JSON.parse(String(jsonTxt)); } catch { const repr = pyodide.runPython("repr(_value)") as string; outValue = { __py_repr__: String(repr) }; } - return { stdout: String(stdout ?? ""), value: outValue ?? null, error: error ? String(error) : null }; + return { + stdout: String(stdout ?? ""), + value: outValue ?? null, + error: error ? String(error) : null, + }; } -self.onmessage = async (ev: MessageEvent) => { - const msg = ev.data; +self.onmessage = async (ev: MessageEvent) => { + const msg = ev.data as any; + + if (msg.type === "rpc_response_chunk") { + const p = rpcPending.get(msg.id); + if (!p) return; + p.chunks.push(String(msg.chunk ?? "")); + return; + } + + if (msg.type === "rpc_response_end") { + const p = rpcPending.get(msg.id); + if (!p) return; + rpcPending.delete(msg.id); + p.resolve(p.chunks.join("")); + return; + } + if (msg.type === "boot") { - rpcState = msg.rpcState; - rpcBuf = msg.rpcBuf; try { await ensureBooted(); (self as any).postMessage({ type: "boot_ok" }); } catch (e) { - (self as any).postMessage({ type: "boot_error", error: String((e as any)?.stack ?? e) }); + (self as any).postMessage({ + type: "boot_error", + error: String((e as any)?.stack ?? e), + }); } return; } if (msg.type === "exec") { try { - await maybeInstallDepsForCode(msg.code); - const res = runWithLastExpr(msg.code); + const res = await runWithLastExprAsync(msg.code); (self as any).postMessage({ type: "exec_result", id: msg.id, ...res }); } catch (e) { (self as any).postMessage({ @@ -425,19 +388,24 @@ self.onmessage = async (ev: MessageEvent) => error: String((e as any)?.stack ?? e), }); } + return; } if (msg.type === "deps_install") { try { const res = await installPackages(msg.packages ?? []); - (self as any).postMessage({ type: "deps_install_result", id: msg.id, ...res }); + (self as any).postMessage({ + type: "deps_install_result", + id: msg.id, + ...res, + }); } catch (e) { (self as any).postMessage({ type: "deps_install_result", id: msg.id, installed: [], already_present: [], - failed: (msg.packages ?? []).map((p) => String(p)), + failed: (msg.packages ?? []).map((p: any) => String(p)), error: String((e as any)?.stack ?? e), }); } diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index 365072a..27a1adb 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -131,7 +131,9 @@ async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: ) async with Session(storage=storage, executor=executor) as session: - r = await session.run("deps.add('packaging')\nimport packaging\npackaging.__version__") + r = await session.run( + "await deps.add('packaging')\nimport packaging\npackaging.__version__" + ) assert r.error is None assert isinstance(r.value, str) assert r.value @@ -186,7 +188,9 @@ async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path ) async with Session(storage=storage, executor=executor) as session: - r = await session.run("deps.add('packaging')\nimport packaging\npackaging.__version__") + r = await session.run( + "await deps.add('packaging')\nimport packaging\npackaging.__version__" + ) assert r.error is not None @@ -211,8 +215,8 @@ async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None async with Session(storage=storage, executor=executor) as session: r = await session.run( - "artifacts.save('obj', {'a': 1, 'b': [2, 3]}, description='t')\n" - "artifacts.load('obj')['b'][1]" + "await artifacts.save('obj', {'a': 1, 'b': [2, 3]}, description='t')\n" + "(await artifacts.load('obj'))['b'][1]" ) assert r.error is None assert r.value == 3 @@ -241,10 +245,10 @@ async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None async with Session(storage=storage, executor=executor) as session: r = await session.run( - "workflows.create('hello', " + "await workflows.create('hello', " f"{source!r}, " "'test wf')\n" - "workflows.get('hello')['source']" + "(await workflows.get('hello'))['source']" ) assert r.error is None assert r.value == source @@ -297,11 +301,11 @@ async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: ) async with Session(storage=storage, executor=executor) as session: - r_list = await session.run("sorted([t['name'] for t in tools.list()])") + r_list = await session.run("_ts = await tools.list()\nsorted([t['name'] for t in _ts])") assert r_list.error is None assert "echo" in r_list.value - r = await session.run("tools.echo.say(message='hi').strip()") + r = await session.run("(await tools.echo.say(message='hi')).strip()") assert r.error is None assert r.value == "hi" @@ -329,11 +333,11 @@ async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> No r = await session.run( "\n".join( [ - "artifacts.save('x', {'n': 1}, description='')", + "await artifacts.save('x', {'n': 1}, description='')", "s = 0", "for _ in range(50):", - " if artifacts.exists('x'):", - " s += artifacts.load('x')['n']", + " if await artifacts.exists('x'):", + " s += (await artifacts.load('x'))['n']", "s", ] ) @@ -461,11 +465,11 @@ async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: ) async with Session(storage=storage, executor=executor) as session: - r_list = await session.run("sorted([t['name'] for t in tools.list()])") + r_list = await session.run("_ts = await tools.list()\nsorted([t['name'] for t in _ts])") assert r_list.error is None assert "math" in r_list.value - r = await session.run("tools.math.add(a=2, b=3).strip()") + r = await session.run("(await tools.math.add(a=2, b=3)).strip()") assert r.error is None assert r.value == "5" @@ -491,10 +495,10 @@ async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> src = "async def run() -> str:\n return 'hello world'\n" async with Session(storage=storage, executor=executor) as session: - r1 = await session.run("workflows.create('wf', " f"{src!r}, " "'greeting workflow')") + r1 = await session.run("await workflows.create('wf', " f"{src!r}, " "'greeting workflow')") assert r1.error is None - r2 = await session.run("workflows.search('greeting', limit=5)[0]['name']") + r2 = await session.run("(await workflows.search('greeting', limit=5))[0]['name']") assert r2.error is None assert r2.value == "wf" @@ -525,7 +529,7 @@ async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path r_ok = await session.run( "\n".join( [ - "len(artifacts.load('small'))", + "len(await artifacts.load('small'))", ] ) ) @@ -535,12 +539,12 @@ async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path r_big = await session.run( "\n".join( [ - "artifacts.load('big')", + "len(await artifacts.load('big'))", ] ) ) - assert r_big.error is not None - assert "rpc payload too large" in r_big.error.lower() + assert r_big.error is None + assert r_big.value == 2 * 1024 * 1024 @pytest.mark.asyncio From 8c5341936c4ed57e4ad182b59f624dc1ecc4d77a Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 03:43:17 -0800 Subject: [PATCH 07/27] DenoPyodide: test chunked RPC for large tool output --- tests/test_deno_pyodide_executor.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index 27a1adb..f0b8943 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -310,6 +310,65 @@ async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: assert r.value == "hi" +@pytest.mark.asyncio +async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path) -> None: + from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + (tools_dir / "pyrun.yaml").write_text( + "\n".join( + [ + "name: pyrun", + "description: Run Python snippet", + "command: python", + "timeout: 10", + "schema:", + " options:", + " command:", + " type: string", + " short: c", + " description: code snippet", + "recipes:", + " run:", + " description: Run snippet via -c", + " params:", + " command: {}", + "", + ] + ), + encoding="utf-8", + ) + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoPyodideExecutor( + DenoPyodideConfig( + deno_dir=deno_dir, + tools_path=tools_dir, + default_timeout=180.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run( + "\n".join( + [ + "code = \"import sys; sys.stdout.write('x' * (2 * 1024 * 1024))\"", + "len(await tools.pyrun.run(command=code))", + ] + ) + ) + assert r.error is None + assert r.value == 2 * 1024 * 1024 + + @pytest.mark.asyncio async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> None: from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor From e9b03380c02d6699762f63976a8d5f440609cf4f Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 04:28:21 -0800 Subject: [PATCH 08/27] Async sandbox: awaitable namespaces in in-process and subprocess --- src/py_code_mode/artifacts/__init__.py | 2 + src/py_code_mode/artifacts/namespace.py | 109 +++++++ src/py_code_mode/deps/async_namespace.py | 70 +++++ .../execution/in_process/executor.py | 102 +++++-- .../in_process/workflows_namespace.py | 93 +++++- .../execution/subprocess/kernel_init.py | 272 +++++++++++++----- src/py_code_mode/tools/namespace.py | 49 +++- tests/test_async_sandbox_interfaces.py | 139 +++++++++ 8 files changed, 728 insertions(+), 108 deletions(-) create mode 100644 src/py_code_mode/artifacts/namespace.py create mode 100644 src/py_code_mode/deps/async_namespace.py create mode 100644 tests/test_async_sandbox_interfaces.py diff --git a/src/py_code_mode/artifacts/__init__.py b/src/py_code_mode/artifacts/__init__.py index 1240bad..79e4454 100644 --- a/src/py_code_mode/artifacts/__init__.py +++ b/src/py_code_mode/artifacts/__init__.py @@ -5,11 +5,13 @@ ArtifactStoreProtocol, ) from py_code_mode.artifacts.file import FileArtifactStore +from py_code_mode.artifacts.namespace import ArtifactsNamespace from py_code_mode.artifacts.redis import RedisArtifactStore __all__ = [ "Artifact", "ArtifactStoreProtocol", "FileArtifactStore", + "ArtifactsNamespace", "RedisArtifactStore", ] diff --git a/src/py_code_mode/artifacts/namespace.py b/src/py_code_mode/artifacts/namespace.py new file mode 100644 index 0000000..f16e483 --- /dev/null +++ b/src/py_code_mode/artifacts/namespace.py @@ -0,0 +1,109 @@ +"""ArtifactsNamespace - agent-facing API for artifact storage. + +This mirrors the sandbox ergonomics used by the Deno/Pyodide executor: a small, +high-level API exposed as `artifacts.*` inside executed code. + +Design goal: allow both sync usage (`artifacts.load(...)`) and async usage +(`await artifacts.load(...)`) depending on execution context. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from py_code_mode.artifacts.base import ArtifactStoreProtocol + + +def _in_async_context() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +class ArtifactsNamespace: + """Agent-facing namespace for artifacts. + + Provides: + - artifacts.save(name, data, description="", metadata=None) + - artifacts.load(name) + - artifacts.list() + - artifacts.exists(name) + - artifacts.get(name) + - artifacts.delete(name) + + This wrapper intentionally exposes a subset of the underlying store API to + keep the sandbox surface stable. + """ + + def __init__(self, store: ArtifactStoreProtocol) -> None: + self._store = store + + @property + def path(self) -> Any: + # Keep compatibility with FileArtifactStore.path. + return getattr(self._store, "path", None) + + def save( + self, + name: str, + data: Any, + description: str = "", + metadata: dict[str, Any] | None = None, + ) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._store.save(name, data, description=description, metadata=metadata) + + return _coro() + return self._store.save(name, data, description=description, metadata=metadata) + + def load(self, name: str) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._store.load(name) + + return _coro() + return self._store.load(name) + + def list(self) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._store.list() + + return _coro() + return self._store.list() + + def exists(self, name: str) -> Any: + if _in_async_context(): + + async def _coro() -> bool: + return bool(self._store.exists(name)) + + return _coro() + return bool(self._store.exists(name)) + + def get(self, name: str) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._store.get(name) + + return _coro() + return self._store.get(name) + + def delete(self, name: str) -> Any: + if _in_async_context(): + + async def _coro() -> None: + self._store.delete(name) + + return _coro() + self._store.delete(name) + return None diff --git a/src/py_code_mode/deps/async_namespace.py b/src/py_code_mode/deps/async_namespace.py new file mode 100644 index 0000000..5ae3b62 --- /dev/null +++ b/src/py_code_mode/deps/async_namespace.py @@ -0,0 +1,70 @@ +"""Async-capable wrapper for deps namespace. + +The base DepsNamespace API is synchronous (it may run pip). For sandbox +consistency with async executors, this wrapper allows: + - deps.add(...) (sync) + - await deps.add(...) (async) + +The async variants simply run the synchronous operations in-process. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from py_code_mode.deps.namespace import DepsNamespace + + +def _in_async_context() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +class AsyncDepsNamespace: + """Wrapper around DepsNamespace providing optional await support.""" + + def __init__(self, wrapped: DepsNamespace) -> None: + self._wrapped = wrapped + + def add(self, package: str) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._wrapped.add(package) + + return _coro() + return self._wrapped.add(package) + + def list(self) -> Any: + if _in_async_context(): + + async def _coro() -> list[str]: + return self._wrapped.list() + + return _coro() + return self._wrapped.list() + + def remove(self, package: str) -> Any: + if _in_async_context(): + + async def _coro() -> bool: + return self._wrapped.remove(package) + + return _coro() + return self._wrapped.remove(package) + + def sync(self) -> Any: + if _in_async_context(): + + async def _coro() -> Any: + return self._wrapped.sync() + + return _coro() + return self._wrapped.sync() + + def __repr__(self) -> str: + return repr(self._wrapped) diff --git a/src/py_code_mode/execution/in_process/executor.py b/src/py_code_mode/execution/in_process/executor.py index ba43519..7ca3467 100644 --- a/src/py_code_mode/execution/in_process/executor.py +++ b/src/py_code_mode/execution/in_process/executor.py @@ -17,6 +17,7 @@ from contextlib import redirect_stdout from typing import TYPE_CHECKING, Any +from py_code_mode.artifacts import ArtifactsNamespace from py_code_mode.deps import ( ControlledDepsNamespace, DepsNamespace, @@ -24,6 +25,7 @@ PackageInstaller, collect_configured_deps, ) +from py_code_mode.deps.async_namespace import AsyncDepsNamespace from py_code_mode.deps.store import DepsStore, MemoryDepsStore from py_code_mode.execution.in_process.config import InProcessConfig from py_code_mode.execution.in_process.workflows_namespace import WorkflowsNamespace @@ -102,16 +104,15 @@ def __init__( # Inject artifacts namespace if artifact_store provided if artifact_store is not None: - self._namespace["artifacts"] = artifact_store + self._namespace["artifacts"] = ArtifactsNamespace(artifact_store) # Inject deps namespace if provided (wrap if runtime deps disabled) if deps_namespace is not None: + deps_obj: Any = AsyncDepsNamespace(deps_namespace) if not self._config.allow_runtime_deps: - self._namespace["deps"] = ControlledDepsNamespace( - deps_namespace, allow_runtime=False - ) + self._namespace["deps"] = ControlledDepsNamespace(deps_obj, allow_runtime=False) else: - self._namespace["deps"] = deps_namespace + self._namespace["deps"] = deps_obj def supports(self, capability: str) -> bool: """Check if this backend supports a capability.""" @@ -151,17 +152,24 @@ async def run(self, code: str, timeout: float | None = None) -> ExecutionResult: timeout = timeout if timeout is not None else self._default_timeout - # Store loop reference for tool/workflow calls from thread context - loop = asyncio.get_running_loop() - if "tools" in self._namespace: - self._namespace["tools"].set_loop(loop) - if "workflows" in self._namespace: - self._namespace["workflows"].set_loop(loop) + sync_mode = not self._requires_top_level_await(code) + + # Store loop reference for tool/workflow calls from thread context. + # Only used for the sync execution path; the async path executes in its + # own event loop in the worker thread. + if sync_mode: + loop = asyncio.get_running_loop() + if "tools" in self._namespace: + self._namespace["tools"].set_loop(loop) + if "workflows" in self._namespace: + self._namespace["workflows"].set_loop(loop) # Run in thread to allow timeout cancellation try: return await asyncio.wait_for( - asyncio.to_thread(self._run_sync, code), + asyncio.to_thread(self._run_sync, code) + if sync_mode + else asyncio.to_thread(self._run_async_in_thread, code), timeout=timeout, ) except TimeoutError: @@ -171,6 +179,19 @@ async def run(self, code: str, timeout: float | None = None) -> ExecutionResult: error=f"Execution timeout after {timeout} seconds", ) + @staticmethod + def _requires_top_level_await(code: str) -> bool: + """Return True if code needs PyCF_ALLOW_TOP_LEVEL_AWAIT to compile.""" + try: + compile(code, "", "exec") + return False + except SyntaxError: + try: + compile(code, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + return True + except SyntaxError: + return False + def _run_sync(self, code: str) -> ExecutionResult: """Run code synchronously, capturing output.""" stdout_capture = io.StringIO() @@ -218,6 +239,52 @@ def _run_sync(self, code: str) -> ExecutionResult: error=traceback.format_exc(), ) + def _run_async_in_thread(self, code: str) -> ExecutionResult: + """Execute code in a fresh event loop (supports top-level await).""" + try: + return asyncio.run(self._run_async(code)) + except Exception: + return ExecutionResult(value=None, stdout="", error=traceback.format_exc()) + + async def _run_async(self, code: str) -> ExecutionResult: + """Run code allowing top-level await, preserving last-expression semantics.""" + stdout_capture = io.StringIO() + try: + tree = ast.parse(code) + flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT + + with redirect_stdout(stdout_capture): + if tree.body and isinstance(tree.body[-1], ast.Expr): + stmts = tree.body[:-1] + expr = tree.body[-1] + + if stmts: + stmt_tree = ast.Module(body=stmts, type_ignores=[]) + stmt_code = compile(stmt_tree, "", "exec", flags=flags) + maybe = _eval_code(stmt_code, self._namespace, self._namespace) + if asyncio.iscoroutine(maybe): + await maybe + + expr_tree = ast.Expression(body=expr.value) + expr_code = compile(expr_tree, "", "eval", flags=flags) + value = _eval_code(expr_code, self._namespace, self._namespace) + if asyncio.iscoroutine(value): + value = await value + else: + stmt_code = compile(tree, "", "exec", flags=flags) + maybe = _eval_code(stmt_code, self._namespace, self._namespace) + if asyncio.iscoroutine(maybe): + await maybe + value = None + + return ExecutionResult(value=value, stdout=stdout_capture.getvalue(), error=None) + except Exception: + return ExecutionResult( + value=None, + stdout=stdout_capture.getvalue(), + error=traceback.format_exc(), + ) + async def close(self) -> None: """Release executor resources.""" self._closed = True @@ -300,12 +367,11 @@ async def start( self._deps_namespace = DepsNamespace(store=deps_store, installer=installer) # Wrap deps namespace if runtime deps disabled + deps_obj: Any = AsyncDepsNamespace(self._deps_namespace) if not self._config.allow_runtime_deps: - self._namespace["deps"] = ControlledDepsNamespace( - self._deps_namespace, allow_runtime=False - ) + self._namespace["deps"] = ControlledDepsNamespace(deps_obj, allow_runtime=False) else: - self._namespace["deps"] = self._deps_namespace + self._namespace["deps"] = deps_obj # Workflows and artifacts from storage (if provided) if storage is not None: @@ -315,7 +381,7 @@ async def start( ) self._artifact_store = storage.get_artifact_store() - self._namespace["artifacts"] = self._artifact_store + self._namespace["artifacts"] = ArtifactsNamespace(self._artifact_store) elif self._workflow_library is not None: # Use workflow_library from __init__ if provided self._namespace["workflows"] = WorkflowsNamespace( @@ -324,7 +390,7 @@ async def start( if storage is None and self._artifact_store is not None: # Use artifact_store from __init__ if provided - self._namespace["artifacts"] = self._artifact_store + self._namespace["artifacts"] = ArtifactsNamespace(self._artifact_store) async def install_deps(self, packages: list[str]) -> dict[str, Any]: """Install packages in the in-process environment. diff --git a/src/py_code_mode/execution/in_process/workflows_namespace.py b/src/py_code_mode/execution/in_process/workflows_namespace.py index 4bbf680..79e33b2 100644 --- a/src/py_code_mode/execution/in_process/workflows_namespace.py +++ b/src/py_code_mode/execution/in_process/workflows_namespace.py @@ -64,19 +64,60 @@ def library(self) -> WorkflowLibrary: """ return self._library - def search(self, query: str, limit: int = 10) -> builtins.list[dict[str, Any]]: - """Search for workflows matching query. Returns simplified workflow info.""" - workflows = self._library.search(query, limit) - return [self._simplify(w) for w in workflows] + def search(self, query: str, limit: int = 10) -> builtins.list[dict[str, Any]] | Any: + """Search for workflows matching query. + + In async context, returns an awaitable so code can use `await workflows.search(...)`. + """ + + def _run() -> builtins.list[dict[str, Any]]: + workflows = self._library.search(query, limit) + return [self._simplify(w) for w in workflows] + + try: + asyncio.get_running_loop() + except RuntimeError: + return _run() + + async def _coro() -> builtins.list[dict[str, Any]]: + return _run() + + return _coro() def get(self, name: str) -> Any: - """Get a workflow by name.""" - return self._library.get(name) + """Get a workflow by name. - def list(self) -> builtins.list[dict[str, Any]]: - """List all available workflows. Returns simplified workflow info.""" - workflows = self._library.list() - return [self._simplify(w) for w in workflows] + In async context, returns an awaitable so code can use `await workflows.get(...)`. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return self._library.get(name) + + async def _coro() -> Any: + return self._library.get(name) + + return _coro() + + def list(self) -> builtins.list[dict[str, Any]] | Any: + """List all available workflows. + + In async context, returns an awaitable so code can use `await workflows.list()`. + """ + + def _run() -> builtins.list[dict[str, Any]]: + workflows = self._library.list() + return [self._simplify(w) for w in workflows] + + try: + asyncio.get_running_loop() + except RuntimeError: + return _run() + + async def _coro() -> builtins.list[dict[str, Any]]: + return _run() + + return _coro() def _simplify(self, workflow: PythonWorkflow) -> dict[str, Any]: """Simplify workflow for agent readability.""" @@ -121,9 +162,20 @@ def create( # Add to library (persists to store if configured) self._library.add(workflow) - return self._simplify(workflow) + result = self._simplify(workflow) + + # Allow awaiting in async contexts for consistency. + try: + asyncio.get_running_loop() + except RuntimeError: + return result + + async def _coro() -> dict[str, Any]: + return result - def delete(self, name: str) -> bool: + return _coro() + + def delete(self, name: str) -> bool | Any: """Remove a workflow from the library. Args: @@ -132,7 +184,16 @@ def delete(self, name: str) -> bool: Returns: True if workflow was deleted, False if not found. """ - return self._library.remove(name) + removed = self._library.remove(name) + try: + asyncio.get_running_loop() + except RuntimeError: + return removed + + async def _coro() -> bool: + return removed + + return _coro() def __getattr__(self, name: str) -> Any: """Allow workflows.workflow_name(...) syntax.""" @@ -172,6 +233,10 @@ def invoke(self, workflow_name: str, **kwargs: Any) -> Any: asyncio.get_running_loop() except RuntimeError: return asyncio.run(result) - raise RuntimeError("Cannot invoke async workflows from a running event loop") + + async def _coro() -> Any: + return await result + + return _coro() return result diff --git a/src/py_code_mode/execution/subprocess/kernel_init.py b/src/py_code_mode/execution/subprocess/kernel_init.py index c74b14f..6946ea0 100644 --- a/src/py_code_mode/execution/subprocess/kernel_init.py +++ b/src/py_code_mode/execution/subprocess/kernel_init.py @@ -27,6 +27,7 @@ def get_kernel_init_code(ipc_timeout: float | None = None) -> str: from __future__ import annotations import json +import asyncio import threading import uuid from typing import Any, NamedTuple @@ -51,6 +52,22 @@ def _namespace_error_handler(self, etype, value, tb, tb_offset=None): # Threading lock to prevent concurrent RPC corruption _rpc_lock = threading.Lock() +# Async lock to prevent concurrent RPC corruption when used with await/to_thread +_rpc_async_lock = asyncio.Lock() + + +def _in_async_context() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +async def _rpc_call_async(method: str, **kwargs) -> Any: + async with _rpc_async_lock: + return await asyncio.to_thread(_rpc_call, method, **kwargs) + # Configurable timeout for RPC calls _RPC_TIMEOUT = {ipc_timeout} @@ -307,7 +324,12 @@ def __init__(self, tool_name: str, recipe_name: str): def __call__(self, **kwargs) -> Any: """Invoke the recipe with given arguments.""" - # Recipe invocation: name is "tool.recipe" + if _in_async_context(): + return _rpc_call_async( + "tools.call", + name=f"{{self._tool_name}}.{{self._recipe_name}}", + args=kwargs, + ) return _rpc_call( "tools.call", name=f"{{self._tool_name}}.{{self._recipe_name}}", @@ -324,6 +346,8 @@ def __init__(self, name: str, validated: bool = False): def __call__(self, **kwargs) -> Any: """Direct tool invocation (escape hatch).""" + if _in_async_context(): + return _rpc_call_async("tools.call", name=self._name, args=kwargs) return _rpc_call("tools.call", name=self._name, args=kwargs) def __getattr__(self, recipe_name: str) -> _ToolRecipeProxy: @@ -334,6 +358,8 @@ def __getattr__(self, recipe_name: str) -> _ToolRecipeProxy: def list(self) -> list[dict[str, Any]]: """List recipes for this tool.""" + if _in_async_context(): + return _rpc_call_async("tools.list_recipes", name=self._name) return _rpc_call("tools.list_recipes", name=self._name) @@ -388,7 +414,8 @@ def list(self) -> list[dict[str, Any]]: Returns: List of dicts with name, description, and tags keys. """ - # Return raw dicts (not NamedTuples) so they serialize cleanly through IPython + if _in_async_context(): + return _rpc_call_async("tools.list") return _rpc_call("tools.list") def search(self, query: str, limit: int = 10) -> list[dict[str, Any]]: @@ -397,7 +424,8 @@ def search(self, query: str, limit: int = 10) -> list[dict[str, Any]]: Returns: List of dicts matching the query. """ - # Return raw dicts (not NamedTuples) so they serialize cleanly through IPython + if _in_async_context(): + return _rpc_call_async("tools.search", query=query, limit=limit) return _rpc_call("tools.search", query=query, limit=limit) @@ -427,7 +455,35 @@ def invoke(self, workflow_name: str, **kwargs) -> Any: Note: Uses workflow_name (not name) to avoid collision with workflows that have a 'name' parameter. """ - import asyncio + if _in_async_context(): + async def _coro() -> Any: + workflow = await _rpc_call_async("workflows.get", name=workflow_name) + if workflow is None: + raise ValueError(f"Workflow not found: {{workflow_name}}") + + source = workflow.get("source") + if not source: + raise ValueError(f"Workflow has no source: {{workflow_name}}") + + workflow_namespace = {{ + "tools": tools, + "workflows": workflows, + "artifacts": artifacts, + "deps": deps, + }} + code = compile(source, f"", "exec") + exec(code, workflow_namespace) + + run_func = workflow_namespace.get("run") + if not callable(run_func): + raise ValueError(f"Workflow {{workflow_name}} has no run() function") + + result = run_func(**kwargs) + if asyncio.iscoroutine(result): + return await result + return result + + return _coro() workflow = _rpc_call("workflows.get", name=workflow_name) if workflow is None: @@ -452,17 +508,6 @@ def invoke(self, workflow_name: str, **kwargs) -> Any: result = run_func(**kwargs) if asyncio.iscoroutine(result): - try: - asyncio.get_running_loop() - has_loop = True - except RuntimeError: - has_loop = False - - if has_loop: - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor() as pool: - future = pool.submit(asyncio.run, result) - return future.result() return asyncio.run(result) return result @@ -472,15 +517,25 @@ def search(self, query: str, limit: int = 5) -> list[Workflow]: Returns: List of Workflow objects matching the query. """ + def _convert(result: list[dict[str, Any]]) -> list[Workflow]: + return [ + Workflow( + name=s["name"], + description=s.get("description", ""), + params=s.get("params", {{}}), + ) + for s in result + ] + + if _in_async_context(): + async def _coro() -> list[Workflow]: + result = await _rpc_call_async("workflows.search", query=query, limit=limit) + return _convert(result) + + return _coro() + result = _rpc_call("workflows.search", query=query, limit=limit) - return [ - Workflow( - name=s["name"], - description=s.get("description", ""), - params=s.get("params", {{}}), - ) - for s in result - ] + return _convert(result) def list(self) -> list[Workflow]: """List all available workflows. @@ -488,21 +543,33 @@ def list(self) -> list[Workflow]: Returns: List of Workflow objects. """ + def _convert(result: list[dict[str, Any]]) -> list[Workflow]: + return [ + Workflow( + name=s["name"], + description=s.get("description", ""), + params=s.get("params", {{}}), + ) + for s in result + ] + + if _in_async_context(): + async def _coro() -> list[Workflow]: + result = await _rpc_call_async("workflows.list") + return _convert(result) + + return _coro() + result = _rpc_call("workflows.list") - return [ - Workflow( - name=s["name"], - description=s.get("description", ""), - params=s.get("params", {{}}), - ) - for s in result - ] + return _convert(result) def get(self, name: str) -> dict[str, Any] | None: """Get a workflow by name. Returns full workflow details including source. """ + if _in_async_context(): + return _rpc_call_async("workflows.get", name=name) return _rpc_call("workflows.get", name=name) def create(self, name: str, source: str, description: str = "") -> Workflow: @@ -511,6 +578,19 @@ def create(self, name: str, source: str, description: str = "") -> Workflow: Returns: Workflow object for the created workflow. """ + if _in_async_context(): + async def _coro() -> Workflow: + result = await _rpc_call_async( + "workflows.create", name=name, source=source, description=description + ) + return Workflow( + name=result["name"], + description=result.get("description", ""), + params=result.get("params", {{}}), + ) + + return _coro() + result = _rpc_call("workflows.create", name=name, source=source, description=description) return Workflow( name=result["name"], @@ -520,6 +600,8 @@ def create(self, name: str, source: str, description: str = "") -> Workflow: def delete(self, name: str) -> bool: """Delete a workflow.""" + if _in_async_context(): + return _rpc_call_async("workflows.delete", name=name) return _rpc_call("workflows.delete", name=name) def __getattr__(self, name: str) -> Any: @@ -543,6 +625,8 @@ class ArtifactsProxy: def load(self, name: str) -> Any: """Load an artifact by name.""" + if _in_async_context(): + return _rpc_call_async("artifacts.load", name=name) return _rpc_call("artifacts.load", name=name) def save(self, name: str, data: Any, description: str = "") -> ArtifactMeta: @@ -551,13 +635,25 @@ def save(self, name: str, data: Any, description: str = "") -> ArtifactMeta: Returns: ArtifactMeta with name, path, description, created_at. """ + def _convert(result: dict[str, Any]) -> ArtifactMeta: + return ArtifactMeta( + name=result["name"], + path=result.get("path", ""), + description=result.get("description", ""), + created_at=result.get("created_at", ""), + ) + + if _in_async_context(): + async def _coro() -> ArtifactMeta: + result = await _rpc_call_async( + "artifacts.save", name=name, data=data, description=description + ) + return _convert(result) + + return _coro() + result = _rpc_call("artifacts.save", name=name, data=data, description=description) - return ArtifactMeta( - name=result["name"], - path=result.get("path", ""), - description=result.get("description", ""), - created_at=result.get("created_at", ""), - ) + return _convert(result) def list(self) -> list[ArtifactMeta]: """List all artifacts. @@ -565,36 +661,60 @@ def list(self) -> list[ArtifactMeta]: Returns: List of ArtifactMeta objects. """ + def _convert(result: list[dict[str, Any]]) -> list[ArtifactMeta]: + return [ + ArtifactMeta( + name=a["name"], + path=a.get("path", ""), + description=a.get("description", ""), + created_at=a.get("created_at", ""), + ) + for a in result + ] + + if _in_async_context(): + async def _coro() -> list[ArtifactMeta]: + result = await _rpc_call_async("artifacts.list") + return _convert(result) + + return _coro() + result = _rpc_call("artifacts.list") - return [ - ArtifactMeta( - name=a["name"], - path=a.get("path", ""), - description=a.get("description", ""), - created_at=a.get("created_at", ""), - ) - for a in result - ] + return _convert(result) def delete(self, name: str) -> None: """Delete an artifact.""" + if _in_async_context(): + return _rpc_call_async("artifacts.delete", name=name) return _rpc_call("artifacts.delete", name=name) def exists(self, name: str) -> bool: """Check if an artifact exists.""" + if _in_async_context(): + return _rpc_call_async("artifacts.exists", name=name) return _rpc_call("artifacts.exists", name=name) def get(self, name: str) -> ArtifactMeta | None: """Get artifact metadata.""" + def _convert(result: dict[str, Any] | None) -> ArtifactMeta | None: + if result is None: + return None + return ArtifactMeta( + name=result["name"], + path=result.get("path", ""), + description=result.get("description", ""), + created_at=result.get("created_at", ""), + ) + + if _in_async_context(): + async def _coro() -> ArtifactMeta | None: + result = await _rpc_call_async("artifacts.get", name=name) + return _convert(result) + + return _coro() + result = _rpc_call("artifacts.get", name=name) - if result is None: - return None - return ArtifactMeta( - name=result["name"], - path=result.get("path", ""), - description=result.get("description", ""), - created_at=result.get("created_at", ""), - ) + return _convert(result) class DepsProxy: @@ -616,19 +736,33 @@ def add(self, package: str) -> SyncResult: Returns: SyncResult with installed, already_present, and failed tuples. """ + def _convert(result: dict[str, Any]) -> SyncResult: + return SyncResult( + installed=tuple(result.get("installed", [])), + already_present=tuple(result.get("already_present", [])), + failed=tuple(result.get("failed", [])), + ) + + if _in_async_context(): + async def _coro() -> SyncResult: + result = await _rpc_call_async("deps.add", package=package) + return _convert(result) + + return _coro() + result = _rpc_call("deps.add", package=package) - return SyncResult( - installed=tuple(result.get("installed", [])), - already_present=tuple(result.get("already_present", [])), - failed=tuple(result.get("failed", [])), - ) + return _convert(result) def remove(self, package: str) -> bool: """Remove a package from configuration.""" + if _in_async_context(): + return _rpc_call_async("deps.remove", package=package) return _rpc_call("deps.remove", package=package) def list(self) -> list[str]: """List configured packages.""" + if _in_async_context(): + return _rpc_call_async("deps.list") return _rpc_call("deps.list") def sync(self) -> SyncResult: @@ -637,12 +771,22 @@ def sync(self) -> SyncResult: Returns: SyncResult with installed, already_present, and failed tuples. """ + def _convert(result: dict[str, Any]) -> SyncResult: + return SyncResult( + installed=tuple(result.get("installed", [])), + already_present=tuple(result.get("already_present", [])), + failed=tuple(result.get("failed", [])), + ) + + if _in_async_context(): + async def _coro() -> SyncResult: + result = await _rpc_call_async("deps.sync") + return _convert(result) + + return _coro() + result = _rpc_call("deps.sync") - return SyncResult( - installed=tuple(result.get("installed", [])), - already_present=tuple(result.get("already_present", [])), - failed=tuple(result.get("failed", [])), - ) + return _convert(result) def __repr__(self) -> str: """String representation showing it's a DepsNamespace.""" diff --git a/src/py_code_mode/tools/namespace.py b/src/py_code_mode/tools/namespace.py index a3f8b6f..1c3d5dd 100644 --- a/src/py_code_mode/tools/namespace.py +++ b/src/py_code_mode/tools/namespace.py @@ -57,21 +57,46 @@ def __getattr__(self, tool_name: str) -> ToolProxy: return ToolProxy(self._registry, tool, self._loop) - def list(self) -> builtins.list[Tool]: - """List all available tools.""" - return self._registry.get_all_tools() + def list(self) -> builtins.list[Tool] | Any: + """List all available tools. - def search(self, query: str, limit: int = 5) -> builtins.list[Tool]: - """Search tools by query string.""" + In async context, returns an awaitable so code can use `await tools.list()`. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return self._registry.get_all_tools() + + async def _coro() -> builtins.list[Tool]: + return self._registry.get_all_tools() + + return _coro() + + def search(self, query: str, limit: int = 5) -> builtins.list[Tool] | Any: + """Search tools by query string. + + In async context, returns an awaitable so code can use `await tools.search(...)`. + """ from py_code_mode.tools.registry import substring_search - return substring_search( - query=query, - items=self._registry.get_all_tools(), - get_name=lambda t: t.name, - get_description=lambda t: t.description, - limit=limit, - ) + def _run() -> builtins.list[Tool]: + return substring_search( + query=query, + items=self._registry.get_all_tools(), + get_name=lambda t: t.name, + get_description=lambda t: t.description, + limit=limit, + ) + + try: + asyncio.get_running_loop() + except RuntimeError: + return _run() + + async def _coro() -> builtins.list[Tool]: + return _run() + + return _coro() class ToolProxy: diff --git a/tests/test_async_sandbox_interfaces.py b/tests/test_async_sandbox_interfaces.py new file mode 100644 index 0000000..d20d799 --- /dev/null +++ b/tests/test_async_sandbox_interfaces.py @@ -0,0 +1,139 @@ +import os +from pathlib import Path + +import pytest + + +@pytest.mark.asyncio +async def test_inprocess_executor_supports_await_tools_and_artifacts(tmp_path: Path) -> None: + from py_code_mode.execution import InProcessConfig, InProcessExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + (tools_dir / "echo.yaml").write_text( + "\n".join( + [ + "name: echo", + "description: Echo text", + "command: /bin/echo", + "timeout: 5", + "schema:", + " positional:", + " - name: message", + " type: string", + " required: true", + " description: message", + "recipes:", + " say:", + " description: Echo message", + " params:", + " message: {}", + "", + ] + ), + encoding="utf-8", + ) + + storage = FileStorage(tmp_path / "storage") + executor = InProcessExecutor(config=InProcessConfig(tools_path=tools_dir)) + + async with Session(storage=storage, executor=executor) as session: + r_list = await session.run("_ts = await tools.list()\nsorted([t.name for t in _ts])") + assert r_list.error is None + assert "echo" in r_list.value + + r_call = await session.run("(await tools.echo.say(message='hi')).strip()") + assert r_call.error is None + assert r_call.value == "hi" + + r_art = await session.run( + "\n".join( + [ + "await artifacts.save('obj', {'a': 1}, description='')", + "(await artifacts.load('obj'))['a']", + ] + ) + ) + assert r_art.error is None + assert r_art.value == 1 + + +pytestmark_subprocess = pytest.mark.skipif( + os.environ.get("PY_CODE_MODE_TEST_SUBPROCESS") == "0", + reason="Subprocess async-sandbox smoke can be disabled with PY_CODE_MODE_TEST_SUBPROCESS=0", +) + + +@pytestmark_subprocess +@pytest.mark.asyncio +async def test_subprocess_executor_supports_await_tools_workflows_artifacts(tmp_path: Path) -> None: + from py_code_mode.execution import SubprocessConfig, SubprocessExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + (tools_dir / "echo.yaml").write_text( + "\n".join( + [ + "name: echo", + "description: Echo text", + "command: /bin/echo", + "timeout: 5", + "schema:", + " positional:", + " - name: message", + " type: string", + " required: true", + " description: message", + "recipes:", + " say:", + " description: Echo message", + " params:", + " message: {}", + "", + ] + ), + encoding="utf-8", + ) + + storage = FileStorage(tmp_path / "storage") + executor = SubprocessExecutor( + config=SubprocessConfig(tools_path=tools_dir, default_timeout=60.0) + ) + + async with Session(storage=storage, executor=executor) as session: + r_list = await session.run( + "_ts = await tools.list()\n" "sorted([t['name'] for t in _ts])", + ) + assert r_list.error is None + assert "echo" in r_list.value + + r_call = await session.run("(await tools.echo.say(message='hi')).strip()") + assert r_call.error is None + assert r_call.value == "hi" + + source = "async def run(name: str) -> str:\n return 'hi ' + name\n" + r_wf = await session.run( + "\n".join( + [ + f"await workflows.create('hello', {source!r}, 'desc')", + "(await workflows.get('hello'))['source']", + ] + ) + ) + assert r_wf.error is None + assert r_wf.value == source + + r_art = await session.run( + "\n".join( + [ + "await artifacts.save('obj', {'a': 1}, description='')", + "(await artifacts.load('obj'))['a']", + ] + ) + ) + assert r_art.error is None + assert r_art.value == 1 From 448fd9c024e98c94cb62568738bacad635f706aa Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 04:29:51 -0800 Subject: [PATCH 09/27] ToolsNamespace: allow awaitable calls even when loop is set --- src/py_code_mode/tools/namespace.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/py_code_mode/tools/namespace.py b/src/py_code_mode/tools/namespace.py index 1c3d5dd..696be4c 100644 --- a/src/py_code_mode/tools/namespace.py +++ b/src/py_code_mode/tools/namespace.py @@ -152,15 +152,9 @@ def __call__(self, **kwargs: Any) -> Any: This delegates to the adapter's call_tool method, bypassing recipes. Returns coroutine in async context, executes sync otherwise. - When set_loop() has been called, always uses sync execution to support - calling tools from within synchronously-executed workflows. + If set_loop() has been called, sync execution in a worker thread uses + run_coroutine_threadsafe to schedule work on the main loop. """ - # If we have an explicit loop reference, always use sync path - # This supports calling tools from sync workflow code within async context - if self._loop is not None: - return self.call_sync(**kwargs) - - # Check if we're in async context (has running loop) try: asyncio.get_running_loop() except RuntimeError: @@ -236,15 +230,9 @@ def __call__(self, **kwargs: Any) -> Any: Returns a coroutine if called from async context, executes sync otherwise. This allows both `await tools.x.y()` and `tools.x.y()` to work. - When set_loop() has been called on the parent namespace, always uses - sync execution to support calling tools from within synchronously-executed workflows. + If set_loop() has been called, sync execution in a worker thread uses + run_coroutine_threadsafe to schedule work on the main loop. """ - # If we have an explicit loop reference, always use sync path - # This supports calling tools from sync workflow code within async context - if self._loop is not None: - return self.call_sync(**kwargs) - - # Check if we're in async context (has running loop) try: asyncio.get_running_loop() except RuntimeError: From 67f8e2e0d89ca739a411c24fc337d366eda93213 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:03:43 -0800 Subject: [PATCH 10/27] DenoPyodide: add workflows.invoke --- .../execution/deno_pyodide/runner/worker.ts | 20 ++++++++++++++++++- tests/test_deno_pyodide_executor.py | 6 +++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts index b57269e..d0257f0 100644 --- a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts +++ b/src/py_code_mode/execution/deno_pyodide/runner/worker.ts @@ -156,11 +156,29 @@ class workflows: async def get(name: str): return await _RPC.call("workflows", "get_workflow", {"name": name}) @staticmethod - async def create(name: str, source: str, description: str): + async def create(name: str, source: str, description: str = ""): return await _RPC.call("workflows", "create_workflow", {"name": name, "source": source, "description": description}) @staticmethod async def delete(name: str): return await _RPC.call("workflows", "delete_workflow", {"name": name}) + @staticmethod + async def invoke(workflow_name: str, **kwargs): + import asyncio + wf = await workflows.get(workflow_name) + if wf is None or not isinstance(wf, dict): + raise ValueError(f"Workflow not found: {workflow_name}") + source = wf.get("source") + if not source: + raise ValueError(f"Workflow has no source: {workflow_name}") + ns = {"tools": tools, "workflows": workflows, "artifacts": artifacts, "deps": deps} + exec(source, ns, ns) + run_func = ns.get("run") + if not callable(run_func): + raise ValueError(f"Workflow {workflow_name} has no run() function") + result = run_func(**kwargs) + if asyncio.iscoroutine(result): + result = await result + return result class artifacts: @staticmethod diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index f0b8943..2ddd43d 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -253,6 +253,10 @@ async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None assert r.error is None assert r.value == source + r2 = await session.run("await workflows.invoke('hello', name='py-code-mode')") + assert r2.error is None + assert r2.value == "hi py-code-mode" + @pytest.mark.asyncio async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: @@ -554,7 +558,7 @@ async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> src = "async def run() -> str:\n return 'hello world'\n" async with Session(storage=storage, executor=executor) as session: - r1 = await session.run("await workflows.create('wf', " f"{src!r}, " "'greeting workflow')") + r1 = await session.run(f"await workflows.create('wf', {src!r}, 'greeting workflow')") assert r1.error is None r2 = await session.run("(await workflows.search('greeting', limit=5))[0]['name']") From 985c83b5be02a57164ec84af782a8f7c1a4518d1 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:03:51 -0800 Subject: [PATCH 11/27] docs: document DenoPyodide executor and async-first sandbox API --- docs/ARCHITECTURE.md | 10 +++---- docs/artifacts.md | 40 +++++++++++++------------- docs/executors.md | 63 ++++++++++++++++++++++++++++++++++------- docs/getting-started.md | 19 +++++++------ docs/integrations.md | 12 ++++---- docs/session-api.md | 2 +- docs/storage.md | 4 +-- docs/tools.md | 20 +++++++------ docs/workflows.md | 34 +++++++++++----------- 9 files changed, 126 insertions(+), 78 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2a5ecaf..1d6758b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -791,14 +791,14 @@ async with Session(storage=storage, executor=executor) as session: ### Tool Execution ``` -Agent writes: "tools.curl.get(url='...')" +Agent writes: "await tools.curl.get(url='...')" | v +------------------------+ | ToolsNamespace | | | | tools.curl(url=...) |--> Escape hatch (direct invocation) -| tools.curl.get(...) |--> Recipe invocation +| tools.curl.get(...) |--> Recipe invocation (awaitable in async contexts) | tools.search(...) | | | tools.list() | v +------------------------+ +--------------+ @@ -811,7 +811,7 @@ Agent writes: "tools.curl.get(url='...')" ### ToolProxy Methods ``` -Agent writes: "tools.curl.get(url='...')" +Agent writes: "await tools.curl.get(url='...')" | v +------------------------+ @@ -835,7 +835,7 @@ Agent writes: "tools.curl.get(url='...')" ### Skill Execution ``` -Agent writes: "workflows.analyze_repo(repo='...')" +Agent writes: "await workflows.analyze_repo(repo='...')" | v +------------------------+ @@ -880,7 +880,7 @@ Skill has access to: ### Artifact Storage ``` -Agent writes: "artifacts.save('data.json', b'...', 'description')" +Agent writes: "await artifacts.save('data.json', b'...', 'description')" | v +------------------------+ diff --git a/docs/artifacts.md b/docs/artifacts.md index 38e2160..23e3273 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -6,19 +6,19 @@ Artifacts provide persistent data storage across sessions. Use them to cache res ```python # Save data -artifacts.save("analysis_results", { +await artifacts.save("analysis_results", { "repos_analyzed": 42, "findings": [...] }) # Load data -data = artifacts.load("analysis_results") +data = await artifacts.load("analysis_results") # List all artifacts -all_artifacts = artifacts.list() +all_artifacts = await artifacts.list() # Delete an artifact -artifacts.delete("old_data") +await artifacts.delete("old_data") ``` ## Storage Formats @@ -27,13 +27,13 @@ Artifacts automatically handle serialization based on data type: ```python # JSON-serializable data (dicts, lists, primitives) -artifacts.save("config", {"api_key": "...", "timeout": 30}) +await artifacts.save("config", {"api_key": "...", "timeout": 30}) # Binary data -artifacts.save("image", image_bytes) +await artifacts.save("image", image_bytes) # Text data -artifacts.save("report", "Analysis results: ...") +await artifacts.save("report", "Analysis results: ...") ``` ## Use Cases @@ -45,15 +45,15 @@ async def run(owner: str, repo: str) -> dict: cache_key = f"repo_{owner}_{repo}" # Check cache first - cached = artifacts.load(cache_key) + cached = await artifacts.load(cache_key) if cached: return cached # Fetch fresh data - data = tools.curl.get(url=f"https://api.github.com/repos/{owner}/{repo}") + data = await tools.curl.get(url=f"https://api.github.com/repos/{owner}/{repo}") # Cache for next time - artifacts.save(cache_key, data) + await artifacts.save(cache_key, data) return data ``` @@ -62,17 +62,17 @@ async def run(owner: str, repo: str) -> dict: ```python async def run(url: str) -> dict: # Load previous crawl state - state = artifacts.load("crawl_state") or {"visited": [], "queue": []} + state = await artifacts.load("crawl_state") or {"visited": [], "queue": []} if url in state["visited"]: return {"status": "already_crawled"} # Process URL - content = tools.fetch(url=url) + content = await tools.fetch(url=url) state["visited"].append(url) # Save updated state - artifacts.save("crawl_state", state) + await artifacts.save("crawl_state", state) return {"status": "success", "content": content} ``` @@ -82,14 +82,14 @@ async def run(url: str) -> dict: # Skill 1: Collect data async def run(sources: list) -> dict: results = [fetch_source(s) for s in sources] - artifacts.save("collected_data", results) + await artifacts.save("collected_data", results) return {"count": len(results)} # Skill 2: Analyze data async def run() -> dict: - data = artifacts.load("collected_data") + data = await artifacts.load("collected_data") analysis = analyze(data) - artifacts.save("analysis_report", analysis) + await artifacts.save("analysis_report", analysis) return analysis ``` @@ -112,16 +112,16 @@ Artifacts are stored according to your storage backend: **Use descriptive names:** ```python # Good -artifacts.save("github_repos_2024_analysis", data) +await artifacts.save("github_repos_2024_analysis", data) # Bad -artifacts.save("data1", data) +await artifacts.save("data1", data) ``` **Clean up old artifacts:** ```python # Remove artifacts you no longer need -artifacts.delete("temp_processing_results") +await artifacts.delete("temp_processing_results") ``` **Consider data size:** @@ -134,7 +134,7 @@ artifacts.delete("temp_processing_results") Artifacts automatically track metadata: ```python -artifacts_list = artifacts.list() +artifacts_list = await artifacts.list() for artifact in artifacts_list: print(f"{artifact['name']}: {artifact['created_at']}") ``` diff --git a/docs/executors.md b/docs/executors.md index f491439..1c6c387 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -1,6 +1,45 @@ # Executors -Executors determine where and how agent code runs. Three backends are available: Subprocess, Container, and InProcess. +Executors determine where and how agent code runs. Four backends are available: Subprocess, Container, InProcess, and DenoPyodide (experimental). + +## DenoPyodideExecutor (Experimental, Sandboxed) + +`DenoPyodideExecutor` runs Python in **Pyodide (WASM)** inside a **Deno** subprocess. It relies on the Deno permission model for sandboxing. + +Key differences vs the other executors: +- **Async-first sandbox API**: use `await tools.*`, `await workflows.*`, `await artifacts.*`, `await deps.*`. +- **Best-effort deps**: dependency installs run via Pyodide `micropip` and many packages (especially those requiring native extensions) will not work. + +```python +from pathlib import Path +from py_code_mode import Session, FileStorage +from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + +storage = FileStorage(base_path=Path("./data")) + +config = DenoPyodideConfig( + tools_path=Path("./tools"), + deno_dir=Path("./.deno-cache"), # Deno cache directory (used with --cached-only) + network_profile="deps-only", # "none" | "deps-only" | "full" + default_timeout=60.0, +) + +executor = DenoPyodideExecutor(config) + +async with Session(storage=storage, executor=executor) as session: + result = await session.run("await tools.list()") +``` + +### Network Profiles + +`DenoPyodideConfig.network_profile` controls network access for the Deno subprocess: +- `none`: deny all network access (no runtime dep installs) +- `deps-only`: allow access to PyPI/CDN hosts needed for common `micropip` installs +- `full`: allow all network access + +### Timeout And Reset + +Timeouts are **soft** (the host stops waiting). If an execution times out, the session may be wedged until you call `session.reset()`, which restarts the sandbox. ## Quick Decision Guide @@ -17,20 +56,24 @@ Need stronger isolation? → ContainerExecutor - Filesystem and network isolation - Requires Docker +Want sandboxing without Docker (and can accept Pyodide limitations)? → DenoPyodideExecutor (experimental) + - WASM-based Python runtime + Deno permission model + - Network and filesystem sandboxing via Deno permissions + Need maximum speed AND trust the code completely? → InProcessExecutor - No isolation (runs in your process) - Only for trusted code you control ``` -| Requirement | Subprocess | Container | InProcess | -|-------------|------------|-----------|-----------| -| **Recommended for most users** | **Yes** | | | -| Process isolation | Yes | Yes | No | -| Crash recovery | Yes | Yes | No | -| Container isolation | No | Yes | No | -| No Docker required | Yes | No | Yes | -| Resource limits | Partial | Full | No | -| Untrusted code | No | Yes | No | +| Requirement | Subprocess | Container | DenoPyodide | InProcess | +|-------------|------------|-----------|------------|-----------| +| **Recommended for most users** | **Yes** | | | | +| Process isolation | Yes | Yes | Yes | No | +| Crash recovery | Yes | Yes | Yes | No | +| Container isolation | No | Yes | No | No | +| No Docker required | Yes | No | Yes | Yes | +| Resource limits | Partial | Full | Partial | No | +| Untrusted code | No | Yes | Yes (experimental) | No | --- diff --git a/docs/getting-started.md b/docs/getting-started.md index 048d7fa..8ee5ebb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -48,14 +48,15 @@ from py_code_mode import Session # One line setup - auto-discovers tools/, workflows/, artifacts/, requirements.txt async with Session.from_base("./data") as session: result = await session.run(''' +# This code runs inside the executor sandbox and supports top-level `await`. # Search for existing workflows -results = workflows.search("data processing") +results = await workflows.search("data processing") # List available tools -all_tools = tools.list() +all_tools = await tools.list() # Create a simple workflow -workflows.create( +await workflows.create( name="hello_world", source="""async def run(name: str = "World") -> str: return f"Hello, {name}!" @@ -64,7 +65,7 @@ workflows.create( ) # Invoke the workflow -greeting = workflows.invoke("hello_world", name="Python") +greeting = await workflows.invoke("hello_world", name="Python") print(greeting) ''') @@ -105,23 +106,23 @@ Claude will use the `search_workflows` MCP tool automatically. ```python # 1. Search -results = workflows.search("fetch json from url") +results = await workflows.search("fetch json from url") # 2. Invoke if found if results: - data = workflows.invoke(results[0]["name"], url="https://api.example.com/data") + data = await workflows.invoke(results[0]["name"], url="https://api.example.com/data") else: # 3. Script the solution import json - response = tools.curl.get(url="https://api.example.com/data") + response = await tools.curl.get(url="https://api.example.com/data") data = json.loads(response) # 4. Save as workflow - workflows.create( + await workflows.create( name="fetch_json", source='''async def run(url: str) -> dict: import json - response = tools.curl.get(url=url) + response = await tools.curl.get(url=url) return json.loads(response) ''', description="Fetch and parse JSON from a URL" diff --git a/docs/integrations.md b/docs/integrations.md index 4fb449a..f804943 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -157,16 +157,16 @@ When registering with your framework, provide a clear tool description: TOOL_DESCRIPTION = """Execute Python code with access to tools, workflows, and artifacts. NAMESPACES: -- tools.* - Call registered tools (e.g., tools.curl.get(url="...")) -- workflows.* - Invoke reusable workflows (e.g., workflows.invoke("fetch_json", url="...")) -- artifacts.* - Persist data (e.g., artifacts.save("key", data)) -- deps.* - Manage packages (e.g., deps.add("pandas")) +- tools.* - Call registered tools (e.g., await tools.curl.get(url="...")) +- workflows.* - Invoke reusable workflows (e.g., await workflows.invoke("fetch_json", url="...")) +- artifacts.* - Persist data (e.g., await artifacts.save("key", data)) +- deps.* - Manage packages (e.g., await deps.add("pandas")) Variables persist across calls within the same session. WORKFLOW: -1. Search for existing workflows: workflows.search("your task") -2. If found, invoke it: workflows.invoke("workflow_name", arg=value) +1. Search for existing workflows: await workflows.search("your task") +2. If found, invoke it: await workflows.invoke("workflow_name", arg=value) 3. Otherwise, write code using tools 4. Save successful workflows as workflows for reuse """ diff --git a/docs/session-api.md b/docs/session-api.md index 4070959..b3d0a64 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -51,7 +51,7 @@ Session.from_base( ```python async with Session.from_base("./.code-mode") as session: - await session.run("tools.list()") + await session.run("await tools.list()") ``` ### subprocess() diff --git a/docs/storage.md b/docs/storage.md index 307a363..c692627 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -91,7 +91,7 @@ RedisStorage( # Agent Instance 1 async with Session(storage=redis_storage) as session: await session.run(''' -workflows.create( +await workflows.create( name="analyze_sentiment", source="""async def run(text: str) -> dict: # Implementation @@ -104,7 +104,7 @@ workflows.create( # Agent Instance 2 (different process, different machine) async with Session(storage=redis_storage) as session: # Workflow is already available! - result = await session.run('workflows.invoke("analyze_sentiment", text="Great product!")') + result = await session.run('await workflows.invoke("analyze_sentiment", text="Great product!")') ``` --- diff --git a/docs/tools.md b/docs/tools.md index 2597817..b80463b 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -89,13 +89,15 @@ recipes: ### Agent Usage +Code executed via `Session.run()` supports **top-level `await`**. For portability across executors (and because `DenoPyodideExecutor` is async-first), prefer `await` when calling tools. + ```python # Recipe invocation (recommended) -tools.curl.get(url="https://api.github.com/repos/owner/repo") -tools.curl.post(url="https://api.example.com/data", data='{"key": "value"}') +await tools.curl.get(url="https://api.github.com/repos/owner/repo") +await tools.curl.post(url="https://api.example.com/data", data='{"key": "value"}') # Escape hatch - raw tool invocation (full control) -tools.curl( +await tools.curl( url="https://example.com", silent=True, location=True, @@ -103,9 +105,9 @@ tools.curl( ) # Discovery -tools.list() # All tools -tools.search("http") # Search by name/description/tags -tools.curl.list() # Recipes for a specific tool +await tools.list() # All tools +await tools.search("http") # Search by name/description/tags +await tools.curl.list() # Recipes for a specific tool ``` ## MCP Tools @@ -192,15 +194,15 @@ Agents can discover and search tools: ```python # List all available tools -all_tools = tools.list() +all_tools = await tools.list() # Returns: [Tool(name="curl", description="...", callables=[...]), ...] # Search by keyword -http_tools = tools.search("http") +http_tools = await tools.search("http") # Searches tool names, descriptions, and tags # List recipes for a tool -curl_recipes = tools.curl.list() +curl_recipes = await tools.curl.list() # Returns: [{"name": "get", "description": "...", "params": {...}}, ...] ``` diff --git a/docs/workflows.md b/docs/workflows.md index b0ca0c5..7bff1ce 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -31,7 +31,7 @@ async def run(url: str, headers: dict = None) -> dict: """ import json try: - response = tools.curl.get(url=url) + response = await tools.curl.get(url=url) return json.loads(response) except json.JSONDecodeError as e: raise RuntimeError(f"Invalid JSON from {url}: {e}") from e @@ -44,12 +44,12 @@ async def run(url: str, headers: dict = None) -> dict: Agents can create workflows dynamically: ```python -workflows.create( +await workflows.create( name="fetch_json", source='''async def run(url: str) -> dict: """Fetch and parse JSON from a URL.""" import json - response = tools.curl.get(url=url) + response = await tools.curl.get(url=url) return json.loads(response) ''', description="Fetch JSON from URL and parse response" @@ -62,14 +62,14 @@ Workflows support semantic search based on descriptions: ```python # Search by intent -results = workflows.search("fetch github repository data") +results = await workflows.search("fetch github repository data") # Returns workflows ranked by relevance to the query # List all workflows -all_workflows = workflows.list() +all_workflows = await workflows.list() # Get specific workflow details -workflow = workflows.get("fetch_json") +workflow = await workflows.get("fetch_json") ``` The search uses embedding-based similarity, so it understands intent even if the exact words don't match. @@ -78,10 +78,10 @@ The search uses embedding-based similarity, so it understands intent even if the ```python # Direct invocation -data = workflows.invoke("fetch_json", url="https://api.github.com/repos/owner/repo") +data = await workflows.invoke("fetch_json", url="https://api.github.com/repos/owner/repo") # With keyword arguments -analysis = workflows.invoke( +analysis = await workflows.invoke( "analyze_repo", owner="anthropics", repo="anthropic-sdk-python" @@ -99,7 +99,7 @@ Workflows can invoke other workflows, enabling layered workflows: async def run(url: str) -> dict: """Fetch and parse JSON from a URL.""" import json - response = tools.curl.get(url=url) + response = await tools.curl.get(url=url) return json.loads(response) ``` @@ -110,8 +110,10 @@ async def run(url: str) -> dict: async def run(owner: str, repo: str) -> dict: """Get GitHub repository metadata.""" # Uses the fetch_json workflow - data = workflows.invoke("fetch_json", - url=f"https://api.github.com/repos/{owner}/{repo}") + data = await workflows.invoke( + "fetch_json", + url=f"https://api.github.com/repos/{owner}/{repo}", + ) return { "name": data["name"], @@ -131,7 +133,7 @@ async def run(repos: list) -> dict: for repo in repos: owner, name = repo.split('/') # Uses the get_repo_metadata workflow - metadata = workflows.invoke("get_repo_metadata", owner=owner, repo=name) + metadata = await workflows.invoke("get_repo_metadata", owner=owner, repo=name) summaries.append(metadata) # Aggregate results @@ -229,7 +231,7 @@ async def run(repo_url: str, incl_contrib: bool = False) -> dict: ```python # Delete a workflow by name -workflows.delete("old_workflow_name") +await workflows.delete("old_workflow_name") ``` ### Updating Workflows @@ -238,10 +240,10 @@ Workflows are immutable. To update, delete and recreate: ```python # Delete old version -workflows.delete("fetch_json") +await workflows.delete("fetch_json") # Create new version -workflows.create( +await workflows.create( name="fetch_json", source='''async def run(url: str, timeout: int = 30) -> dict: # Updated implementation with timeout @@ -264,7 +266,7 @@ Create `.py` files in the workflows directory: """Fetch a URL and extract key information.""" async def run(url: str) -> dict: - content = tools.fetch(url=url) + content = await tools.fetch(url=url) paragraphs = [p for p in content.split("\n\n") if p.strip()] return { "url": url, From 84a5ff2bace881940f19454e8e26181873bdbb58 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:22:02 -0800 Subject: [PATCH 12/27] Deno sandbox: friendly aliases + docs --- docs/executors.md | 22 +++-- docs/tools.md | 2 +- src/py_code_mode/execution/__init__.py | 11 ++- .../execution/deno_pyodide/__init__.py | 12 ++- .../execution/deno_pyodide/executor.py | 1 + tests/test_deno_pyodide_executor.py | 90 +++++++++---------- tests/test_deno_sandbox_alias.py | 18 ++++ 7 files changed, 99 insertions(+), 57 deletions(-) create mode 100644 tests/test_deno_sandbox_alias.py diff --git a/docs/executors.md b/docs/executors.md index 1c6c387..46c20a5 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -1,10 +1,14 @@ # Executors -Executors determine where and how agent code runs. Four backends are available: Subprocess, Container, InProcess, and DenoPyodide (experimental). +Executors determine where and how agent code runs. Four backends are available: Subprocess, Container, InProcess, and DenoSandbox (experimental). -## DenoPyodideExecutor (Experimental, Sandboxed) +## DenoSandboxExecutor (Experimental, Sandboxed) -`DenoPyodideExecutor` runs Python in **Pyodide (WASM)** inside a **Deno** subprocess. It relies on the Deno permission model for sandboxing. +`DenoSandboxExecutor` runs Python in **Pyodide (WASM)** inside a **Deno** subprocess. It relies on the Deno permission model for sandboxing. + +Notes: +- `DenoSandboxExecutor` is the friendly public name (an alias of `DenoPyodideExecutor`). +- Backend keys: `"deno-sandbox"` and `"deno-pyodide"`. Key differences vs the other executors: - **Async-first sandbox API**: use `await tools.*`, `await workflows.*`, `await artifacts.*`, `await deps.*`. @@ -13,18 +17,18 @@ Key differences vs the other executors: ```python from pathlib import Path from py_code_mode import Session, FileStorage -from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor +from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor storage = FileStorage(base_path=Path("./data")) -config = DenoPyodideConfig( +config = DenoSandboxConfig( tools_path=Path("./tools"), deno_dir=Path("./.deno-cache"), # Deno cache directory (used with --cached-only) network_profile="deps-only", # "none" | "deps-only" | "full" default_timeout=60.0, ) -executor = DenoPyodideExecutor(config) +executor = DenoSandboxExecutor(config) async with Session(storage=storage, executor=executor) as session: result = await session.run("await tools.list()") @@ -32,7 +36,7 @@ async with Session(storage=storage, executor=executor) as session: ### Network Profiles -`DenoPyodideConfig.network_profile` controls network access for the Deno subprocess: +`DenoSandboxConfig.network_profile` controls network access for the Deno subprocess: - `none`: deny all network access (no runtime dep installs) - `deps-only`: allow access to PyPI/CDN hosts needed for common `micropip` installs - `full`: allow all network access @@ -56,7 +60,7 @@ Need stronger isolation? → ContainerExecutor - Filesystem and network isolation - Requires Docker -Want sandboxing without Docker (and can accept Pyodide limitations)? → DenoPyodideExecutor (experimental) +Want sandboxing without Docker (and can accept Pyodide limitations)? → DenoSandboxExecutor (experimental) - WASM-based Python runtime + Deno permission model - Network and filesystem sandboxing via Deno permissions @@ -65,7 +69,7 @@ Need maximum speed AND trust the code completely? → InProcessExecutor - Only for trusted code you control ``` -| Requirement | Subprocess | Container | DenoPyodide | InProcess | +| Requirement | Subprocess | Container | DenoSandbox | InProcess | |-------------|------------|-----------|------------|-----------| | **Recommended for most users** | **Yes** | | | | | Process isolation | Yes | Yes | Yes | No | diff --git a/docs/tools.md b/docs/tools.md index b80463b..c93630c 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -89,7 +89,7 @@ recipes: ### Agent Usage -Code executed via `Session.run()` supports **top-level `await`**. For portability across executors (and because `DenoPyodideExecutor` is async-first), prefer `await` when calling tools. +Code executed via `Session.run()` supports **top-level `await`**. For portability across executors (and because `DenoSandboxExecutor` is async-first), prefer `await` when calling tools. ```python # Recipe invocation (recommended) diff --git a/src/py_code_mode/execution/__init__.py b/src/py_code_mode/execution/__init__.py index c5ca16c..d982831 100644 --- a/src/py_code_mode/execution/__init__.py +++ b/src/py_code_mode/execution/__init__.py @@ -37,13 +37,20 @@ # Deno/Pyodide is optional at runtime (requires deno + pyodide assets). try: - from py_code_mode.execution.deno_pyodide import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution.deno_pyodide import ( + DenoPyodideConfig, + DenoPyodideExecutor, + DenoSandboxConfig, + DenoSandboxExecutor, + ) DENO_PYODIDE_AVAILABLE = True except Exception: DENO_PYODIDE_AVAILABLE = False DenoPyodideConfig = None # type: ignore DenoPyodideExecutor = None # type: ignore + DenoSandboxConfig = None # type: ignore + DenoSandboxExecutor = None # type: ignore __all__ = [ "Capability", @@ -64,5 +71,7 @@ "SUBPROCESS_AVAILABLE", "DenoPyodideExecutor", "DenoPyodideConfig", + "DenoSandboxExecutor", + "DenoSandboxConfig", "DENO_PYODIDE_AVAILABLE", ] diff --git a/src/py_code_mode/execution/deno_pyodide/__init__.py b/src/py_code_mode/execution/deno_pyodide/__init__.py index 126365e..aeeb31e 100644 --- a/src/py_code_mode/execution/deno_pyodide/__init__.py +++ b/src/py_code_mode/execution/deno_pyodide/__init__.py @@ -7,4 +7,14 @@ from py_code_mode.execution.deno_pyodide.config import DenoPyodideConfig from py_code_mode.execution.deno_pyodide.executor import DenoPyodideExecutor -__all__ = ["DenoPyodideConfig", "DenoPyodideExecutor"] +# Friendly public names. These describe the execution style (sandboxed via Deno +# permissions + Pyodide), not the implementation internals. +DenoSandboxConfig = DenoPyodideConfig +DenoSandboxExecutor = DenoPyodideExecutor + +__all__ = [ + "DenoPyodideConfig", + "DenoPyodideExecutor", + "DenoSandboxConfig", + "DenoSandboxExecutor", +] diff --git a/src/py_code_mode/execution/deno_pyodide/executor.py b/src/py_code_mode/execution/deno_pyodide/executor.py index 52f4a77..04d6abf 100644 --- a/src/py_code_mode/execution/deno_pyodide/executor.py +++ b/src/py_code_mode/execution/deno_pyodide/executor.py @@ -610,3 +610,4 @@ async def sync_deps(self) -> dict[str, Any]: register_backend("deno-pyodide", DenoPyodideExecutor) +register_backend("deno-sandbox", DenoPyodideExecutor) diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_pyodide_executor.py index 2ddd43d..178ed56 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_pyodide_executor.py @@ -80,7 +80,7 @@ def get_artifact_store(self): @pytest.mark.asyncio async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -88,8 +88,8 @@ async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, @@ -112,7 +112,7 @@ async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -120,8 +120,8 @@ async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=300.0, deps_timeout=300.0, @@ -141,7 +141,7 @@ async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -149,8 +149,8 @@ async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, deps_timeout=300.0, @@ -169,7 +169,7 @@ async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -177,8 +177,8 @@ async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=120.0, deps_timeout=120.0, @@ -196,7 +196,7 @@ async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path @pytest.mark.asyncio async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -204,8 +204,8 @@ async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, @@ -224,7 +224,7 @@ async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None @pytest.mark.asyncio async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -232,8 +232,8 @@ async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, @@ -260,7 +260,7 @@ async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None @pytest.mark.asyncio async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -294,8 +294,8 @@ async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, tools_path=tools_dir, default_timeout=60.0, @@ -316,7 +316,7 @@ async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -350,8 +350,8 @@ async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, tools_path=tools_dir, default_timeout=180.0, @@ -375,7 +375,7 @@ async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path @pytest.mark.asyncio async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -383,8 +383,8 @@ async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> No deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=15.0, ipc_timeout=120.0, @@ -411,7 +411,7 @@ async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> No @pytest.mark.asyncio async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -419,8 +419,8 @@ async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, @@ -441,7 +441,7 @@ async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -449,8 +449,8 @@ async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=300.0, deps_timeout=300.0, @@ -470,7 +470,7 @@ async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> @pytest.mark.asyncio async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -517,8 +517,8 @@ async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, tools_path=tools_dir, default_timeout=60.0, @@ -539,15 +539,15 @@ async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session storage = _TestStorage(tmp_path / "storage") deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=60.0, ipc_timeout=120.0, @@ -568,7 +568,7 @@ async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> @pytest.mark.asyncio async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -579,8 +579,8 @@ async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=180.0, ipc_timeout=120.0, @@ -612,7 +612,7 @@ async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path @pytest.mark.asyncio async def test_deno_pyodide_executor_soft_timeout_wedges_until_reset(tmp_path: Path) -> None: - from py_code_mode.execution import DenoPyodideConfig, DenoPyodideExecutor + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -620,8 +620,8 @@ async def test_deno_pyodide_executor_soft_timeout_wedges_until_reset(tmp_path: P deno_dir = tmp_path / "deno_dir" deno_dir.mkdir(parents=True, exist_ok=True) - executor = DenoPyodideExecutor( - DenoPyodideConfig( + executor = DenoSandboxExecutor( + DenoSandboxConfig( deno_dir=deno_dir, default_timeout=0.05, ipc_timeout=120.0, diff --git a/tests/test_deno_sandbox_alias.py b/tests/test_deno_sandbox_alias.py new file mode 100644 index 0000000..4daee10 --- /dev/null +++ b/tests/test_deno_sandbox_alias.py @@ -0,0 +1,18 @@ +import pytest + + +def test_deno_sandbox_aliases_exist() -> None: + from py_code_mode.execution import DENO_PYODIDE_AVAILABLE + + if not DENO_PYODIDE_AVAILABLE: + pytest.skip("Deno sandbox backend is optional and not available in this environment.") + + from py_code_mode.execution import ( + DenoPyodideConfig, + DenoPyodideExecutor, + DenoSandboxConfig, + DenoSandboxExecutor, + ) + + assert DenoSandboxConfig is DenoPyodideConfig + assert DenoSandboxExecutor is DenoPyodideExecutor From 4ccb2848a6a67ac095bc1a24c47d662fad499414 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 13:14:08 -0800 Subject: [PATCH 13/27] Deno sandbox: rename + keep agent API sync --- docs/ARCHITECTURE.md | 14 +- docs/artifacts.md | 40 ++--- docs/executors.md | 3 +- docs/getting-started.md | 19 ++- docs/integrations.md | 12 +- docs/session-api.md | 2 +- docs/storage.md | 4 +- docs/tools.md | 28 ++-- docs/workflows.md | 45 +++--- src/py_code_mode/artifacts/namespace.py | 54 +------ src/py_code_mode/deps/async_namespace.py | 9 +- src/py_code_mode/execution/__init__.py | 19 +-- .../execution/deno_pyodide/__init__.py | 20 --- .../execution/deno_sandbox/__init__.py | 10 ++ .../{deno_pyodide => deno_sandbox}/config.py | 6 +- .../executor.py | 17 +-- .../runner/main.ts | 0 .../runner/worker.ts | 0 .../execution/in_process/executor.py | 23 ++- .../in_process/workflows_namespace.py | 112 +++----------- .../execution/subprocess/kernel_init.py | 22 ++- src/py_code_mode/tools/namespace.py | 139 +++++++++--------- tests/test_async_sandbox_interfaces.py | 28 ++-- tests/test_deno_sandbox_alias.py | 18 --- ...cutor.py => test_deno_sandbox_executor.py} | 30 ++-- tests/test_deno_sandbox_imports.py | 13 ++ tests/test_namespace.py | 2 +- tests/test_tool_proxy_explicit_methods.py | 9 +- 28 files changed, 279 insertions(+), 419 deletions(-) delete mode 100644 src/py_code_mode/execution/deno_pyodide/__init__.py create mode 100644 src/py_code_mode/execution/deno_sandbox/__init__.py rename src/py_code_mode/execution/{deno_pyodide => deno_sandbox}/config.py (92%) rename src/py_code_mode/execution/{deno_pyodide => deno_sandbox}/executor.py (97%) rename src/py_code_mode/execution/{deno_pyodide => deno_sandbox}/runner/main.ts (100%) rename src/py_code_mode/execution/{deno_pyodide => deno_sandbox}/runner/worker.ts (100%) delete mode 100644 tests/test_deno_sandbox_alias.py rename tests/{test_deno_pyodide_executor.py => test_deno_sandbox_executor.py} (95%) create mode 100644 tests/test_deno_sandbox_imports.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1d6758b..af5b800 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -791,14 +791,14 @@ async with Session(storage=storage, executor=executor) as session: ### Tool Execution ``` -Agent writes: "await tools.curl.get(url='...')" +Agent writes: "tools.curl.get(url='...')" (use `await` only in DenoSandboxExecutor) | v +------------------------+ | ToolsNamespace | | | | tools.curl(url=...) |--> Escape hatch (direct invocation) -| tools.curl.get(...) |--> Recipe invocation (awaitable in async contexts) +| tools.curl.get(...) |--> Recipe invocation | tools.search(...) | | | tools.list() | v +------------------------+ +--------------+ @@ -811,7 +811,7 @@ Agent writes: "await tools.curl.get(url='...')" ### ToolProxy Methods ``` -Agent writes: "await tools.curl.get(url='...')" +Agent writes: "tools.curl.get(url='...')" | v +------------------------+ @@ -819,7 +819,7 @@ Agent writes: "await tools.curl.get(url='...')" | | | .call_async(**kwargs) |--> Always returns awaitable | .call_sync(**kwargs) |--> Always blocks, returns result -| .__call__(**kwargs) |--> Context-aware (sync/async detection) +| .__call__(**kwargs) |--> Synchronous invocation +------------------------+ | v @@ -828,14 +828,14 @@ Agent writes: "await tools.curl.get(url='...')" | | | .call_async(**kwargs) |--> Always returns awaitable | .call_sync(**kwargs) |--> Always blocks, returns result -| .__call__(**kwargs) |--> Context-aware (sync/async detection) +| .__call__(**kwargs) |--> Synchronous invocation +------------------------+ ``` ### Skill Execution ``` -Agent writes: "await workflows.analyze_repo(repo='...')" +Agent writes: "workflows.analyze_repo(repo='...')" (use `await` only in DenoSandboxExecutor) | v +------------------------+ @@ -880,7 +880,7 @@ Skill has access to: ### Artifact Storage ``` -Agent writes: "await artifacts.save('data.json', b'...', 'description')" +Agent writes: "artifacts.save('data.json', b'...', 'description')" (use `await` only in DenoSandboxExecutor) | v +------------------------+ diff --git a/docs/artifacts.md b/docs/artifacts.md index 23e3273..38e2160 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -6,19 +6,19 @@ Artifacts provide persistent data storage across sessions. Use them to cache res ```python # Save data -await artifacts.save("analysis_results", { +artifacts.save("analysis_results", { "repos_analyzed": 42, "findings": [...] }) # Load data -data = await artifacts.load("analysis_results") +data = artifacts.load("analysis_results") # List all artifacts -all_artifacts = await artifacts.list() +all_artifacts = artifacts.list() # Delete an artifact -await artifacts.delete("old_data") +artifacts.delete("old_data") ``` ## Storage Formats @@ -27,13 +27,13 @@ Artifacts automatically handle serialization based on data type: ```python # JSON-serializable data (dicts, lists, primitives) -await artifacts.save("config", {"api_key": "...", "timeout": 30}) +artifacts.save("config", {"api_key": "...", "timeout": 30}) # Binary data -await artifacts.save("image", image_bytes) +artifacts.save("image", image_bytes) # Text data -await artifacts.save("report", "Analysis results: ...") +artifacts.save("report", "Analysis results: ...") ``` ## Use Cases @@ -45,15 +45,15 @@ async def run(owner: str, repo: str) -> dict: cache_key = f"repo_{owner}_{repo}" # Check cache first - cached = await artifacts.load(cache_key) + cached = artifacts.load(cache_key) if cached: return cached # Fetch fresh data - data = await tools.curl.get(url=f"https://api.github.com/repos/{owner}/{repo}") + data = tools.curl.get(url=f"https://api.github.com/repos/{owner}/{repo}") # Cache for next time - await artifacts.save(cache_key, data) + artifacts.save(cache_key, data) return data ``` @@ -62,17 +62,17 @@ async def run(owner: str, repo: str) -> dict: ```python async def run(url: str) -> dict: # Load previous crawl state - state = await artifacts.load("crawl_state") or {"visited": [], "queue": []} + state = artifacts.load("crawl_state") or {"visited": [], "queue": []} if url in state["visited"]: return {"status": "already_crawled"} # Process URL - content = await tools.fetch(url=url) + content = tools.fetch(url=url) state["visited"].append(url) # Save updated state - await artifacts.save("crawl_state", state) + artifacts.save("crawl_state", state) return {"status": "success", "content": content} ``` @@ -82,14 +82,14 @@ async def run(url: str) -> dict: # Skill 1: Collect data async def run(sources: list) -> dict: results = [fetch_source(s) for s in sources] - await artifacts.save("collected_data", results) + artifacts.save("collected_data", results) return {"count": len(results)} # Skill 2: Analyze data async def run() -> dict: - data = await artifacts.load("collected_data") + data = artifacts.load("collected_data") analysis = analyze(data) - await artifacts.save("analysis_report", analysis) + artifacts.save("analysis_report", analysis) return analysis ``` @@ -112,16 +112,16 @@ Artifacts are stored according to your storage backend: **Use descriptive names:** ```python # Good -await artifacts.save("github_repos_2024_analysis", data) +artifacts.save("github_repos_2024_analysis", data) # Bad -await artifacts.save("data1", data) +artifacts.save("data1", data) ``` **Clean up old artifacts:** ```python # Remove artifacts you no longer need -await artifacts.delete("temp_processing_results") +artifacts.delete("temp_processing_results") ``` **Consider data size:** @@ -134,7 +134,7 @@ await artifacts.delete("temp_processing_results") Artifacts automatically track metadata: ```python -artifacts_list = await artifacts.list() +artifacts_list = artifacts.list() for artifact in artifacts_list: print(f"{artifact['name']}: {artifact['created_at']}") ``` diff --git a/docs/executors.md b/docs/executors.md index 46c20a5..9ef852b 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -7,8 +7,7 @@ Executors determine where and how agent code runs. Four backends are available: `DenoSandboxExecutor` runs Python in **Pyodide (WASM)** inside a **Deno** subprocess. It relies on the Deno permission model for sandboxing. Notes: -- `DenoSandboxExecutor` is the friendly public name (an alias of `DenoPyodideExecutor`). -- Backend keys: `"deno-sandbox"` and `"deno-pyodide"`. +- Backend key: `"deno-sandbox"`. Key differences vs the other executors: - **Async-first sandbox API**: use `await tools.*`, `await workflows.*`, `await artifacts.*`, `await deps.*`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 8ee5ebb..048d7fa 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -48,15 +48,14 @@ from py_code_mode import Session # One line setup - auto-discovers tools/, workflows/, artifacts/, requirements.txt async with Session.from_base("./data") as session: result = await session.run(''' -# This code runs inside the executor sandbox and supports top-level `await`. # Search for existing workflows -results = await workflows.search("data processing") +results = workflows.search("data processing") # List available tools -all_tools = await tools.list() +all_tools = tools.list() # Create a simple workflow -await workflows.create( +workflows.create( name="hello_world", source="""async def run(name: str = "World") -> str: return f"Hello, {name}!" @@ -65,7 +64,7 @@ await workflows.create( ) # Invoke the workflow -greeting = await workflows.invoke("hello_world", name="Python") +greeting = workflows.invoke("hello_world", name="Python") print(greeting) ''') @@ -106,23 +105,23 @@ Claude will use the `search_workflows` MCP tool automatically. ```python # 1. Search -results = await workflows.search("fetch json from url") +results = workflows.search("fetch json from url") # 2. Invoke if found if results: - data = await workflows.invoke(results[0]["name"], url="https://api.example.com/data") + data = workflows.invoke(results[0]["name"], url="https://api.example.com/data") else: # 3. Script the solution import json - response = await tools.curl.get(url="https://api.example.com/data") + response = tools.curl.get(url="https://api.example.com/data") data = json.loads(response) # 4. Save as workflow - await workflows.create( + workflows.create( name="fetch_json", source='''async def run(url: str) -> dict: import json - response = await tools.curl.get(url=url) + response = tools.curl.get(url=url) return json.loads(response) ''', description="Fetch and parse JSON from a URL" diff --git a/docs/integrations.md b/docs/integrations.md index f804943..4fb449a 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -157,16 +157,16 @@ When registering with your framework, provide a clear tool description: TOOL_DESCRIPTION = """Execute Python code with access to tools, workflows, and artifacts. NAMESPACES: -- tools.* - Call registered tools (e.g., await tools.curl.get(url="...")) -- workflows.* - Invoke reusable workflows (e.g., await workflows.invoke("fetch_json", url="...")) -- artifacts.* - Persist data (e.g., await artifacts.save("key", data)) -- deps.* - Manage packages (e.g., await deps.add("pandas")) +- tools.* - Call registered tools (e.g., tools.curl.get(url="...")) +- workflows.* - Invoke reusable workflows (e.g., workflows.invoke("fetch_json", url="...")) +- artifacts.* - Persist data (e.g., artifacts.save("key", data)) +- deps.* - Manage packages (e.g., deps.add("pandas")) Variables persist across calls within the same session. WORKFLOW: -1. Search for existing workflows: await workflows.search("your task") -2. If found, invoke it: await workflows.invoke("workflow_name", arg=value) +1. Search for existing workflows: workflows.search("your task") +2. If found, invoke it: workflows.invoke("workflow_name", arg=value) 3. Otherwise, write code using tools 4. Save successful workflows as workflows for reuse """ diff --git a/docs/session-api.md b/docs/session-api.md index b3d0a64..4070959 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -51,7 +51,7 @@ Session.from_base( ```python async with Session.from_base("./.code-mode") as session: - await session.run("await tools.list()") + await session.run("tools.list()") ``` ### subprocess() diff --git a/docs/storage.md b/docs/storage.md index c692627..307a363 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -91,7 +91,7 @@ RedisStorage( # Agent Instance 1 async with Session(storage=redis_storage) as session: await session.run(''' -await workflows.create( +workflows.create( name="analyze_sentiment", source="""async def run(text: str) -> dict: # Implementation @@ -104,7 +104,7 @@ await workflows.create( # Agent Instance 2 (different process, different machine) async with Session(storage=redis_storage) as session: # Workflow is already available! - result = await session.run('await workflows.invoke("analyze_sentiment", text="Great product!")') + result = await session.run('workflows.invoke("analyze_sentiment", text="Great product!")') ``` --- diff --git a/docs/tools.md b/docs/tools.md index c93630c..b31ee9e 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -89,15 +89,19 @@ recipes: ### Agent Usage -Code executed via `Session.run()` supports **top-level `await`**. For portability across executors (and because `DenoSandboxExecutor` is async-first), prefer `await` when calling tools. +Tool calls inside `Session.run()` are **synchronous** in the default executors (Subprocess/Container/InProcess). + +Notes: +- If you need async tool calls in Python code, use `call_async(...)` explicitly. +- In `DenoSandboxExecutor`, tool calls are **async-first** and you must use `await tools.*`. ```python # Recipe invocation (recommended) -await tools.curl.get(url="https://api.github.com/repos/owner/repo") -await tools.curl.post(url="https://api.example.com/data", data='{"key": "value"}') +tools.curl.get(url="https://api.github.com/repos/owner/repo") +tools.curl.post(url="https://api.example.com/data", data='{"key": "value"}') # Escape hatch - raw tool invocation (full control) -await tools.curl( +tools.curl( url="https://example.com", silent=True, location=True, @@ -105,9 +109,13 @@ await tools.curl( ) # Discovery -await tools.list() # All tools -await tools.search("http") # Search by name/description/tags -await tools.curl.list() # Recipes for a specific tool +tools.list() # All tools +tools.search("http") # Search by name/description/tags +tools.curl.list() # Recipes for a specific tool + +# Explicit async (if you need it) +await tools.curl.call_async(url="https://example.com") +await tools.curl.get.call_async(url="https://example.com") ``` ## MCP Tools @@ -194,15 +202,15 @@ Agents can discover and search tools: ```python # List all available tools -all_tools = await tools.list() +all_tools = tools.list() # Returns: [Tool(name="curl", description="...", callables=[...]), ...] # Search by keyword -http_tools = await tools.search("http") +http_tools = tools.search("http") # Searches tool names, descriptions, and tags # List recipes for a tool -curl_recipes = await tools.curl.list() +curl_recipes = tools.curl.list() # Returns: [{"name": "get", "description": "...", "params": {...}}, ...] ``` diff --git a/docs/workflows.md b/docs/workflows.md index 7bff1ce..0857f8c 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -10,13 +10,14 @@ Over time, the workflow library grows. Simple workflows become building blocks f ## Creating Workflows -Workflows are async Python functions with an `async def run()` entry point: +Workflows are Python functions with a `run()` entry point. Both `def run(...)` and +`async def run(...)` are supported: ```python # workflows/fetch_json.py """Fetch and parse JSON from a URL.""" -async def run(url: str, headers: dict = None) -> dict: +def run(url: str, headers: dict = None) -> dict: """Fetch JSON data from a URL. Args: @@ -31,25 +32,25 @@ async def run(url: str, headers: dict = None) -> dict: """ import json try: - response = await tools.curl.get(url=url) + response = tools.curl.get(url=url) return json.loads(response) except json.JSONDecodeError as e: raise RuntimeError(f"Invalid JSON from {url}: {e}") from e ``` -> **Note:** All workflows must use `async def run()`. Synchronous `def run()` is not supported. +> **Note:** If your workflow uses `async def run(...)`, it can still call tools/workflows/artifacts synchronously. ### Runtime Creation Agents can create workflows dynamically: ```python -await workflows.create( +workflows.create( name="fetch_json", - source='''async def run(url: str) -> dict: + source='''def run(url: str) -> dict: """Fetch and parse JSON from a URL.""" import json - response = await tools.curl.get(url=url) + response = tools.curl.get(url=url) return json.loads(response) ''', description="Fetch JSON from URL and parse response" @@ -62,14 +63,14 @@ Workflows support semantic search based on descriptions: ```python # Search by intent -results = await workflows.search("fetch github repository data") +results = workflows.search("fetch github repository data") # Returns workflows ranked by relevance to the query # List all workflows -all_workflows = await workflows.list() +all_workflows = workflows.list() # Get specific workflow details -workflow = await workflows.get("fetch_json") +workflow = workflows.get("fetch_json") ``` The search uses embedding-based similarity, so it understands intent even if the exact words don't match. @@ -78,10 +79,10 @@ The search uses embedding-based similarity, so it understands intent even if the ```python # Direct invocation -data = await workflows.invoke("fetch_json", url="https://api.github.com/repos/owner/repo") +data = workflows.invoke("fetch_json", url="https://api.github.com/repos/owner/repo") # With keyword arguments -analysis = await workflows.invoke( +analysis = workflows.invoke( "analyze_repo", owner="anthropics", repo="anthropic-sdk-python" @@ -96,10 +97,10 @@ Workflows can invoke other workflows, enabling layered workflows: ```python # workflows/fetch_json.py -async def run(url: str) -> dict: +def run(url: str) -> dict: """Fetch and parse JSON from a URL.""" import json - response = await tools.curl.get(url=url) + response = tools.curl.get(url=url) return json.loads(response) ``` @@ -107,10 +108,10 @@ async def run(url: str) -> dict: ```python # workflows/get_repo_metadata.py -async def run(owner: str, repo: str) -> dict: +def run(owner: str, repo: str) -> dict: """Get GitHub repository metadata.""" # Uses the fetch_json workflow - data = await workflows.invoke( + data = workflows.invoke( "fetch_json", url=f"https://api.github.com/repos/{owner}/{repo}", ) @@ -127,13 +128,13 @@ async def run(owner: str, repo: str) -> dict: ```python # workflows/analyze_multiple_repos.py -async def run(repos: list) -> dict: +def run(repos: list) -> dict: """Analyze multiple GitHub repositories.""" summaries = [] for repo in repos: owner, name = repo.split('/') # Uses the get_repo_metadata workflow - metadata = await workflows.invoke("get_repo_metadata", owner=owner, repo=name) + metadata = workflows.invoke("get_repo_metadata", owner=owner, repo=name) summaries.append(metadata) # Aggregate results @@ -231,7 +232,7 @@ async def run(repo_url: str, incl_contrib: bool = False) -> dict: ```python # Delete a workflow by name -await workflows.delete("old_workflow_name") +workflows.delete("old_workflow_name") ``` ### Updating Workflows @@ -240,10 +241,10 @@ Workflows are immutable. To update, delete and recreate: ```python # Delete old version -await workflows.delete("fetch_json") +workflows.delete("fetch_json") # Create new version -await workflows.create( +workflows.create( name="fetch_json", source='''async def run(url: str, timeout: int = 30) -> dict: # Updated implementation with timeout @@ -266,7 +267,7 @@ Create `.py` files in the workflows directory: """Fetch a URL and extract key information.""" async def run(url: str) -> dict: - content = await tools.fetch(url=url) + content = tools.fetch(url=url) paragraphs = [p for p in content.split("\n\n") if p.strip()] return { "url": url, diff --git a/src/py_code_mode/artifacts/namespace.py b/src/py_code_mode/artifacts/namespace.py index f16e483..6fb85a1 100644 --- a/src/py_code_mode/artifacts/namespace.py +++ b/src/py_code_mode/artifacts/namespace.py @@ -1,29 +1,13 @@ -"""ArtifactsNamespace - agent-facing API for artifact storage. - -This mirrors the sandbox ergonomics used by the Deno/Pyodide executor: a small, -high-level API exposed as `artifacts.*` inside executed code. - -Design goal: allow both sync usage (`artifacts.load(...)`) and async usage -(`await artifacts.load(...)`) depending on execution context. -""" +"""ArtifactsNamespace - agent-facing API for artifact storage.""" from __future__ import annotations -import asyncio from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from py_code_mode.artifacts.base import ArtifactStoreProtocol -def _in_async_context() -> bool: - try: - asyncio.get_running_loop() - except RuntimeError: - return False - return True - - class ArtifactsNamespace: """Agent-facing namespace for artifacts. @@ -54,56 +38,20 @@ def save( description: str = "", metadata: dict[str, Any] | None = None, ) -> Any: - if _in_async_context(): - - async def _coro() -> Any: - return self._store.save(name, data, description=description, metadata=metadata) - - return _coro() return self._store.save(name, data, description=description, metadata=metadata) def load(self, name: str) -> Any: - if _in_async_context(): - - async def _coro() -> Any: - return self._store.load(name) - - return _coro() return self._store.load(name) def list(self) -> Any: - if _in_async_context(): - - async def _coro() -> Any: - return self._store.list() - - return _coro() return self._store.list() def exists(self, name: str) -> Any: - if _in_async_context(): - - async def _coro() -> bool: - return bool(self._store.exists(name)) - - return _coro() return bool(self._store.exists(name)) def get(self, name: str) -> Any: - if _in_async_context(): - - async def _coro() -> Any: - return self._store.get(name) - - return _coro() return self._store.get(name) def delete(self, name: str) -> Any: - if _in_async_context(): - - async def _coro() -> None: - self._store.delete(name) - - return _coro() self._store.delete(name) return None diff --git a/src/py_code_mode/deps/async_namespace.py b/src/py_code_mode/deps/async_namespace.py index 5ae3b62..d357624 100644 --- a/src/py_code_mode/deps/async_namespace.py +++ b/src/py_code_mode/deps/async_namespace.py @@ -1,11 +1,6 @@ -"""Async-capable wrapper for deps namespace. +"""Deprecated: async-capable wrapper for deps namespace. -The base DepsNamespace API is synchronous (it may run pip). For sandbox -consistency with async executors, this wrapper allows: - - deps.add(...) (sync) - - await deps.add(...) (async) - -The async variants simply run the synchronous operations in-process. +Kept for potential future use. The primary agent-facing API is synchronous. """ from __future__ import annotations diff --git a/src/py_code_mode/execution/__init__.py b/src/py_code_mode/execution/__init__.py index d982831..b2ed079 100644 --- a/src/py_code_mode/execution/__init__.py +++ b/src/py_code_mode/execution/__init__.py @@ -35,20 +35,13 @@ SubprocessConfig = None # type: ignore SubprocessExecutor = None # type: ignore -# Deno/Pyodide is optional at runtime (requires deno + pyodide assets). +# Deno sandbox is optional at runtime (requires deno + pyodide assets). try: - from py_code_mode.execution.deno_pyodide import ( - DenoPyodideConfig, - DenoPyodideExecutor, - DenoSandboxConfig, - DenoSandboxExecutor, - ) + from py_code_mode.execution.deno_sandbox import DenoSandboxConfig, DenoSandboxExecutor - DENO_PYODIDE_AVAILABLE = True + DENO_SANDBOX_AVAILABLE = True except Exception: - DENO_PYODIDE_AVAILABLE = False - DenoPyodideConfig = None # type: ignore - DenoPyodideExecutor = None # type: ignore + DENO_SANDBOX_AVAILABLE = False DenoSandboxConfig = None # type: ignore DenoSandboxExecutor = None # type: ignore @@ -69,9 +62,7 @@ "SubprocessExecutor", "SubprocessConfig", "SUBPROCESS_AVAILABLE", - "DenoPyodideExecutor", - "DenoPyodideConfig", "DenoSandboxExecutor", "DenoSandboxConfig", - "DENO_PYODIDE_AVAILABLE", + "DENO_SANDBOX_AVAILABLE", ] diff --git a/src/py_code_mode/execution/deno_pyodide/__init__.py b/src/py_code_mode/execution/deno_pyodide/__init__.py deleted file mode 100644 index aeeb31e..0000000 --- a/src/py_code_mode/execution/deno_pyodide/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Deno + Pyodide execution backend. - -This backend runs Python inside Pyodide (WASM) hosted by a Deno subprocess. -It is intended to provide a sandboxed runtime without requiring Docker. -""" - -from py_code_mode.execution.deno_pyodide.config import DenoPyodideConfig -from py_code_mode.execution.deno_pyodide.executor import DenoPyodideExecutor - -# Friendly public names. These describe the execution style (sandboxed via Deno -# permissions + Pyodide), not the implementation internals. -DenoSandboxConfig = DenoPyodideConfig -DenoSandboxExecutor = DenoPyodideExecutor - -__all__ = [ - "DenoPyodideConfig", - "DenoPyodideExecutor", - "DenoSandboxConfig", - "DenoSandboxExecutor", -] diff --git a/src/py_code_mode/execution/deno_sandbox/__init__.py b/src/py_code_mode/execution/deno_sandbox/__init__.py new file mode 100644 index 0000000..c3d1e15 --- /dev/null +++ b/src/py_code_mode/execution/deno_sandbox/__init__.py @@ -0,0 +1,10 @@ +"""Deno sandbox execution backend. + +This backend runs Python inside Pyodide (WASM) hosted by a Deno subprocess. +It is intended to provide a sandboxed runtime without requiring Docker. +""" + +from py_code_mode.execution.deno_sandbox.config import DenoSandboxConfig +from py_code_mode.execution.deno_sandbox.executor import DenoSandboxExecutor + +__all__ = ["DenoSandboxConfig", "DenoSandboxExecutor"] diff --git a/src/py_code_mode/execution/deno_pyodide/config.py b/src/py_code_mode/execution/deno_sandbox/config.py similarity index 92% rename from src/py_code_mode/execution/deno_pyodide/config.py rename to src/py_code_mode/execution/deno_sandbox/config.py index d1d76d5..19dfff0 100644 --- a/src/py_code_mode/execution/deno_pyodide/config.py +++ b/src/py_code_mode/execution/deno_sandbox/config.py @@ -1,4 +1,4 @@ -"""Configuration for DenoPyodideExecutor.""" +"""Configuration for DenoSandboxExecutor.""" from __future__ import annotations @@ -8,8 +8,8 @@ @dataclass(frozen=True) -class DenoPyodideConfig: - """Configuration for DenoPyodideExecutor. +class DenoSandboxConfig: + """Configuration for DenoSandboxExecutor. Notes: - This executor expects Pyodide runtime assets (WASM + stdlib files) to be diff --git a/src/py_code_mode/execution/deno_pyodide/executor.py b/src/py_code_mode/execution/deno_sandbox/executor.py similarity index 97% rename from src/py_code_mode/execution/deno_pyodide/executor.py rename to src/py_code_mode/execution/deno_sandbox/executor.py index 04d6abf..818dbfe 100644 --- a/src/py_code_mode/execution/deno_pyodide/executor.py +++ b/src/py_code_mode/execution/deno_sandbox/executor.py @@ -25,7 +25,7 @@ collect_configured_deps, ) from py_code_mode.deps.store import DepsStore -from py_code_mode.execution.deno_pyodide.config import DenoPyodideConfig +from py_code_mode.execution.deno_sandbox.config import DenoSandboxConfig from py_code_mode.execution.protocol import Capability, validate_storage_not_access from py_code_mode.execution.registry import register_backend from py_code_mode.execution.resource_provider import StorageResourceProvider @@ -45,7 +45,7 @@ class _Pending: fut: asyncio.Future[dict[str, Any]] -class DenoPyodideExecutor: +class DenoSandboxExecutor: """Execute code in Pyodide hosted by Deno. Capabilities: @@ -70,8 +70,8 @@ class DenoPyodideExecutor: } ) - def __init__(self, config: DenoPyodideConfig | None = None) -> None: - self._config = config or DenoPyodideConfig() + def __init__(self, config: DenoSandboxConfig | None = None) -> None: + self._config = config or DenoSandboxConfig() self._proc: Process | None = None self._stdout_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None @@ -94,14 +94,14 @@ def supported_capabilities(self) -> set[str]: def get_configured_deps(self) -> list[str]: return collect_configured_deps(self._config.deps, self._config.deps_file) - async def __aenter__(self) -> DenoPyodideExecutor: + async def __aenter__(self) -> DenoSandboxExecutor: return self async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: await self.close() async def start(self, storage: StorageBackend | None = None) -> None: - validate_storage_not_access(storage, "DenoPyodideExecutor") + validate_storage_not_access(storage, "DenoSandboxExecutor") if self._proc is not None: return @@ -142,7 +142,7 @@ async def start(self, storage: StorageBackend | None = None) -> None: await self._spawn_runner() def _default_deno_dir(self) -> Path: - return Path.home() / ".cache" / "py-code-mode" / "deno-pyodide" + return Path.home() / ".cache" / "py-code-mode" / "deno-sandbox" async def _ensure_deno_cache(self) -> None: """Cache runner modules + npm:pyodide outside the sandbox.""" @@ -609,5 +609,4 @@ async def sync_deps(self) -> dict[str, Any]: return await self._deps_install(self._deps_store.list()) -register_backend("deno-pyodide", DenoPyodideExecutor) -register_backend("deno-sandbox", DenoPyodideExecutor) +register_backend("deno-sandbox", DenoSandboxExecutor) diff --git a/src/py_code_mode/execution/deno_pyodide/runner/main.ts b/src/py_code_mode/execution/deno_sandbox/runner/main.ts similarity index 100% rename from src/py_code_mode/execution/deno_pyodide/runner/main.ts rename to src/py_code_mode/execution/deno_sandbox/runner/main.ts diff --git a/src/py_code_mode/execution/deno_pyodide/runner/worker.ts b/src/py_code_mode/execution/deno_sandbox/runner/worker.ts similarity index 100% rename from src/py_code_mode/execution/deno_pyodide/runner/worker.ts rename to src/py_code_mode/execution/deno_sandbox/runner/worker.ts diff --git a/src/py_code_mode/execution/in_process/executor.py b/src/py_code_mode/execution/in_process/executor.py index 7ca3467..999501e 100644 --- a/src/py_code_mode/execution/in_process/executor.py +++ b/src/py_code_mode/execution/in_process/executor.py @@ -25,7 +25,6 @@ PackageInstaller, collect_configured_deps, ) -from py_code_mode.deps.async_namespace import AsyncDepsNamespace from py_code_mode.deps.store import DepsStore, MemoryDepsStore from py_code_mode.execution.in_process.config import InProcessConfig from py_code_mode.execution.in_process.workflows_namespace import WorkflowsNamespace @@ -108,11 +107,12 @@ def __init__( # Inject deps namespace if provided (wrap if runtime deps disabled) if deps_namespace is not None: - deps_obj: Any = AsyncDepsNamespace(deps_namespace) if not self._config.allow_runtime_deps: - self._namespace["deps"] = ControlledDepsNamespace(deps_obj, allow_runtime=False) + self._namespace["deps"] = ControlledDepsNamespace( + deps_namespace, allow_runtime=False + ) else: - self._namespace["deps"] = deps_obj + self._namespace["deps"] = deps_namespace def supports(self, capability: str) -> bool: """Check if this backend supports a capability.""" @@ -154,15 +154,13 @@ async def run(self, code: str, timeout: float | None = None) -> ExecutionResult: sync_mode = not self._requires_top_level_await(code) - # Store loop reference for tool/workflow calls from thread context. + # Store loop reference for tool calls from thread context. # Only used for the sync execution path; the async path executes in its # own event loop in the worker thread. if sync_mode: loop = asyncio.get_running_loop() if "tools" in self._namespace: self._namespace["tools"].set_loop(loop) - if "workflows" in self._namespace: - self._namespace["workflows"].set_loop(loop) # Run in thread to allow timeout cancellation try: @@ -185,11 +183,11 @@ def _requires_top_level_await(code: str) -> bool: try: compile(code, "", "exec") return False - except SyntaxError: + except (SyntaxError, RecursionError): try: compile(code, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) return True - except SyntaxError: + except (SyntaxError, RecursionError): return False def _run_sync(self, code: str) -> ExecutionResult: @@ -367,11 +365,12 @@ async def start( self._deps_namespace = DepsNamespace(store=deps_store, installer=installer) # Wrap deps namespace if runtime deps disabled - deps_obj: Any = AsyncDepsNamespace(self._deps_namespace) if not self._config.allow_runtime_deps: - self._namespace["deps"] = ControlledDepsNamespace(deps_obj, allow_runtime=False) + self._namespace["deps"] = ControlledDepsNamespace( + self._deps_namespace, allow_runtime=False + ) else: - self._namespace["deps"] = deps_obj + self._namespace["deps"] = self._deps_namespace # Workflows and artifacts from storage (if provided) if storage is not None: diff --git a/src/py_code_mode/execution/in_process/workflows_namespace.py b/src/py_code_mode/execution/in_process/workflows_namespace.py index 79e33b2..bef9125 100644 --- a/src/py_code_mode/execution/in_process/workflows_namespace.py +++ b/src/py_code_mode/execution/in_process/workflows_namespace.py @@ -6,7 +6,6 @@ from __future__ import annotations -import asyncio import builtins import inspect from typing import TYPE_CHECKING, Any @@ -46,15 +45,8 @@ def __init__(self, library: WorkflowLibrary, namespace: dict[str, Any]) -> None: self._library = library self._namespace = namespace - self._loop: asyncio.AbstractEventLoop | None = None - - def set_loop(self, loop: asyncio.AbstractEventLoop) -> None: - """Set the event loop to use for async workflow invocations. - - When code runs in a thread (via asyncio.to_thread), we need a reference - to the main event loop to execute async workflows via run_coroutine_threadsafe. - """ - self._loop = loop + # Intentionally synchronous surface. Async workflows are supported via + # asyncio.run(...) when invoked from sync code (the default execution mode). @property def library(self) -> WorkflowLibrary: @@ -64,60 +56,19 @@ def library(self) -> WorkflowLibrary: """ return self._library - def search(self, query: str, limit: int = 10) -> builtins.list[dict[str, Any]] | Any: - """Search for workflows matching query. - - In async context, returns an awaitable so code can use `await workflows.search(...)`. - """ - - def _run() -> builtins.list[dict[str, Any]]: - workflows = self._library.search(query, limit) - return [self._simplify(w) for w in workflows] - - try: - asyncio.get_running_loop() - except RuntimeError: - return _run() - - async def _coro() -> builtins.list[dict[str, Any]]: - return _run() - - return _coro() + def search(self, query: str, limit: int = 10) -> builtins.list[dict[str, Any]]: + """Search for workflows matching query.""" + workflows = self._library.search(query, limit) + return [self._simplify(w) for w in workflows] def get(self, name: str) -> Any: - """Get a workflow by name. - - In async context, returns an awaitable so code can use `await workflows.get(...)`. - """ - try: - asyncio.get_running_loop() - except RuntimeError: - return self._library.get(name) - - async def _coro() -> Any: - return self._library.get(name) - - return _coro() + """Get a workflow by name.""" + return self._library.get(name) - def list(self) -> builtins.list[dict[str, Any]] | Any: - """List all available workflows. - - In async context, returns an awaitable so code can use `await workflows.list()`. - """ - - def _run() -> builtins.list[dict[str, Any]]: - workflows = self._library.list() - return [self._simplify(w) for w in workflows] - - try: - asyncio.get_running_loop() - except RuntimeError: - return _run() - - async def _coro() -> builtins.list[dict[str, Any]]: - return _run() - - return _coro() + def list(self) -> builtins.list[dict[str, Any]]: + """List all available workflows.""" + workflows = self._library.list() + return [self._simplify(w) for w in workflows] def _simplify(self, workflow: PythonWorkflow) -> dict[str, Any]: """Simplify workflow for agent readability.""" @@ -162,20 +113,9 @@ def create( # Add to library (persists to store if configured) self._library.add(workflow) - result = self._simplify(workflow) + return self._simplify(workflow) - # Allow awaiting in async contexts for consistency. - try: - asyncio.get_running_loop() - except RuntimeError: - return result - - async def _coro() -> dict[str, Any]: - return result - - return _coro() - - def delete(self, name: str) -> bool | Any: + def delete(self, name: str) -> bool: """Remove a workflow from the library. Args: @@ -184,16 +124,7 @@ def delete(self, name: str) -> bool | Any: Returns: True if workflow was deleted, False if not found. """ - removed = self._library.remove(name) - try: - asyncio.get_running_loop() - except RuntimeError: - return removed - - async def _coro() -> bool: - return removed - - return _coro() + return self._library.remove(name) def __getattr__(self, name: str) -> Any: """Allow workflows.workflow_name(...) syntax.""" @@ -229,14 +160,11 @@ def invoke(self, workflow_name: str, **kwargs: Any) -> Any: result = run_func(**kwargs) if inspect.iscoroutine(result): - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(result) - - async def _coro() -> Any: - return await result + # Default agent code execution runs in a worker thread without a + # running event loop, so asyncio.run(...) is safe and yields a + # synchronous result. + import asyncio - return _coro() + return asyncio.run(result) return result diff --git a/src/py_code_mode/execution/subprocess/kernel_init.py b/src/py_code_mode/execution/subprocess/kernel_init.py index 6946ea0..327b3a8 100644 --- a/src/py_code_mode/execution/subprocess/kernel_init.py +++ b/src/py_code_mode/execution/subprocess/kernel_init.py @@ -57,11 +57,12 @@ def _namespace_error_handler(self, etype, value, tb, tb_offset=None): def _in_async_context() -> bool: - try: - asyncio.get_running_loop() - except RuntimeError: - return False - return True + # IPython / ipykernel runs an event loop even for "sync" code execution, + # so using get_running_loop() here would force the entire sandbox surface + # to return coroutine objects and break existing agent code. + # + # SubprocessExecutor's agent-facing namespaces are intentionally synchronous. + return False async def _rpc_call_async(method: str, **kwargs) -> Any: @@ -446,7 +447,8 @@ def invoke(self, workflow_name: str, **kwargs) -> Any: Gets workflow source from host and executes it locally in the kernel. This ensures workflows can import packages installed at runtime. - Handles async workflows by running them with asyncio.run(). + Handles async workflows by running them to completion in the kernel's + event loop (ipykernel runs a loop even for "sync" execution). Args: workflow_name: Name of the workflow to invoke. @@ -508,7 +510,13 @@ async def _coro() -> Any: result = run_func(**kwargs) if asyncio.iscoroutine(result): - return asyncio.run(result) + # ipykernel runs an event loop already, so asyncio.run() is not valid. + # Use nest_asyncio to allow re-entrant run_until_complete. + import nest_asyncio + + nest_asyncio.apply() + loop = asyncio.get_event_loop() + return loop.run_until_complete(result) return result def search(self, query: str, limit: int = 5) -> list[Workflow]: diff --git a/src/py_code_mode/tools/namespace.py b/src/py_code_mode/tools/namespace.py index 696be4c..7227e5b 100644 --- a/src/py_code_mode/tools/namespace.py +++ b/src/py_code_mode/tools/namespace.py @@ -4,6 +4,7 @@ import asyncio import builtins +import threading from typing import TYPE_CHECKING, Any from py_code_mode.tools.types import Tool, ToolCallable @@ -35,7 +36,11 @@ def __init__(self, registry: ToolRegistry) -> None: self._loop: asyncio.AbstractEventLoop | None = None def set_loop(self, loop: asyncio.AbstractEventLoop) -> None: - """Set the event loop to use for async tool calls.""" + """Set the event loop used by sync calls made from worker threads. + + InProcessExecutor executes user code in a worker thread. Tool adapters are + async, so sync tool calls use run_coroutine_threadsafe against this loop. + """ self._loop = loop def __getattr__(self, tool_name: str) -> ToolProxy: @@ -57,46 +62,21 @@ def __getattr__(self, tool_name: str) -> ToolProxy: return ToolProxy(self._registry, tool, self._loop) - def list(self) -> builtins.list[Tool] | Any: - """List all available tools. - - In async context, returns an awaitable so code can use `await tools.list()`. - """ - try: - asyncio.get_running_loop() - except RuntimeError: - return self._registry.get_all_tools() - - async def _coro() -> builtins.list[Tool]: - return self._registry.get_all_tools() - - return _coro() - - def search(self, query: str, limit: int = 5) -> builtins.list[Tool] | Any: - """Search tools by query string. + def list(self) -> builtins.list[Tool]: + """List all available tools.""" + return self._registry.get_all_tools() - In async context, returns an awaitable so code can use `await tools.search(...)`. - """ + def search(self, query: str, limit: int = 5) -> builtins.list[Tool]: + """Search tools by query string.""" from py_code_mode.tools.registry import substring_search - def _run() -> builtins.list[Tool]: - return substring_search( - query=query, - items=self._registry.get_all_tools(), - get_name=lambda t: t.name, - get_description=lambda t: t.description, - limit=limit, - ) - - try: - asyncio.get_running_loop() - except RuntimeError: - return _run() - - async def _coro() -> builtins.list[Tool]: - return _run() - - return _coro() + return substring_search( + query=query, + items=self._registry.get_all_tools(), + get_name=lambda t: t.name, + get_description=lambda t: t.description, + limit=limit, + ) class ToolProxy: @@ -138,31 +118,42 @@ def call_sync(self, **kwargs: Any) -> Any: """ coro = self._execute(**kwargs) - # When called from a thread with loop reference, use run_coroutine_threadsafe + # When called from a worker thread, schedule onto the main loop. if self._loop is not None: future = asyncio.run_coroutine_threadsafe(coro, self._loop) return future.result() - # Standalone sync usage - create new loop - return asyncio.run(coro) + # If we're already inside an event loop on this thread, we cannot call + # asyncio.run(). For sync ergonomics (tools.x.y()), execute the coroutine + # in a dedicated thread with its own loop and block for completion. + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + out: dict[str, Any] = {} + err: list[BaseException] = [] + + def _runner() -> None: + try: + out["value"] = asyncio.run(coro) + except BaseException as e: # propagate exception across threads + err.append(e) + + t = threading.Thread(target=_runner, daemon=True) + t.start() + t.join() + if err: + raise err[0] + return out.get("value") def __call__(self, **kwargs: Any) -> Any: """Escape hatch - invoke tool directly without recipe. - This delegates to the adapter's call_tool method, bypassing recipes. - Returns coroutine in async context, executes sync otherwise. - - If set_loop() has been called, sync execution in a worker thread uses - run_coroutine_threadsafe to schedule work on the main loop. + Tool calls are synchronous in the agent-facing namespace. The underlying + adapter is async, so this blocks until completion. """ - try: - asyncio.get_running_loop() - except RuntimeError: - # No event loop running - use sync path - return self.call_sync(**kwargs) - else: - # In async context - return coroutine for await - return self._execute(**kwargs) + return self.call_sync(**kwargs) def __getattr__(self, callable_name: str) -> CallableProxy: """Get a callable proxy by name.""" @@ -216,31 +207,39 @@ def call_sync(self, **kwargs: Any) -> Any: """ coro = self._execute(**kwargs) - # When called from a thread with loop reference, use run_coroutine_threadsafe + # When called from a worker thread, schedule onto the main loop. if self._loop is not None: future = asyncio.run_coroutine_threadsafe(coro, self._loop) return future.result() - # Standalone sync usage - create new loop - return asyncio.run(coro) + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + out: dict[str, Any] = {} + err: list[BaseException] = [] + + def _runner() -> None: + try: + out["value"] = asyncio.run(coro) + except BaseException as e: + err.append(e) + + t = threading.Thread(target=_runner, daemon=True) + t.start() + t.join() + if err: + raise err[0] + return out.get("value") def __call__(self, **kwargs: Any) -> Any: """Invoke the callable with the given arguments. - Returns a coroutine if called from async context, executes sync otherwise. - This allows both `await tools.x.y()` and `tools.x.y()` to work. - - If set_loop() has been called, sync execution in a worker thread uses - run_coroutine_threadsafe to schedule work on the main loop. + Callables are synchronous in the agent-facing namespace. The underlying + adapter is async, so this blocks until completion. """ - try: - asyncio.get_running_loop() - except RuntimeError: - # No event loop running - use sync path - return self.call_sync(**kwargs) - else: - # In async context - return coroutine for await - return self._execute(**kwargs) + return self.call_sync(**kwargs) async def describe(self) -> dict[str, str]: """Get parameter descriptions for this callable.""" diff --git a/tests/test_async_sandbox_interfaces.py b/tests/test_async_sandbox_interfaces.py index d20d799..abf89e1 100644 --- a/tests/test_async_sandbox_interfaces.py +++ b/tests/test_async_sandbox_interfaces.py @@ -5,7 +5,9 @@ @pytest.mark.asyncio -async def test_inprocess_executor_supports_await_tools_and_artifacts(tmp_path: Path) -> None: +async def test_inprocess_executor_tools_and_artifacts_are_sync_in_agent_code( + tmp_path: Path, +) -> None: from py_code_mode.execution import InProcessConfig, InProcessExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -40,19 +42,19 @@ async def test_inprocess_executor_supports_await_tools_and_artifacts(tmp_path: P executor = InProcessExecutor(config=InProcessConfig(tools_path=tools_dir)) async with Session(storage=storage, executor=executor) as session: - r_list = await session.run("_ts = await tools.list()\nsorted([t.name for t in _ts])") + r_list = await session.run("_ts = tools.list()\nsorted([t.name for t in _ts])") assert r_list.error is None assert "echo" in r_list.value - r_call = await session.run("(await tools.echo.say(message='hi')).strip()") + r_call = await session.run("tools.echo.say(message='hi').strip()") assert r_call.error is None assert r_call.value == "hi" r_art = await session.run( "\n".join( [ - "await artifacts.save('obj', {'a': 1}, description='')", - "(await artifacts.load('obj'))['a']", + "artifacts.save('obj', {'a': 1}, description='')", + "artifacts.load('obj')['a']", ] ) ) @@ -68,7 +70,9 @@ async def test_inprocess_executor_supports_await_tools_and_artifacts(tmp_path: P @pytestmark_subprocess @pytest.mark.asyncio -async def test_subprocess_executor_supports_await_tools_workflows_artifacts(tmp_path: Path) -> None: +async def test_subprocess_executor_tools_workflows_artifacts_are_sync_in_agent_code( + tmp_path: Path, +) -> None: from py_code_mode.execution import SubprocessConfig, SubprocessExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -106,12 +110,12 @@ async def test_subprocess_executor_supports_await_tools_workflows_artifacts(tmp_ async with Session(storage=storage, executor=executor) as session: r_list = await session.run( - "_ts = await tools.list()\n" "sorted([t['name'] for t in _ts])", + "_ts = tools.list()\nsorted([t['name'] for t in _ts])", ) assert r_list.error is None assert "echo" in r_list.value - r_call = await session.run("(await tools.echo.say(message='hi')).strip()") + r_call = await session.run("tools.echo.say(message='hi').strip()") assert r_call.error is None assert r_call.value == "hi" @@ -119,8 +123,8 @@ async def test_subprocess_executor_supports_await_tools_workflows_artifacts(tmp_ r_wf = await session.run( "\n".join( [ - f"await workflows.create('hello', {source!r}, 'desc')", - "(await workflows.get('hello'))['source']", + f"workflows.create('hello', {source!r}, 'desc')", + "workflows.get('hello')['source']", ] ) ) @@ -130,8 +134,8 @@ async def test_subprocess_executor_supports_await_tools_workflows_artifacts(tmp_ r_art = await session.run( "\n".join( [ - "await artifacts.save('obj', {'a': 1}, description='')", - "(await artifacts.load('obj'))['a']", + "artifacts.save('obj', {'a': 1}, description='')", + "artifacts.load('obj')['a']", ] ) ) diff --git a/tests/test_deno_sandbox_alias.py b/tests/test_deno_sandbox_alias.py deleted file mode 100644 index 4daee10..0000000 --- a/tests/test_deno_sandbox_alias.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest - - -def test_deno_sandbox_aliases_exist() -> None: - from py_code_mode.execution import DENO_PYODIDE_AVAILABLE - - if not DENO_PYODIDE_AVAILABLE: - pytest.skip("Deno sandbox backend is optional and not available in this environment.") - - from py_code_mode.execution import ( - DenoPyodideConfig, - DenoPyodideExecutor, - DenoSandboxConfig, - DenoSandboxExecutor, - ) - - assert DenoSandboxConfig is DenoPyodideConfig - assert DenoSandboxExecutor is DenoPyodideExecutor diff --git a/tests/test_deno_pyodide_executor.py b/tests/test_deno_sandbox_executor.py similarity index 95% rename from tests/test_deno_pyodide_executor.py rename to tests/test_deno_sandbox_executor.py index 178ed56..4fb87e0 100644 --- a/tests/test_deno_pyodide_executor.py +++ b/tests/test_deno_sandbox_executor.py @@ -79,7 +79,7 @@ def get_artifact_store(self): @pytest.mark.asyncio -async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_basic(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -111,7 +111,7 @@ async def test_deno_pyodide_executor_basic(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_deps_add_installs(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -140,7 +140,7 @@ async def test_deno_pyodide_executor_deps_add_installs(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_sync_deps_on_start(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -168,7 +168,7 @@ async def test_deno_pyodide_executor_sync_deps_on_start(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_network_none_blocks_installs(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -195,7 +195,7 @@ async def test_deno_pyodide_executor_network_none_blocks_installs(tmp_path: Path @pytest.mark.asyncio -async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_artifacts_roundtrip(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -223,7 +223,7 @@ async def test_deno_pyodide_executor_artifacts_roundtrip(tmp_path: Path) -> None @pytest.mark.asyncio -async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_workflows_roundtrip(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -259,7 +259,7 @@ async def test_deno_pyodide_executor_workflows_roundtrip(tmp_path: Path) -> None @pytest.mark.asyncio -async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_tools_via_rpc(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -315,7 +315,7 @@ async def test_deno_pyodide_executor_tools_via_rpc(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_tool_large_output_is_chunked(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -374,7 +374,7 @@ async def test_deno_pyodide_executor_tool_large_output_is_chunked(tmp_path: Path @pytest.mark.asyncio -async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_rpc_does_not_deadlock(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -410,7 +410,7 @@ async def test_deno_pyodide_executor_rpc_does_not_deadlock(tmp_path: Path) -> No @pytest.mark.asyncio -async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_reset_clears_state(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -440,7 +440,7 @@ async def test_deno_pyodide_executor_reset_clears_state(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_session_add_dep_installs(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -469,7 +469,7 @@ async def test_deno_pyodide_executor_session_add_dep_installs(tmp_path: Path) -> @pytest.mark.asyncio -async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -538,7 +538,7 @@ async def test_deno_pyodide_executor_mcp_tool_via_rpc(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_workflows_search_via_rpc(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session @@ -567,7 +567,7 @@ async def test_deno_pyodide_executor_workflows_search_via_rpc(tmp_path: Path) -> @pytest.mark.asyncio -async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_artifact_payload_size_limits(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage @@ -611,7 +611,7 @@ async def test_deno_pyodide_executor_artifact_payload_size_limits(tmp_path: Path @pytest.mark.asyncio -async def test_deno_pyodide_executor_soft_timeout_wedges_until_reset(tmp_path: Path) -> None: +async def test_deno_sandbox_executor_soft_timeout_wedges_until_reset(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor from py_code_mode.session import Session from py_code_mode.storage import FileStorage diff --git a/tests/test_deno_sandbox_imports.py b/tests/test_deno_sandbox_imports.py new file mode 100644 index 0000000..b16fe5d --- /dev/null +++ b/tests/test_deno_sandbox_imports.py @@ -0,0 +1,13 @@ +import pytest + + +def test_deno_sandbox_imports() -> None: + from py_code_mode.execution import DENO_SANDBOX_AVAILABLE + + if not DENO_SANDBOX_AVAILABLE: + pytest.skip("Deno sandbox backend is optional and not available in this environment.") + + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor + + assert DenoSandboxConfig is not None + assert DenoSandboxExecutor is not None diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 78f7b96..23186d0 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -93,7 +93,7 @@ async def communicate(self): monkeypatch.setattr(asyncio, "create_subprocess_exec", mock_create_subprocess_exec) - result = await namespace.nmap.syn_scan(target="10.0.0.1") + result = namespace.nmap.syn_scan(target="10.0.0.1") assert result == "Mock nmap output" diff --git a/tests/test_tool_proxy_explicit_methods.py b/tests/test_tool_proxy_explicit_methods.py index 7669cb0..b14c336 100644 --- a/tests/test_tool_proxy_explicit_methods.py +++ b/tests/test_tool_proxy_explicit_methods.py @@ -196,14 +196,11 @@ class TestBackwardCompatibility: @pytest.mark.asyncio async def test_dunder_call_still_works_async(self, callable_proxy: CallableProxy) -> None: - """__call__ from async context still returns coroutine.""" + """__call__ from async context still returns a sync result.""" result = callable_proxy(value="dunder_async") - # In async context, __call__ returns coroutine - assert inspect.iscoroutine(result) - - final = await result - assert "testtool.action" in final + assert not inspect.iscoroutine(result) + assert "testtool.action" in result def test_dunder_call_still_works_sync(self, callable_proxy: CallableProxy) -> None: """__call__ from sync context still returns result.""" From 492c8773349e90be59a66af4fb5a0187b90ea670 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 14:15:07 -0800 Subject: [PATCH 14/27] Docs: clarify DenoSandbox tool execution boundary --- docs/ARCHITECTURE.md | 3 +++ docs/executors.md | 10 ++++++++++ docs/tools.md | 1 + 3 files changed, 14 insertions(+) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index af5b800..4d7d282 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -808,6 +808,9 @@ Agent writes: "tools.curl.get(url='...')" (use `await` only in DenoSandboxExecut +--------------+ ``` +Note on sandboxing: +- `DenoSandboxExecutor` sandboxes Python execution in Pyodide, but **tools execute host-side** (the sandbox calls back to the host over RPC to run tools). If you need strict sandbox boundaries, avoid `tools.*` and stick to pure Python plus `deps.*` in the sandbox. + ### ToolProxy Methods ``` diff --git a/docs/executors.md b/docs/executors.md index 9ef852b..7389f40 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -33,6 +33,16 @@ async with Session(storage=storage, executor=executor) as session: result = await session.run("await tools.list()") ``` +### Security Model: Where Tools Execute + +`DenoSandboxExecutor` sandboxes **Python execution** (the Pyodide runtime) inside a Deno subprocess. However, **tool execution is host-side**: +- If your agent calls `tools.*` while using `DenoSandboxExecutor`, the call is proxied over RPC back to the host Python process, and the tool runs there (using the configured ToolAdapters). +- This means a YAML tool that can read files, run commands, or access the network will do so with **host permissions**, not Deno sandbox permissions. + +Practical guidance: +- If you want "true sandboxed code exec", keep agent code to **pure Python + `deps.*`** (Pyodide `micropip`) and avoid `tools.*`. +- If you attach host tools, treat them as a privileged escape hatch from the sandbox boundary. + ### Network Profiles `DenoSandboxConfig.network_profile` controls network access for the Deno subprocess: diff --git a/docs/tools.md b/docs/tools.md index b31ee9e..36e85e6 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -94,6 +94,7 @@ Tool calls inside `Session.run()` are **synchronous** in the default executors ( Notes: - If you need async tool calls in Python code, use `call_async(...)` explicitly. - In `DenoSandboxExecutor`, tool calls are **async-first** and you must use `await tools.*`. +- In `DenoSandboxExecutor`, tool calls execute **outside** the sandbox: `await tools.*` is an RPC back to the host Python process, and the tool runs with host permissions (or container permissions if the tool adapter/executor is containerized). ```python # Recipe invocation (recommended) From 163d055634bc3856b62cb22e888d18d9428bdbe6 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 16:02:03 -0800 Subject: [PATCH 15/27] Tools: add middleware plumbing (DenoSandbox wired) --- .../execution/deno_sandbox/config.py | 10 +- .../execution/deno_sandbox/executor.py | 6 + src/py_code_mode/tools/__init__.py | 3 + src/py_code_mode/tools/adapters/__init__.py | 2 + src/py_code_mode/tools/adapters/middleware.py | 63 ++++++++++ src/py_code_mode/tools/middleware.py | 72 +++++++++++ src/py_code_mode/tools/registry.py | 52 +++++++- tests/test_deno_sandbox_executor.py | 68 ++++++++++ tests/test_tool_middleware_plumbing.py | 117 ++++++++++++++++++ 9 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 src/py_code_mode/tools/adapters/middleware.py create mode 100644 src/py_code_mode/tools/middleware.py create mode 100644 tests/test_tool_middleware_plumbing.py diff --git a/src/py_code_mode/execution/deno_sandbox/config.py b/src/py_code_mode/execution/deno_sandbox/config.py index 19dfff0..c16eefd 100644 --- a/src/py_code_mode/execution/deno_sandbox/config.py +++ b/src/py_code_mode/execution/deno_sandbox/config.py @@ -4,7 +4,10 @@ from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + from py_code_mode.tools.middleware import ToolMiddleware @dataclass(frozen=True) @@ -46,3 +49,8 @@ class DenoSandboxConfig: "files.pythonhosted.org", "cdn.jsdelivr.net", ) + + # Optional host-side middleware invoked around tool calls. + # This is enforced for DenoSandbox tool calls because all `tools.*` calls + # are proxied back to the host Python process. + tool_middlewares: tuple[ToolMiddleware, ...] = () diff --git a/src/py_code_mode/execution/deno_sandbox/executor.py b/src/py_code_mode/execution/deno_sandbox/executor.py index 818dbfe..1de2cab 100644 --- a/src/py_code_mode/execution/deno_sandbox/executor.py +++ b/src/py_code_mode/execution/deno_sandbox/executor.py @@ -117,6 +117,12 @@ async def start(self, storage: StorageBackend | None = None) -> None: self._tool_registry = await load_tools_from_path(self._config.tools_path) else: self._tool_registry = ToolRegistry() + if self._config.tool_middlewares: + self._tool_registry.apply_tool_middlewares( + self._config.tool_middlewares, + executor_type="deno-sandbox", + origin="deno-sandbox", + ) # Deps store from executor config (persistence only; installs happen in sandbox) initial_deps = collect_configured_deps(self._config.deps, self._config.deps_file) diff --git a/src/py_code_mode/tools/__init__.py b/src/py_code_mode/tools/__init__.py index 07c96fa..b5f4b83 100644 --- a/src/py_code_mode/tools/__init__.py +++ b/src/py_code_mode/tools/__init__.py @@ -1,6 +1,7 @@ """py_code_mode.tools - Tool registry and namespace.""" from py_code_mode.tools.loader import load_tools_from_path +from py_code_mode.tools.middleware import ToolCallContext, ToolMiddleware from py_code_mode.tools.namespace import ( CallableProxy, ToolProxy, @@ -26,4 +27,6 @@ "ToolProxy", "ToolsNamespace", "load_tools_from_path", + "ToolCallContext", + "ToolMiddleware", ] diff --git a/src/py_code_mode/tools/adapters/__init__.py b/src/py_code_mode/tools/adapters/__init__.py index 93b836c..e44dbf5 100644 --- a/src/py_code_mode/tools/adapters/__init__.py +++ b/src/py_code_mode/tools/adapters/__init__.py @@ -4,6 +4,7 @@ from py_code_mode.tools.adapters.cli import CLIAdapter from py_code_mode.tools.adapters.http import Endpoint, HTTPAdapter from py_code_mode.tools.adapters.mcp import MCPAdapter +from py_code_mode.tools.adapters.middleware import MiddlewareAdapter __all__ = [ "ToolAdapter", @@ -11,4 +12,5 @@ "MCPAdapter", "HTTPAdapter", "Endpoint", + "MiddlewareAdapter", ] diff --git a/src/py_code_mode/tools/adapters/middleware.py b/src/py_code_mode/tools/adapters/middleware.py new file mode 100644 index 0000000..04ab506 --- /dev/null +++ b/src/py_code_mode/tools/adapters/middleware.py @@ -0,0 +1,63 @@ +"""ToolAdapter wrapper that applies tool call middleware.""" + +from __future__ import annotations + +from typing import Any + +from py_code_mode.tools.adapters.base import ToolAdapter +from py_code_mode.tools.middleware import ToolCallContext, ToolMiddleware, compose_tool_middlewares +from py_code_mode.tools.types import Tool + + +class MiddlewareAdapter: + """Wrap a ToolAdapter with a middleware chain for call_tool().""" + + def __init__( + self, + inner: ToolAdapter, + middlewares: tuple[ToolMiddleware, ...], + *, + executor_type: str | None = None, + origin: str | None = None, + ) -> None: + self._inner = inner + self._middlewares = middlewares + self._executor_type = executor_type + self._origin = origin + + @property + def inner(self) -> ToolAdapter: + return self._inner + + def list_tools(self) -> list[Tool]: + return self._inner.list_tools() + + async def describe(self, tool_name: str, callable_name: str) -> dict[str, str]: + return await self._inner.describe(tool_name, callable_name) + + async def call_tool( + self, + name: str, + callable_name: str | None, + args: dict[str, Any], + ) -> Any: + if not self._middlewares: + return await self._inner.call_tool(name, callable_name, args) + + ctx = ToolCallContext( + tool_name=name, + callable_name=callable_name, + args=args, + adapter_name=type(self._inner).__name__, + executor_type=self._executor_type, + origin=self._origin, + ) + + async def _terminal(c: ToolCallContext) -> Any: + return await self._inner.call_tool(c.tool_name, c.callable_name, c.args) + + chain = compose_tool_middlewares(self._middlewares, _terminal) + return await chain(ctx) + + async def close(self) -> None: + await self._inner.close() diff --git a/src/py_code_mode/tools/middleware.py b/src/py_code_mode/tools/middleware.py new file mode 100644 index 0000000..fe09d39 --- /dev/null +++ b/src/py_code_mode/tools/middleware.py @@ -0,0 +1,72 @@ +"""Tool call middleware and context types. + +This layer is designed to be generic: it can be used for audit logging, +allow/deny decisions, interactive approvals, argument rewriting, retries, etc. + +Middleware runs on the host side where tools are executed (CLI/MCP/HTTP). +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass +class ToolCallContext: + """Context for a single tool call. + + Notes: + - `args` is intentionally mutable to support argument rewriting. + - `executor_type` and `origin` are best-effort labels (not security boundaries). + """ + + tool_name: str + callable_name: str | None + args: dict[str, Any] + + # Optional metadata for middleware consumers. + adapter_name: str | None = None + executor_type: str | None = None # e.g. "deno-sandbox", "subprocess" + origin: str | None = None # e.g. "deno-sandbox", "host" + request_id: str | None = None + timeout: float | None = None + + @property + def full_name(self) -> str: + if self.callable_name: + return f"{self.tool_name}.{self.callable_name}" + return self.tool_name + + +type CallNext = Callable[[ToolCallContext], Awaitable[Any]] + + +class ToolMiddleware(Protocol): + async def __call__(self, ctx: ToolCallContext, call_next: CallNext) -> Any: ... + + +def compose_tool_middlewares( + middlewares: tuple[ToolMiddleware, ...], + terminal: CallNext, +) -> CallNext: + """Compose middlewares around a terminal call. + + Middleware order is outer-to-inner: + middlewares[0] wraps middlewares[1] wraps ... wraps terminal + """ + + call_next = terminal + for mw in reversed(middlewares): + + async def _wrapped( + ctx: ToolCallContext, + _mw: ToolMiddleware = mw, + _n: CallNext = call_next, + ): + return await _mw(ctx, _n) + + call_next = _wrapped + + return call_next diff --git a/src/py_code_mode/tools/registry.py b/src/py_code_mode/tools/registry.py index b726fe6..29052e4 100644 --- a/src/py_code_mode/tools/registry.py +++ b/src/py_code_mode/tools/registry.py @@ -3,11 +3,12 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import TYPE_CHECKING, Any, TypeVar from py_code_mode.errors import CodeModeError, ToolCallError, ToolNotFoundError from py_code_mode.tools.adapters.base import ToolAdapter +from py_code_mode.tools.middleware import ToolMiddleware from py_code_mode.tools.types import Tool from py_code_mode.workflows import EmbeddingProvider, cosine_similarity @@ -501,6 +502,55 @@ async def close(self) -> None: self._tool_to_adapter.clear() self._vectors.clear() + # ------------------------------------------------------------------------- + # Middleware + # ------------------------------------------------------------------------- + + def apply_tool_middlewares( + self, + middlewares: Sequence[ToolMiddleware], + *, + executor_type: str | None = None, + origin: str | None = None, + ) -> None: + """Wrap all adapters with a tool call middleware chain. + + This method mutates the registry in-place and updates internal mappings + so that *all* tool call paths (ToolsNamespace, RPC providers, registry.call_tool) + are routed through the middleware chain. + + Notes: + - Only `call_tool()` is wrapped. `list_tools()` and `describe()` are passed through. + - Applying multiple times will stack wrappers; call once per registry instance. + """ + mws = tuple(middlewares) + if not mws: + return + + from py_code_mode.tools.adapters.middleware import MiddlewareAdapter + + old_adapters = list(self._adapters) + adapter_map: dict[int, ToolAdapter] = {} + new_adapters: list[ToolAdapter] = [] + + for adapter in old_adapters: + wrapped: ToolAdapter = MiddlewareAdapter( + adapter, + mws, + executor_type=executor_type, + origin=origin, + ) + new_adapters.append(wrapped) + adapter_map[id(adapter)] = wrapped + + self._adapters = new_adapters + + # Update tool->adapter mapping used by registry.call_tool(). + for tool_name, adapter in list(self._tool_to_adapter.items()): + mapped = adapter_map.get(id(adapter)) + if mapped is not None: + self._tool_to_adapter[tool_name] = mapped + class ScopedToolRegistry: """A scoped view of a ToolRegistry that only exposes matching tools. diff --git a/tests/test_deno_sandbox_executor.py b/tests/test_deno_sandbox_executor.py index 4fb87e0..6bd29da 100644 --- a/tests/test_deno_sandbox_executor.py +++ b/tests/test_deno_sandbox_executor.py @@ -314,6 +314,74 @@ async def test_deno_sandbox_executor_tools_via_rpc(tmp_path: Path) -> None: assert r.value == "hi" +@pytest.mark.asyncio +async def test_deno_sandbox_executor_tool_middleware_is_invoked(tmp_path: Path) -> None: + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + from py_code_mode.tools import ToolCallContext, ToolMiddleware + + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + (tools_dir / "echo.yaml").write_text( + "\n".join( + [ + "name: echo", + "description: Echo text", + "command: /bin/echo", + "timeout: 5", + "schema:", + " positional:", + " - name: message", + " type: string", + " required: true", + " description: message", + "recipes:", + " say:", + " description: Echo message", + " params:", + " message: {}", + "", + ] + ), + encoding="utf-8", + ) + + events: list[str] = [] + + class _Recorder(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + events.append(f"pre:{ctx.full_name}:{ctx.executor_type}:{ctx.origin}") + out = await call_next(ctx) + events.append(f"post:{ctx.full_name}") + return out + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoSandboxExecutor( + DenoSandboxConfig( + deno_dir=deno_dir, + tools_path=tools_dir, + tool_middlewares=(_Recorder(),), + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + async with Session(storage=storage, executor=executor) as session: + r = await session.run("(await tools.echo.say(message='hi')).strip()") + assert r.error is None + assert r.value == "hi" + + assert events == [ + "pre:echo.say:deno-sandbox:deno-sandbox", + "post:echo.say", + ] + + @pytest.mark.asyncio async def test_deno_sandbox_executor_tool_large_output_is_chunked(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor diff --git a/tests/test_tool_middleware_plumbing.py b/tests/test_tool_middleware_plumbing.py new file mode 100644 index 0000000..7aea567 --- /dev/null +++ b/tests/test_tool_middleware_plumbing.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from py_code_mode.tools import ToolCallContext, ToolMiddleware, ToolRegistry +from py_code_mode.tools.types import Tool, ToolCallable, ToolParameter + + +class _FakeAdapter: + def __init__(self) -> None: + self.calls: list[tuple[str, str | None, dict[str, Any]]] = [] + + def list_tools(self) -> list[Tool]: + return [ + Tool( + name="echo", + description="Echo", + callables=( + ToolCallable( + name="say", + description="Say", + parameters=( + ToolParameter( + name="message", + type="str", + required=True, + default=None, + description="msg", + ), + ), + ), + ), + ) + ] + + async def call_tool(self, name: str, callable_name: str | None, args: dict[str, Any]) -> Any: + self.calls.append((name, callable_name, dict(args))) + return {"ok": True, "name": name, "callable": callable_name, "args": dict(args)} + + async def describe(self, tool_name: str, callable_name: str) -> dict[str, str]: + return {"message": "msg"} + + async def close(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_tool_middleware_wraps_registry_call_tool() -> None: + adapter = _FakeAdapter() + registry = ToolRegistry() + registry.register_adapter(adapter) # registers echo + + events: list[str] = [] + + class _Recorder(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + events.append(f"pre:{ctx.full_name}:{ctx.executor_type}:{ctx.origin}") + out = await call_next(ctx) + events.append(f"post:{ctx.full_name}") + return out + + registry.apply_tool_middlewares( + (_Recorder(),), + executor_type="deno-sandbox", + origin="deno-sandbox", + ) + + res = await registry.call_tool("echo", "say", {"message": "hi"}) + assert res["ok"] is True + assert adapter.calls == [("echo", "say", {"message": "hi"})] + assert events == [ + "pre:echo.say:deno-sandbox:deno-sandbox", + "post:echo.say", + ] + + +@pytest.mark.asyncio +async def test_tool_middleware_can_rewrite_args() -> None: + adapter = _FakeAdapter() + registry = ToolRegistry() + registry.register_adapter(adapter) + + class _Rewrite(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + ctx.args["message"] = "rewritten" + return await call_next(ctx) + + registry.apply_tool_middlewares( + (_Rewrite(),), + executor_type="deno-sandbox", + origin="deno-sandbox", + ) + + res = await registry.call_tool("echo", "say", {"message": "hi"}) + assert res["args"]["message"] == "rewritten" + assert adapter.calls == [("echo", "say", {"message": "rewritten"})] + + +@pytest.mark.asyncio +async def test_tool_middleware_can_short_circuit() -> None: + from py_code_mode.errors import ToolCallError + + adapter = _FakeAdapter() + registry = ToolRegistry() + registry.register_adapter(adapter) + + class _Deny(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + raise RuntimeError(f"denied: {ctx.full_name}") + + registry.apply_tool_middlewares((_Deny(),), executor_type="deno-sandbox", origin="deno-sandbox") + + with pytest.raises(ToolCallError, match="denied: echo.say"): + await registry.call_tool("echo", "say", {"message": "hi"}) + assert adapter.calls == [] From 4561bebb0237df34e4f42be729548b90061fda2d Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 17:12:09 -0800 Subject: [PATCH 16/27] Tools middleware: enforce on new adapters + unify dispatch --- .../execution/deno_sandbox/config.py | 5 +- .../execution/resource_provider.py | 6 +- src/py_code_mode/tools/adapters/middleware.py | 2 + src/py_code_mode/tools/namespace.py | 10 +-- src/py_code_mode/tools/registry.py | 75 ++++++++++++------- tests/test_tool_middleware_plumbing.py | 60 ++++++++++++++- 6 files changed, 113 insertions(+), 45 deletions(-) diff --git a/src/py_code_mode/execution/deno_sandbox/config.py b/src/py_code_mode/execution/deno_sandbox/config.py index c16eefd..c0bc5a3 100644 --- a/src/py_code_mode/execution/deno_sandbox/config.py +++ b/src/py_code_mode/execution/deno_sandbox/config.py @@ -4,10 +4,9 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import Literal -if TYPE_CHECKING: - from py_code_mode.tools.middleware import ToolMiddleware +from py_code_mode.tools.middleware import ToolMiddleware @dataclass(frozen=True) diff --git a/src/py_code_mode/execution/resource_provider.py b/src/py_code_mode/execution/resource_provider.py index 68f2eaa..baa7867 100644 --- a/src/py_code_mode/execution/resource_provider.py +++ b/src/py_code_mode/execution/resource_provider.py @@ -77,11 +77,7 @@ async def call_tool(self, name: str, args: dict[str, Any]) -> Any: tool_name = name recipe_name = None - adapter = registry.find_adapter_for_tool(tool_name) - if adapter is None: - raise ValueError(f"Unknown tool: {tool_name}") - - return await adapter.call_tool(tool_name, recipe_name, args) + return await registry.call_tool(tool_name, recipe_name, args) async def list_tools(self) -> list[dict[str, Any]]: registry = self._get_tool_registry() diff --git a/src/py_code_mode/tools/adapters/middleware.py b/src/py_code_mode/tools/adapters/middleware.py index 04ab506..9cc1198 100644 --- a/src/py_code_mode/tools/adapters/middleware.py +++ b/src/py_code_mode/tools/adapters/middleware.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid from typing import Any from py_code_mode.tools.adapters.base import ToolAdapter @@ -51,6 +52,7 @@ async def call_tool( adapter_name=type(self._inner).__name__, executor_type=self._executor_type, origin=self._origin, + request_id=uuid.uuid4().hex, ) async def _terminal(c: ToolCallContext) -> Any: diff --git a/src/py_code_mode/tools/namespace.py b/src/py_code_mode/tools/namespace.py index 7227e5b..a43b14a 100644 --- a/src/py_code_mode/tools/namespace.py +++ b/src/py_code_mode/tools/namespace.py @@ -99,10 +99,7 @@ def __init__( async def _execute(self, **kwargs: Any) -> Any: """Execute the tool asynchronously.""" tool_name = self._tool.name - adapter = self._registry.find_adapter_for_tool(tool_name) - if adapter is None: - raise RuntimeError(f"No adapter found for tool: {tool_name}") - return await adapter.call_tool(tool_name, None, kwargs) + return await self._registry.call_tool(tool_name, None, kwargs) async def call_async(self, **kwargs: Any) -> Any: """Execute tool asynchronously. Always returns awaitable. @@ -188,10 +185,7 @@ def __init__( async def _execute(self, **kwargs: Any) -> Any: """Execute the callable asynchronously.""" - adapter = self._registry.find_adapter_for_tool(self._tool_name) - if adapter is None: - raise RuntimeError(f"No adapter found for tool: {self._tool_name}") - return await adapter.call_tool(self._tool_name, self._callable.name, kwargs) + return await self._registry.call_tool(self._tool_name, self._callable.name, kwargs) async def call_async(self, **kwargs: Any) -> Any: """Execute callable asynchronously. Always returns awaitable. diff --git a/src/py_code_mode/tools/registry.py b/src/py_code_mode/tools/registry.py index 29052e4..b6ab258 100644 --- a/src/py_code_mode/tools/registry.py +++ b/src/py_code_mode/tools/registry.py @@ -153,6 +153,32 @@ def __init__( self._tools: dict[str, Tool] = {} # name -> Tool self._tool_to_adapter: dict[str, ToolAdapter] = {} # name -> adapter self._vectors: dict[str, list[float]] = {} # name -> embedding vector + self._tool_middlewares: tuple[ToolMiddleware, ...] = () + self._tool_middleware_executor_type: str | None = None + self._tool_middleware_origin: str | None = None + + def _unwrap_adapter(self, adapter: ToolAdapter) -> ToolAdapter: + # Local import to avoid mandatory dependency on middleware wrapper. + from py_code_mode.tools.adapters.middleware import MiddlewareAdapter + + while isinstance(adapter, MiddlewareAdapter): + adapter = adapter.inner + return adapter + + def _wrap_adapter_if_needed(self, adapter: ToolAdapter) -> ToolAdapter: + if not self._tool_middlewares: + return adapter + + from py_code_mode.tools.adapters.middleware import MiddlewareAdapter + + # Avoid stacking wrappers. + adapter = self._unwrap_adapter(adapter) + return MiddlewareAdapter( + adapter, + self._tool_middlewares, + executor_type=self._tool_middleware_executor_type, + origin=self._tool_middleware_origin, + ) @classmethod async def from_dir( @@ -253,7 +279,7 @@ def add_adapter(self, adapter: ToolAdapter) -> None: Args: adapter: The adapter to add. """ - self._adapters.append(adapter) + self._adapters.append(self._wrap_adapter_if_needed(adapter)) def get_adapters(self) -> list[ToolAdapter]: """Get all registered adapters. @@ -309,6 +335,7 @@ def register_adapter( Raises: ValueError: If a tool name conflicts with an existing tool. """ + adapter = self._wrap_adapter_if_needed(adapter) self._adapters.append(adapter) adapter_tools = adapter.list_tools() registered = [] @@ -397,10 +424,16 @@ async def call_tool( ToolNotFoundError: If tool not found. ToolCallError: If tool execution fails. """ - if name not in self._tools: - raise ToolNotFoundError(name, list(self._tools.keys())) + # Normal path: tools registered via register_adapter() + adapter = self._tool_to_adapter.get(name) - adapter = self._tool_to_adapter[name] + # Compatibility path: some code uses add_adapter() (no registration), + # but ToolsNamespace still discovers tools via adapter.list_tools(). + if adapter is None: + adapter = self.find_adapter_for_tool(name) + if adapter is None: + available = [t.name for t in self.get_all_tools()] + raise ToolNotFoundError(name, available) try: return await adapter.call_tool(name, callable_name, args) @@ -513,7 +546,7 @@ def apply_tool_middlewares( executor_type: str | None = None, origin: str | None = None, ) -> None: - """Wrap all adapters with a tool call middleware chain. + """Apply tool call middleware chain to all adapters. This method mutates the registry in-place and updates internal mappings so that *all* tool call paths (ToolsNamespace, RPC providers, registry.call_tool) @@ -521,33 +554,21 @@ def apply_tool_middlewares( Notes: - Only `call_tool()` is wrapped. `list_tools()` and `describe()` are passed through. - - Applying multiple times will stack wrappers; call once per registry instance. + - Applying multiple times replaces the active middleware chain. """ - mws = tuple(middlewares) - if not mws: - return - - from py_code_mode.tools.adapters.middleware import MiddlewareAdapter - - old_adapters = list(self._adapters) - adapter_map: dict[int, ToolAdapter] = {} - new_adapters: list[ToolAdapter] = [] - - for adapter in old_adapters: - wrapped: ToolAdapter = MiddlewareAdapter( - adapter, - mws, - executor_type=executor_type, - origin=origin, - ) - new_adapters.append(wrapped) - adapter_map[id(adapter)] = wrapped + self._tool_middlewares = tuple(middlewares) + self._tool_middleware_executor_type = executor_type + self._tool_middleware_origin = origin - self._adapters = new_adapters + base_adapters = [self._unwrap_adapter(a) for a in self._adapters] + wrapped_adapters = [self._wrap_adapter_if_needed(a) for a in base_adapters] + base_to_wrapped = {id(b): w for b, w in zip(base_adapters, wrapped_adapters, strict=True)} + self._adapters = wrapped_adapters # Update tool->adapter mapping used by registry.call_tool(). for tool_name, adapter in list(self._tool_to_adapter.items()): - mapped = adapter_map.get(id(adapter)) + base = self._unwrap_adapter(adapter) + mapped = base_to_wrapped.get(id(base)) if mapped is not None: self._tool_to_adapter[tool_name] = mapped diff --git a/tests/test_tool_middleware_plumbing.py b/tests/test_tool_middleware_plumbing.py index 7aea567..4da05b0 100644 --- a/tests/test_tool_middleware_plumbing.py +++ b/tests/test_tool_middleware_plumbing.py @@ -9,13 +9,14 @@ class _FakeAdapter: - def __init__(self) -> None: + def __init__(self, tool_name: str = "echo") -> None: + self._tool_name = tool_name self.calls: list[tuple[str, str | None, dict[str, Any]]] = [] def list_tools(self) -> list[Tool]: return [ Tool( - name="echo", + name=self._tool_name, description="Echo", callables=( ToolCallable( @@ -115,3 +116,58 @@ async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[overr with pytest.raises(ToolCallError, match="denied: echo.say"): await registry.call_tool("echo", "say", {"message": "hi"}) assert adapter.calls == [] + + +@pytest.mark.asyncio +async def test_tool_middleware_applies_to_new_adapters_registered_later() -> None: + adapter1 = _FakeAdapter("echo1") + registry = ToolRegistry() + registry.register_adapter(adapter1) + + events: list[str] = [] + + class _Recorder(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + events.append(f"pre:{ctx.full_name}") + return await call_next(ctx) + + registry.apply_tool_middlewares( + (_Recorder(),), executor_type="deno-sandbox", origin="deno-sandbox" + ) + + # Register a new adapter after middleware is applied. + adapter2 = _FakeAdapter("echo2") + registry.register_adapter(adapter2) + + res = await registry.call_tool("echo2", "say", {"message": "hi"}) + assert res["ok"] is True + assert events == ["pre:echo2.say"] + + +@pytest.mark.asyncio +async def test_apply_tool_middlewares_replaces_chain_not_stacks() -> None: + adapter = _FakeAdapter() + registry = ToolRegistry() + registry.register_adapter(adapter) + + events: list[str] = [] + + class _RecorderA(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + events.append("A") + return await call_next(ctx) + + class _RecorderB(ToolMiddleware): + async def __call__(self, ctx: ToolCallContext, call_next): # type: ignore[override] + events.append("B") + return await call_next(ctx) + + registry.apply_tool_middlewares( + (_RecorderA(),), executor_type="deno-sandbox", origin="deno-sandbox" + ) + registry.apply_tool_middlewares( + (_RecorderB(),), executor_type="deno-sandbox", origin="deno-sandbox" + ) + + await registry.call_tool("echo", "say", {"message": "hi"}) + assert events == ["B"] From ae3949e353274bf70d89f16c74cff3bfe2776f98 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 18:08:06 -0800 Subject: [PATCH 17/27] Docs: mention tool middleware hook --- docs/executors.md | 1 + docs/tools.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/executors.md b/docs/executors.md index 7389f40..56a368d 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -12,6 +12,7 @@ Notes: Key differences vs the other executors: - **Async-first sandbox API**: use `await tools.*`, `await workflows.*`, `await artifacts.*`, `await deps.*`. - **Best-effort deps**: dependency installs run via Pyodide `micropip` and many packages (especially those requiring native extensions) will not work. +- **Tool middleware support (host-side)**: you can attach tool call middleware via `DenoSandboxConfig.tool_middlewares` (useful for audit logging, approvals, allow/deny, etc.). ```python from pathlib import Path diff --git a/docs/tools.md b/docs/tools.md index 36e85e6..c0544d3 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -8,6 +8,23 @@ Tools wrap external capabilities as callable functions. Three adapter types supp Define command-line tools with YAML schema + recipes. +## Tool Middleware (Experimental) + +py-code-mode supports a host-side middleware chain around tool execution. This is intended for: +- Audit logging and metrics +- Allow/deny decisions and interactive approvals +- Argument rewriting, retries, caching, etc. + +Notes: +- Middleware runs where tools execute (host-side ToolAdapters). +- Enforcement guarantees are strongest with `DenoSandboxExecutor` because sandboxed Python can only access tools via RPC back to the host. + +API surface: +- `ToolMiddleware`: `async def __call__(ctx: ToolCallContext, call_next) -> Any` +- `ToolCallContext`: includes `tool_name`, `callable_name`, `args`, and metadata like `executor_type`, `origin`, `request_id`. + +To enable for `DenoSandboxExecutor`, pass `tool_middlewares` in `DenoSandboxConfig`. + ### Schema Definition ```yaml From bdb7f55af14f0071e8dcebe8700f5aec058675a2 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 19:23:54 -0800 Subject: [PATCH 18/27] Docs: note tool middleware in architecture --- docs/ARCHITECTURE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4d7d282..3ca41b5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -811,6 +811,9 @@ Agent writes: "tools.curl.get(url='...')" (use `await` only in DenoSandboxExecut Note on sandboxing: - `DenoSandboxExecutor` sandboxes Python execution in Pyodide, but **tools execute host-side** (the sandbox calls back to the host over RPC to run tools). If you need strict sandbox boundaries, avoid `tools.*` and stick to pure Python plus `deps.*` in the sandbox. +Note on tool middleware: +- Tool calls can be wrapped by a host-side middleware chain (audit logging, approvals, allow/deny, retries, etc.). Enforcement guarantees are strongest for `DenoSandboxExecutor`, because sandboxed Python can only access tools via host RPC. + ### ToolProxy Methods ``` From e72f2aa4cc5d1e8205ad956033ddedeb4aac3362 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 19:27:47 -0800 Subject: [PATCH 19/27] Examples: add DenoSandboxExecutor --- docs/executors.md | 1 + examples/deno-sandbox/.gitignore | 5 ++ examples/deno-sandbox/README.md | 36 ++++++++++++++ examples/deno-sandbox/example.py | 71 +++++++++++++++++++++++++++ examples/deno-sandbox/pyproject.toml | 15 ++++++ examples/deno-sandbox/tools/echo.yaml | 18 +++++++ 6 files changed, 146 insertions(+) create mode 100644 examples/deno-sandbox/.gitignore create mode 100644 examples/deno-sandbox/README.md create mode 100644 examples/deno-sandbox/example.py create mode 100644 examples/deno-sandbox/pyproject.toml create mode 100644 examples/deno-sandbox/tools/echo.yaml diff --git a/docs/executors.md b/docs/executors.md index 56a368d..e8ff816 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -8,6 +8,7 @@ Executors determine where and how agent code runs. Four backends are available: Notes: - Backend key: `"deno-sandbox"`. +- Example: `examples/deno-sandbox/`. Key differences vs the other executors: - **Async-first sandbox API**: use `await tools.*`, `await workflows.*`, `await artifacts.*`, `await deps.*`. diff --git a/examples/deno-sandbox/.gitignore b/examples/deno-sandbox/.gitignore new file mode 100644 index 0000000..cc2f05a --- /dev/null +++ b/examples/deno-sandbox/.gitignore @@ -0,0 +1,5 @@ +data/ +.deno-cache/ +__pycache__/ +.pytest_cache/ + diff --git a/examples/deno-sandbox/README.md b/examples/deno-sandbox/README.md new file mode 100644 index 0000000..ae37461 --- /dev/null +++ b/examples/deno-sandbox/README.md @@ -0,0 +1,36 @@ +# DenoSandboxExecutor Example + +Demonstrates sandboxed Python execution using `DenoSandboxExecutor` (Deno + Pyodide). + +## What Is DenoSandboxExecutor? + +`DenoSandboxExecutor` runs Python in **Pyodide (WASM)** inside a **Deno** subprocess and uses Deno permissions to sandbox the Python runtime. + +Important: **tools execute host-side**. Any `tools.*` call is sent over RPC to the host Python process and executed by ToolAdapters with host permissions. + +## Prerequisites + +- Python 3.12+ +- Deno installed and on PATH + +## Setup + +```bash +cd examples/deno-sandbox +uv sync +``` + +## Run + +```bash +uv run python example.py +``` + +## Notes + +- In `DenoSandboxExecutor`, namespace calls are async-first: use `await tools.*`, `await artifacts.*`, `await workflows.*`, `await deps.*`. +- `DenoSandboxConfig.network_profile` controls *sandbox* network access: + - `none`: deny sandbox network (no runtime `micropip` installs) + - `deps-only`: allow typical PyPI/CDN hosts for `micropip` + - `full`: allow all sandbox network + diff --git a/examples/deno-sandbox/example.py b/examples/deno-sandbox/example.py new file mode 100644 index 0000000..1180993 --- /dev/null +++ b/examples/deno-sandbox/example.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""DenoSandboxExecutor example - Deno + Pyodide (WASM) sandbox. + +Highlights: +- Python code executes inside Pyodide (WASM) within a Deno subprocess. +- In DenoSandboxExecutor, namespace calls are async-first: + - use `await tools.*`, `await artifacts.*`, `await workflows.*`, `await deps.*` +- Tool execution is host-side: `tools.*` calls are RPC'd back to the host and + executed by ToolAdapters with host permissions. +""" + +from __future__ import annotations + +import asyncio +import shutil +from pathlib import Path + +from py_code_mode import FileStorage, Session +from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor + +HERE = Path(__file__).resolve().parent + + +async def main() -> None: + if shutil.which("deno") is None: + raise SystemExit( + "deno not found on PATH. Install Deno to run this example: https://deno.com/" + ) + + storage = FileStorage(base_path=HERE / "data") + deno_dir = HERE / ".deno-cache" + tools_dir = HERE / "tools" + + config = DenoSandboxConfig( + tools_path=tools_dir, + deno_dir=deno_dir, + # Start with no sandbox network access. (Tools still run host-side.) + network_profile="none", + default_timeout=60.0, + ipc_timeout=120.0, + ) + executor = DenoSandboxExecutor(config) + + async with Session(storage=storage, executor=executor) as session: + print("Basic execution (sandboxed Python)...") + r = await session.run("1 + 1") + print(" 1 + 1 =", r.value) + + print("\nTool call (HOST-SIDE, via RPC)...") + r = await session.run("(await tools.echo.say(message='hello from host tool')).strip()") + print(" tools.echo.say ->", r.value) + + print("\nArtifacts (stored via host storage backend)...") + await session.run("await artifacts.save('hello.txt', 'hi', description='demo')") + r = await session.run("await artifacts.load('hello.txt')") + print(" artifacts.load('hello.txt') ->", r.value) + + print("\nWorkflows (stored in host storage; executed in sandbox)...") + src = "async def run(x: int) -> int:\\n return x * 2\\n" + await session.run(f"await workflows.create('double', {src!r}, description='Multiply by 2')") + r = await session.run("await workflows.invoke(workflow_name='double', x=21)") + print(" workflows.invoke('double', x=21) ->", r.value) + + print("\nStdout capture...") + r = await session.run('print("hello from sandbox stdout")\\n"done"') + print(" value:", r.value) + print(" stdout:", r.stdout.strip()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/deno-sandbox/pyproject.toml b/examples/deno-sandbox/pyproject.toml new file mode 100644 index 0000000..ec09701 --- /dev/null +++ b/examples/deno-sandbox/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "py-code-mode-deno-sandbox-example" +version = "0.1.0" +description = "DenoSandboxExecutor example using Deno + Pyodide sandbox" +requires-python = ">=3.12" +dependencies = [ + "py-code-mode", +] + +[tool.uv] +dev-dependencies = [] + +[tool.uv.sources] +py-code-mode = { path = "../..", editable = true } + diff --git a/examples/deno-sandbox/tools/echo.yaml b/examples/deno-sandbox/tools/echo.yaml new file mode 100644 index 0000000..85a44cb --- /dev/null +++ b/examples/deno-sandbox/tools/echo.yaml @@ -0,0 +1,18 @@ +name: echo +description: Echo text (host-side tool; executed outside the Deno sandbox) +command: /bin/echo +timeout: 5 + +schema: + positional: + - name: message + type: string + required: true + description: Message to echo + +recipes: + say: + description: Echo the message + params: + message: {} + From 0a00e25a22da4cb42f7e102a4978b62666b73718 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 20:37:43 -0800 Subject: [PATCH 20/27] DenoSandbox: test workflow invoking another workflow --- tests/test_deno_sandbox_executor.py | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_deno_sandbox_executor.py b/tests/test_deno_sandbox_executor.py index 6bd29da..2c57ee6 100644 --- a/tests/test_deno_sandbox_executor.py +++ b/tests/test_deno_sandbox_executor.py @@ -258,6 +258,48 @@ async def test_deno_sandbox_executor_workflows_roundtrip(tmp_path: Path) -> None assert r2.value == "hi py-code-mode" +@pytest.mark.asyncio +async def test_deno_sandbox_executor_workflow_calls_other_workflow(tmp_path: Path) -> None: + """Ensure a workflow can invoke another workflow inside the sandbox.""" + + from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor + from py_code_mode.session import Session + from py_code_mode.storage import FileStorage + + storage = FileStorage(tmp_path / "storage") + deno_dir = tmp_path / "deno_dir" + deno_dir.mkdir(parents=True, exist_ok=True) + + executor = DenoSandboxExecutor( + DenoSandboxConfig( + deno_dir=deno_dir, + default_timeout=60.0, + ipc_timeout=120.0, + network_profile="none", + ) + ) + + src_double = "async def run(x: int) -> int:\n return x * 2\n" + src_quadruple = ( + "async def run(x: int) -> int:\n" + " d = await workflows.invoke(workflow_name='double', x=x)\n" + " return await workflows.invoke(workflow_name='double', x=d)\n" + ) + + async with Session(storage=storage, executor=executor) as session: + r1 = await session.run(f"await workflows.create('double', {src_double!r}, 'double')") + assert r1.error is None + + r2 = await session.run( + f"await workflows.create('quadruple', {src_quadruple!r}, 'quadruple')" + ) + assert r2.error is None + + r3 = await session.run("await workflows.invoke('quadruple', x=10)") + assert r3.error is None + assert r3.value == 40 + + @pytest.mark.asyncio async def test_deno_sandbox_executor_tools_via_rpc(tmp_path: Path) -> None: from py_code_mode.execution import DenoSandboxConfig, DenoSandboxExecutor From f2038e437800a4c53506109a3247551e8377ca51 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 20:51:06 -0800 Subject: [PATCH 21/27] CI: run DenoSandbox integration tests --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d86f522..3f89e9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,3 +49,23 @@ jobs: verbose: true env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + test-deno-sandbox: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v7 + - uses: denoland/setup-deno@v2 + - name: Cache Deno modules + uses: actions/cache@v5 + with: + path: ~/.cache/deno + key: deno-${{ runner.os }}-${{ hashFiles('src/py_code_mode/execution/deno_sandbox/runner/**') }} + restore-keys: | + deno-${{ runner.os }}- + - run: uv sync --all-extras + - name: Run Deno sandbox integration tests + env: + PY_CODE_MODE_TEST_DENO: "1" + DENO_DIR: ~/.cache/deno + run: uv run pytest -n 0 tests/test_deno_sandbox_imports.py tests/test_deno_sandbox_executor.py -v From d2e40112964043b2d36c6d3a2e36f010e038265d Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 20:57:24 -0800 Subject: [PATCH 22/27] Tests: define pytest markers + auto-mark xdist groups --- pyproject.toml | 8 ++++++++ tests/conftest.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index eacc782..b55b556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,14 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-n auto --dist loadgroup" +markers = [ + "slow: Long-running tests (often subprocess/docker integration).", + "integration: Integration tests that may require external services (e.g. Redis).", + "docker: Tests that require Docker (typically xdist_group('docker')).", + "subprocess: Tests that use the SubprocessExecutor / kernel subprocess (typically xdist_group('subprocess')).", + "venv: Tests that touch venv caching/management (typically xdist_group('venv')).", + "deno: Tests that require Deno (DenoSandboxExecutor integration).", +] [tool.ruff] target-version = "py312" diff --git a/tests/conftest.py b/tests/conftest.py index 5c6c8cf..3a5b2f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -215,9 +215,14 @@ def pytest_collection_modifyitems(config, items): # noqa: ARG001 # Check if test is in the docker xdist group for marker in item.iter_markers("xdist_group"): if marker.args and marker.args[0] == "docker": + item.add_marker(pytest.mark.docker) # Add the fixture as a dependency if "docker_image_check" not in item.fixturenames: item.fixturenames.insert(0, "docker_image_check") + elif marker.args and marker.args[0] == "subprocess": + item.add_marker(pytest.mark.subprocess) + elif marker.args and marker.args[0] == "venv": + item.add_marker(pytest.mark.venv) class MockAdapter: From df7a18275141ffc82dc15d92c982e851f4fafcaf Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 21:08:14 -0800 Subject: [PATCH 23/27] Docs: update AGENTS.md for DenoSandbox + test markers --- AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2200cfd..a7eaa81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,7 @@ src/py_code_mode/ subprocess/ # Jupyter kernel-based subprocess executor container/ # Docker container executor in_process/ # Same-process executor + deno_sandbox/ # Deno + Pyodide (WASM) sandbox executor (experimental) workflows/ # Skill storage, library, and vector stores tools/ # Tool adapters: CLI, MCP, HTTP adapters/ # CLI, MCP, HTTP adapter implementations @@ -76,6 +77,7 @@ When agents write code, four namespaces are available: | SubprocessExecutor | Recommended default. Process isolation via Jupyter kernel. | | ContainerExecutor | Docker isolation for untrusted code. | | InProcessExecutor | Maximum speed for trusted code. | +| DenoSandboxExecutor | Sandboxed Python via Deno + Pyodide (WASM). Tools execute host-side. | --- @@ -98,6 +100,12 @@ uv run pytest -k "test_workflow" # Run without parallelism (for debugging) uv run pytest -n 0 + +# Run Deno sandbox integration tests (requires Deno installed) +PY_CODE_MODE_TEST_DENO=1 uv run pytest -n 0 tests/test_deno_sandbox_executor.py -v + +# Filter subsets (markers are defined in pyproject.toml) +uv run pytest -m "not docker" ``` ### Linting and Type Checking @@ -192,6 +200,7 @@ Tools are defined in YAML files. Key patterns: - Container tests are in `tests/container/` - require Docker - Use `@pytest.mark.xdist_group("group_name")` for tests that need isolation - Redis tests use testcontainers - spin up automatically +- Markers like `docker`, `subprocess`, `venv`, and `deno` are used to filter test subsets --- From 88d30e364fb9b2c387dd7b4052247b130c7fe3a1 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 21:10:10 -0800 Subject: [PATCH 24/27] Docs: expand AGENTS.md test subset commands --- AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index a7eaa81..0e999b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,6 +106,15 @@ PY_CODE_MODE_TEST_DENO=1 uv run pytest -n 0 tests/test_deno_sandbox_executor.py # Filter subsets (markers are defined in pyproject.toml) uv run pytest -m "not docker" + +# Common subsets +uv run pytest -m "not slow" +uv run pytest -m docker +uv run pytest -m subprocess +uv run pytest -m "not docker and not subprocess" + +# CI uses all extras; for local repro of CI failures you may want: +uv sync --all-extras ``` ### Linting and Type Checking From 664cd7ef711c49559a4f59ce5d4e1ebc3dc8db61 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 22:15:23 -0800 Subject: [PATCH 25/27] Prune performative tests and improve docker test gating --- tests/conftest.py | 16 ++ tests/test_backend.py | 100 ---------- tests/test_deps_installer.py | 65 ------- tests/test_deps_namespace.py | 56 ------ tests/test_deps_store.py | 45 ----- tests/test_error_handling.py | 122 +++--------- tests/test_executor_configs.py | 58 +----- tests/test_executor_deps_methods.py | 96 ---------- tests/test_executor_protocol.py | 187 +------------------ tests/test_mcp_adapter.py | 25 --- tests/test_rpc_errors.py | 14 +- tests/test_semantic.py | 15 -- tests/test_session.py | 119 +++--------- tests/test_skill_library_vector_store.py | 13 -- tests/test_skill_store.py | 7 +- tests/test_storage.py | 7 +- tests/test_storage_vector_store.py | 59 ++---- tests/test_storage_wrapper_cleanup.py | 27 --- tests/test_subprocess_executor.py | 100 ++-------- tests/test_subprocess_namespace_injection.py | 87 +++++---- tests/test_vector_store.py | 64 ------- 21 files changed, 159 insertions(+), 1123 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3a5b2f1..a809d8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,6 +59,16 @@ def _patched_embedder_init(self, model_name=None, start_loading=False): TESTCONTAINERS_AVAILABLE = False +def _docker_daemon_is_available() -> bool: + try: + import docker + + docker.from_env().ping() + return True + except Exception: + return False + + # ============================================================================= # Docker Image Staleness Check # ============================================================================= @@ -211,7 +221,11 @@ def docker_image_check(): def pytest_collection_modifyitems(config, items): # noqa: ARG001 """Add docker_image_check fixture to all tests in xdist_group('docker').""" + docker_fixtures = {"redis_container", "redis_client", "redis_url"} for item in items: + if docker_fixtures.intersection(getattr(item, "fixturenames", ())): + item.add_marker(pytest.mark.docker) + # Check if test is in the docker xdist group for marker in item.iter_markers("xdist_group"): if marker.args and marker.args[0] == "docker": @@ -786,6 +800,8 @@ def redis_container(): """ if not TESTCONTAINERS_AVAILABLE: pytest.skip("testcontainers[redis] not installed") + if not _docker_daemon_is_available(): + pytest.skip("Docker daemon not available for testcontainers Redis") with RedisContainer(image="redis:7-alpine") as container: yield container diff --git a/tests/test_backend.py b/tests/test_backend.py index f899c78..fdc3540 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -102,106 +102,6 @@ async def test_context_manager_support(self, tmp_path) -> None: # After exit, resources should be released -class TestCapabilityConstants: - """Tests for Capability constants.""" - - def test_timeout_capability_exists(self) -> None: - """TIMEOUT capability must be defined.""" - assert hasattr(Capability, "TIMEOUT") - assert Capability.TIMEOUT == "timeout" - - def test_process_isolation_capability_exists(self) -> None: - """PROCESS_ISOLATION capability must be defined.""" - assert hasattr(Capability, "PROCESS_ISOLATION") - assert Capability.PROCESS_ISOLATION == "process_isolation" - - def test_network_isolation_capability_exists(self) -> None: - """NETWORK_ISOLATION capability must be defined.""" - assert hasattr(Capability, "NETWORK_ISOLATION") - assert Capability.NETWORK_ISOLATION == "network_isolation" - - def test_network_filtering_capability_exists(self) -> None: - """NETWORK_FILTERING capability must be defined.""" - assert hasattr(Capability, "NETWORK_FILTERING") - assert Capability.NETWORK_FILTERING == "network_filtering" - - def test_filesystem_isolation_capability_exists(self) -> None: - """FILESYSTEM_ISOLATION capability must be defined.""" - assert hasattr(Capability, "FILESYSTEM_ISOLATION") - assert Capability.FILESYSTEM_ISOLATION == "filesystem_isolation" - - def test_memory_limit_capability_exists(self) -> None: - """MEMORY_LIMIT capability must be defined.""" - assert hasattr(Capability, "MEMORY_LIMIT") - assert Capability.MEMORY_LIMIT == "memory_limit" - - def test_cpu_limit_capability_exists(self) -> None: - """CPU_LIMIT capability must be defined.""" - assert hasattr(Capability, "CPU_LIMIT") - assert Capability.CPU_LIMIT == "cpu_limit" - - def test_reset_capability_exists(self) -> None: - """RESET capability must be defined.""" - assert hasattr(Capability, "RESET") - assert Capability.RESET == "reset" - - def test_all_returns_set_of_all_capabilities(self) -> None: - """Capability.all() must return set of all defined capabilities.""" - all_caps = Capability.all() - - assert isinstance(all_caps, set) - assert Capability.TIMEOUT in all_caps - assert Capability.NETWORK_ISOLATION in all_caps - assert len(all_caps) >= 8 # At least 8 capabilities defined - - -class TestExecutionResult: - """Tests for the unified ExecutionResult type.""" - - def test_result_has_value(self) -> None: - """ExecutionResult must have value field.""" - result = ExecutionResult(value=42, stdout="", error=None) - assert result.value == 42 - - def test_result_has_stdout(self) -> None: - """ExecutionResult must have stdout field.""" - result = ExecutionResult(value=None, stdout="output", error=None) - assert result.stdout == "output" - - def test_result_has_error(self) -> None: - """ExecutionResult must have error field.""" - result = ExecutionResult(value=None, stdout="", error="Something broke") - assert result.error == "Something broke" - - def test_result_is_ok_property(self) -> None: - """ExecutionResult must have is_ok property.""" - success = ExecutionResult(value=42, stdout="", error=None) - failure = ExecutionResult(value=None, stdout="", error="oops") - - assert success.is_ok is True - assert failure.is_ok is False - - def test_result_has_optional_execution_time(self) -> None: - """ExecutionResult may have execution_time_ms field.""" - result = ExecutionResult( - value=42, - stdout="", - error=None, - execution_time_ms=123.45, - ) - assert result.execution_time_ms == 123.45 - - def test_result_has_optional_backend_info(self) -> None: - """ExecutionResult may have backend_info dict.""" - result = ExecutionResult( - value=42, - stdout="", - error=None, - backend_info={"session_id": "abc123"}, - ) - assert result.backend_info["session_id"] == "abc123" - - class TestBackendRegistry: """Tests for backend registration and discovery.""" diff --git a/tests/test_deps_installer.py b/tests/test_deps_installer.py index cc9c076..6b4f4c1 100644 --- a/tests/test_deps_installer.py +++ b/tests/test_deps_installer.py @@ -121,59 +121,6 @@ async def test_developer_adds_more_deps_after_sync(self, tmp_path: Path) -> None assert "numpy" in result.installed -# ============================================================================= -# Contract Tests (SyncResult) -# ============================================================================= - - -class TestSyncResultContract: - """Tests for SyncResult dataclass/structure.""" - - def test_sync_result_has_installed_attribute(self) -> None: - """SyncResult has 'installed' attribute (set of packages). - - Breaks when: SyncResult structure is wrong. - """ - from py_code_mode.deps import SyncResult - - result = SyncResult(installed=set(), already_present=set(), failed=set()) - assert hasattr(result, "installed") - assert isinstance(result.installed, set) - - def test_sync_result_has_already_present_attribute(self) -> None: - """SyncResult has 'already_present' attribute (set of packages). - - Breaks when: SyncResult structure is wrong. - """ - from py_code_mode.deps import SyncResult - - result = SyncResult(installed=set(), already_present=set(), failed=set()) - assert hasattr(result, "already_present") - assert isinstance(result.already_present, set) - - def test_sync_result_has_failed_attribute(self) -> None: - """SyncResult has 'failed' attribute (set of packages). - - Breaks when: SyncResult structure is wrong. - """ - from py_code_mode.deps import SyncResult - - result = SyncResult(installed=set(), already_present=set(), failed=set()) - assert hasattr(result, "failed") - assert isinstance(result.failed, set) - - def test_sync_result_is_dataclass(self) -> None: - """SyncResult is a dataclass for easy construction. - - Breaks when: SyncResult is not a dataclass. - """ - from dataclasses import is_dataclass - - from py_code_mode.deps import SyncResult - - assert is_dataclass(SyncResult) - - # ============================================================================= # PackageInstaller Contract Tests # ============================================================================= @@ -182,17 +129,6 @@ def test_sync_result_is_dataclass(self) -> None: class TestPackageInstallerContract: """Tests for PackageInstaller public API.""" - def test_installer_has_sync_method(self) -> None: - """PackageInstaller has sync(store) method. - - Breaks when: sync method is missing. - """ - from py_code_mode.deps import PackageInstaller - - installer = PackageInstaller() - assert hasattr(installer, "sync") - assert callable(installer.sync) - def test_sync_returns_sync_result(self, tmp_path: Path) -> None: """sync() returns SyncResult. @@ -576,7 +512,6 @@ def test_installer_default_timeout(self) -> None: from py_code_mode.deps import PackageInstaller installer = PackageInstaller() - assert hasattr(installer, "timeout") assert installer.timeout >= 60 # At least 60 seconds def test_installer_accepts_extra_pip_args(self, tmp_path: Path) -> None: diff --git a/tests/test_deps_namespace.py b/tests/test_deps_namespace.py index b7c1819..6687d5b 100644 --- a/tests/test_deps_namespace.py +++ b/tests/test_deps_namespace.py @@ -150,62 +150,6 @@ def test_agent_syncs_all_deps(self, tmp_path: Path) -> None: class TestDepsNamespaceContract: """Tests for DepsNamespace public API.""" - def test_namespace_has_add_method(self, tmp_path: Path) -> None: - """DepsNamespace has add(package) method. - - Breaks when: add method is missing. - """ - from py_code_mode.deps import DepsNamespace, FileDepsStore, PackageInstaller - - store = FileDepsStore(tmp_path) - installer = PackageInstaller() - namespace = DepsNamespace(store=store, installer=installer) - - assert hasattr(namespace, "add") - assert callable(namespace.add) - - def test_namespace_has_list_method(self, tmp_path: Path) -> None: - """DepsNamespace has list() method. - - Breaks when: list method is missing. - """ - from py_code_mode.deps import DepsNamespace, FileDepsStore, PackageInstaller - - store = FileDepsStore(tmp_path) - installer = PackageInstaller() - namespace = DepsNamespace(store=store, installer=installer) - - assert hasattr(namespace, "list") - assert callable(namespace.list) - - def test_namespace_has_remove_method(self, tmp_path: Path) -> None: - """DepsNamespace has remove(package) method. - - Breaks when: remove method is missing. - """ - from py_code_mode.deps import DepsNamespace, FileDepsStore, PackageInstaller - - store = FileDepsStore(tmp_path) - installer = PackageInstaller() - namespace = DepsNamespace(store=store, installer=installer) - - assert hasattr(namespace, "remove") - assert callable(namespace.remove) - - def test_namespace_has_sync_method(self, tmp_path: Path) -> None: - """DepsNamespace has sync() method. - - Breaks when: sync method is missing. - """ - from py_code_mode.deps import DepsNamespace, FileDepsStore, PackageInstaller - - store = FileDepsStore(tmp_path) - installer = PackageInstaller() - namespace = DepsNamespace(store=store, installer=installer) - - assert hasattr(namespace, "sync") - assert callable(namespace.sync) - def test_list_returns_list_of_strings(self, tmp_path: Path) -> None: """list() returns list[str]. diff --git a/tests/test_deps_store.py b/tests/test_deps_store.py index 412d35b..1275127 100644 --- a/tests/test_deps_store.py +++ b/tests/test_deps_store.py @@ -48,51 +48,6 @@ def test_redis_deps_store_is_deps_store(self, mock_redis: "MockRedisClient") -> store = RedisDepsStore(mock_redis, prefix="test") assert isinstance(store, DepsStore) - def test_protocol_requires_list_method(self) -> None: - """DepsStore protocol requires list() -> list[str]. - - Breaks when: Protocol doesn't define list method. - """ - from py_code_mode.deps import DepsStore - - assert hasattr(DepsStore, "list") - - def test_protocol_requires_add_method(self) -> None: - """DepsStore protocol requires add(package: str) -> None. - - Breaks when: Protocol doesn't define add method. - """ - from py_code_mode.deps import DepsStore - - assert hasattr(DepsStore, "add") - - def test_protocol_requires_remove_method(self) -> None: - """DepsStore protocol requires remove(package: str) -> bool. - - Breaks when: Protocol doesn't define remove method. - """ - from py_code_mode.deps import DepsStore - - assert hasattr(DepsStore, "remove") - - def test_protocol_requires_clear_method(self) -> None: - """DepsStore protocol requires clear() -> None. - - Breaks when: Protocol doesn't define clear method. - """ - from py_code_mode.deps import DepsStore - - assert hasattr(DepsStore, "clear") - - def test_protocol_requires_hash_method(self) -> None: - """DepsStore protocol requires hash() -> str. - - Breaks when: Protocol doesn't define hash method. - """ - from py_code_mode.deps import DepsStore - - assert hasattr(DepsStore, "hash") - # ============================================================================= # FileDepsStore Tests diff --git a/tests/test_error_handling.py b/tests/test_error_handling.py index c4f1f4c..b19e0b0 100644 --- a/tests/test_error_handling.py +++ b/tests/test_error_handling.py @@ -167,71 +167,43 @@ def hlen(self, key: str) -> int: class TestStorageExceptionHierarchy: - """Tests for new exception types that need to be added. - - These tests will FAIL until the exceptions are added to errors.py: - - StorageError (base) - - StorageReadError (read failures) - - StorageWriteError (write failures) - - ConfigurationError (invalid configuration) - """ + """Tests for storage/configuration exception hierarchy.""" def test_storage_error_exists(self): - """StorageError should be defined in errors module.""" - from py_code_mode import errors - - assert hasattr(errors, "StorageError"), ( - "StorageError not defined. Add to py_code_mode/errors.py:\n" - "class StorageError(CodeModeError):\n" - ' """Base class for storage-related errors."""\n' - " pass" - ) + """StorageError is defined and inherits from CodeModeError.""" + from py_code_mode.errors import StorageError + + assert issubclass(StorageError, CodeModeError) def test_storage_read_error_exists(self): """StorageReadError should be defined and inherit from StorageError.""" - from py_code_mode import errors + from py_code_mode.errors import StorageError, StorageReadError - assert hasattr(errors, "StorageReadError"), ( - "StorageReadError not defined. Add to py_code_mode/errors.py" - ) - # This will fail until both are defined - if hasattr(errors, "StorageError") and hasattr(errors, "StorageReadError"): - assert issubclass(errors.StorageReadError, errors.StorageError) + assert issubclass(StorageReadError, StorageError) def test_storage_write_error_exists(self): """StorageWriteError should be defined and inherit from StorageError.""" - from py_code_mode import errors + from py_code_mode.errors import StorageError, StorageWriteError - assert hasattr(errors, "StorageWriteError"), ( - "StorageWriteError not defined. Add to py_code_mode/errors.py" - ) - if hasattr(errors, "StorageError") and hasattr(errors, "StorageWriteError"): - assert issubclass(errors.StorageWriteError, errors.StorageError) + assert issubclass(StorageWriteError, StorageError) def test_configuration_error_exists(self): """ConfigurationError should be defined.""" - from py_code_mode import errors + from py_code_mode.errors import ConfigurationError - assert hasattr(errors, "ConfigurationError"), ( - "ConfigurationError not defined. Add to py_code_mode/errors.py" - ) - if hasattr(errors, "ConfigurationError"): - assert issubclass(errors.ConfigurationError, CodeModeError) + assert issubclass(ConfigurationError, CodeModeError) def test_storage_read_error_preserves_cause(self): """StorageReadError should preserve the original exception as __cause__.""" - from py_code_mode import errors - - if not hasattr(errors, "StorageReadError"): - pytest.skip("StorageReadError not yet implemented") + from py_code_mode.errors import StorageReadError original = ValueError("original error") try: try: raise original except ValueError as e: - raise errors.StorageReadError("read failed", path="/some/path") from e - except errors.StorageReadError as err: + raise StorageReadError("read failed", path="/some/path") from e + except StorageReadError as err: assert err.__cause__ is original assert "read failed" in str(err) assert hasattr(err, "path") or "/some/path" in str(err) @@ -446,28 +418,13 @@ def test_load_returns_none_for_missing_file(self, tmp_path: Path): assert result is None # Expected behavior - def test_load_raises_for_syntax_error( - self, workflows_dir_with_corruption: Path, log_capture: pytest.LogCaptureFixture - ): + def test_load_raises_for_syntax_error(self, workflows_dir_with_corruption: Path): """load() should raise StorageReadError for Python syntax errors.""" + from py_code_mode.errors import StorageReadError from py_code_mode.workflows import FileWorkflowStore store = FileWorkflowStore(workflows_dir_with_corruption) - - from py_code_mode import errors - - if not hasattr(errors, "StorageReadError"): - # Current behavior: returns None with warning - result = store.load("syntax_error") - if result is None: - # Check at least warning is logged (current behavior has this) - assert any("syntax_error" in record.message for record in log_capture.records), ( - "At minimum, a warning should be logged for syntax errors" - ) - pytest.skip("StorageReadError not yet implemented - upgrade test when added") - - # After fix: should raise StorageReadError - with pytest.raises(errors.StorageReadError): + with pytest.raises(StorageReadError): store.load("syntax_error") @@ -514,49 +471,22 @@ def test_load_returns_none_for_missing_key(self, mock_redis_with_corruption): assert result is None - def test_load_raises_for_invalid_json( - self, mock_redis_with_corruption, log_capture: pytest.LogCaptureFixture - ): + def test_load_raises_for_invalid_json(self, mock_redis_with_corruption): """load() should raise StorageReadError for invalid JSON.""" + from py_code_mode.errors import StorageReadError from py_code_mode.workflows import RedisWorkflowStore store = RedisWorkflowStore(mock_redis_with_corruption, prefix="workflows") - - from py_code_mode import errors - - if not hasattr(errors, "StorageReadError"): - # Current behavior: returns None with warning - result = store.load("corrupt_json") - if result is None: - # At least check warning is logged - assert any("corrupt_json" in record.message for record in log_capture.records), ( - "Warning should be logged for corrupt JSON" - ) - pytest.skip("StorageReadError not yet implemented") - - # After fix: should raise StorageReadError - with pytest.raises(errors.StorageReadError): + with pytest.raises(StorageReadError): store.load("corrupt_json") - def test_load_raises_for_missing_fields( - self, mock_redis_with_corruption, log_capture: pytest.LogCaptureFixture - ): + def test_load_raises_for_missing_fields(self, mock_redis_with_corruption): """load() should raise StorageReadError for incomplete workflow data.""" + from py_code_mode.errors import StorageReadError from py_code_mode.workflows import RedisWorkflowStore store = RedisWorkflowStore(mock_redis_with_corruption, prefix="workflows") - - from py_code_mode import errors - - if not hasattr(errors, "StorageReadError"): - result = store.load("missing_fields") - if result is None: - assert any("missing" in record.message.lower() for record in log_capture.records), ( - "Warning should be logged for missing fields" - ) - pytest.skip("StorageReadError not yet implemented") - - with pytest.raises(errors.StorageReadError): + with pytest.raises(StorageReadError): store.load("missing_fields") def test_list_all_logs_warning_for_corrupt_entries( @@ -797,9 +727,9 @@ async def test_developer_discovers_workflow_parse_error_through_logs( log_messages = " ".join(r.message for r in log_capture.records) assert "syntax_error" in log_messages, "Log should mention which file had errors" # Ideally also shows the error type - assert "syntax" in log_messages.lower() or "error" in log_messages.lower(), ( - "Log should indicate type of error" - ) + assert ( + "syntax" in log_messages.lower() or "error" in log_messages.lower() + ), "Log should indicate type of error" # ============================================================================= diff --git a/tests/test_executor_configs.py b/tests/test_executor_configs.py index 0fc884f..1dd3c84 100644 --- a/tests/test_executor_configs.py +++ b/tests/test_executor_configs.py @@ -422,57 +422,7 @@ def test_to_docker_config_includes_deps_file(self, tmp_path: Path) -> None: assert str(deps_file.parent.absolute()) in docker_config["volumes"] -class TestAllConfigsHaveNewFields: - """Cross-cutting tests to ensure all configs have the new fields.""" - - def test_all_configs_have_tools_path(self) -> None: - """All executor configs have tools_path field. - - Contract: Consistent API across all executors. - Breaks when: Any config missing the field. - """ - from py_code_mode.execution import ContainerConfig, InProcessConfig, SubprocessConfig - - assert hasattr(InProcessConfig(), "tools_path") - assert hasattr(SubprocessConfig(), "tools_path") - assert hasattr(ContainerConfig(), "tools_path") - - def test_all_configs_have_deps(self) -> None: - """All executor configs have deps field. - - Contract: Consistent API across all executors. - Breaks when: Any config missing the field. - """ - from py_code_mode.execution import ContainerConfig, InProcessConfig, SubprocessConfig - - assert hasattr(InProcessConfig(), "deps") - assert hasattr(SubprocessConfig(), "deps") - assert hasattr(ContainerConfig(), "deps") - - def test_all_configs_have_deps_file(self) -> None: - """All executor configs have deps_file field. - - Contract: Consistent API across all executors. - Breaks when: Any config missing the field. - """ - from py_code_mode.execution import ContainerConfig, InProcessConfig, SubprocessConfig - - assert hasattr(InProcessConfig(), "deps_file") - assert hasattr(SubprocessConfig(), "deps_file") - assert hasattr(ContainerConfig(), "deps_file") - - def test_all_configs_have_ipc_timeout(self) -> None: - """All executor configs have ipc_timeout field. - - Contract: All configs support ipc_timeout. - Breaks when: Any config missing ipc_timeout. - Note: SubprocessConfig defaults to None (unlimited), others to 30.0. - """ - from py_code_mode.execution import ContainerConfig, InProcessConfig, SubprocessConfig - - assert hasattr(InProcessConfig(), "ipc_timeout") - assert hasattr(SubprocessConfig(), "ipc_timeout") - assert hasattr(ContainerConfig(), "ipc_timeout") - assert InProcessConfig().ipc_timeout == 30.0 - assert SubprocessConfig().ipc_timeout is None - assert ContainerConfig().ipc_timeout == 30.0 +# +# NOTE: Previously this file had "field presence" tests (mostly `hasattr(...)`) +# to enforce config consistency. Those were removed as low-value/performative: +# any real API break here should be caught by executor construction/start tests. diff --git a/tests/test_executor_deps_methods.py b/tests/test_executor_deps_methods.py index 05a1ed8..58d0a05 100644 --- a/tests/test_executor_deps_methods.py +++ b/tests/test_executor_deps_methods.py @@ -34,30 +34,6 @@ class TestInProcessExecutorInstallDepsContract: """Contract tests for InProcessExecutor.install_deps().""" - def test_executor_has_install_deps_method(self) -> None: - """InProcessExecutor has install_deps() method. - - Contract: Executor protocol requires install_deps(packages) method. - Breaks when: Method not implemented. - """ - from py_code_mode.execution.in_process import InProcessExecutor - - executor = InProcessExecutor() - assert hasattr(executor, "install_deps") - assert callable(executor.install_deps) - - def test_executor_has_uninstall_deps_method(self) -> None: - """InProcessExecutor has uninstall_deps() method. - - Contract: Executor protocol requires uninstall_deps(packages) method. - Breaks when: Method not implemented. - """ - from py_code_mode.execution.in_process import InProcessExecutor - - executor = InProcessExecutor() - assert hasattr(executor, "uninstall_deps") - assert callable(executor.uninstall_deps) - def test_executor_supports_deps_install_capability(self) -> None: """InProcessExecutor reports DEPS_INSTALL capability. @@ -482,22 +458,6 @@ class TestVenvManagerRemovePackage: should not run in parallel. """ - @pytest.mark.asyncio - async def test_remove_package_exists(self) -> None: - """VenvManager has remove_package() method. - - Contract: VenvManager.remove_package(venv, package) exists. - Breaks when: Method not implemented. - """ - from py_code_mode.execution.subprocess.config import SubprocessConfig - from py_code_mode.execution.subprocess.venv import VenvManager - - config = SubprocessConfig(python_version="3.12") - manager = VenvManager(config) - - assert hasattr(manager, "remove_package") - assert callable(manager.remove_package) - @pytest.mark.asyncio async def test_remove_package_uninstalls_from_venv(self, tmp_path: Path) -> None: """remove_package() uninstalls package from venv. @@ -582,30 +542,6 @@ async def test_remove_package_validates_package_spec(self, tmp_path: Path) -> No class TestSubprocessExecutorDepsMethodsContract: """Contract tests for SubprocessExecutor.install_deps() and uninstall_deps().""" - def test_executor_has_install_deps_method(self) -> None: - """SubprocessExecutor has install_deps() method. - - Contract: Executor protocol requires install_deps(packages) method. - Breaks when: Method not implemented. - """ - from py_code_mode.execution.subprocess import SubprocessExecutor - - executor = SubprocessExecutor() - assert hasattr(executor, "install_deps") - assert callable(executor.install_deps) - - def test_executor_has_uninstall_deps_method(self) -> None: - """SubprocessExecutor has uninstall_deps() method. - - Contract: Executor protocol requires uninstall_deps(packages) method. - Breaks when: Method not implemented. - """ - from py_code_mode.execution.subprocess import SubprocessExecutor - - executor = SubprocessExecutor() - assert hasattr(executor, "uninstall_deps") - assert callable(executor.uninstall_deps) - @pytest.mark.asyncio async def test_install_deps_returns_dict(self, tmp_path: Path) -> None: """install_deps() returns dict with expected keys. @@ -899,38 +835,6 @@ async def test_install_deps_handles_invalid_package_spec(self, tmp_path: Path) - # ============================================================================= -class TestDepsCapabilityConstants: - """Tests for DEPS_INSTALL and DEPS_UNINSTALL capability constants.""" - - def test_deps_install_capability_exists(self) -> None: - """DEPS_INSTALL capability constant exists. - - Contract: Capability.DEPS_INSTALL is defined. - Breaks when: Constant not added to Capability class. - """ - assert hasattr(Capability, "DEPS_INSTALL") - assert Capability.DEPS_INSTALL == "deps_install" - - def test_deps_uninstall_capability_exists(self) -> None: - """DEPS_UNINSTALL capability constant exists. - - Contract: Capability.DEPS_UNINSTALL is defined. - Breaks when: Constant not added to Capability class. - """ - assert hasattr(Capability, "DEPS_UNINSTALL") - assert Capability.DEPS_UNINSTALL == "deps_uninstall" - - def test_deps_capabilities_in_all_set(self) -> None: - """DEPS_INSTALL and DEPS_UNINSTALL in Capability.all() set. - - Contract: All capabilities returned by Capability.all(). - Breaks when: New capabilities not added to all() method. - """ - all_caps = Capability.all() - assert Capability.DEPS_INSTALL in all_caps - assert Capability.DEPS_UNINSTALL in all_caps - - # ============================================================================= # Session + Executor Integration Tests # ============================================================================= diff --git a/tests/test_executor_protocol.py b/tests/test_executor_protocol.py index 7d1fde8..4e6cb5e 100644 --- a/tests/test_executor_protocol.py +++ b/tests/test_executor_protocol.py @@ -1,86 +1,14 @@ -"""Tests for Executor protocol changes - Step 5 of implementation plan. +"""Executor/storage integration tests. -These tests verify: -1. Executor protocol defines start() method -2. All executors accept StorageBackend | None (not StorageAccess) -3. StorageBackendAccess class is deleted -4. Each executor correctly handles StorageBackend - -Written to FAIL initially (TDD RED phase). +These focus on behavior at the executor<->storage boundary, not protocol/shape +introspection (which is brittle and mostly redundant with behavioral tests). """ from pathlib import Path -from typing import get_type_hints from unittest.mock import AsyncMock, MagicMock, patch import pytest -# ============================================================================= -# Protocol Compliance Tests -# ============================================================================= - - -class TestExecutorProtocolDefinesStart: - """Verify the Executor protocol includes start() method.""" - - def test_protocol_has_start_method(self) -> None: - """Executor protocol must define start() method.""" - from py_code_mode.execution.protocol import Executor - - # Protocol should have start method - assert hasattr(Executor, "start"), "Executor protocol must define start() method" - - def test_protocol_start_accepts_storage_backend(self) -> None: - """Executor.start() must accept StorageBackend | None parameter.""" - from py_code_mode.execution.protocol import Executor - from py_code_mode.storage.backends import StorageBackend - - # Get type hints for start method, providing StorageBackend for forward reference - hints = get_type_hints(Executor.start, globalns={"StorageBackend": StorageBackend}) - - # Should have 'storage' parameter that accepts StorageBackend | None - assert "storage" in hints, "start() must have 'storage' parameter" - - # Check the type annotation includes StorageBackend - storage_type = hints["storage"] - # Handle Union types (StorageBackend | None) - storage_type_str = str(storage_type) - assert "StorageBackend" in storage_type_str, ( - f"start() storage parameter must accept StorageBackend, got {storage_type_str}" - ) - - def test_protocol_start_is_async(self) -> None: - """Executor.start() must be an async method.""" - import asyncio - - from py_code_mode.execution.protocol import Executor - - # start method should be a coroutine function - assert asyncio.iscoroutinefunction(Executor.start), "Executor.start() must be async" - - -class TestStorageBackendAccessDeleted: - """Verify StorageBackendAccess class is removed.""" - - def test_storage_backend_access_not_in_protocol(self) -> None: - """StorageBackendAccess should not exist in protocol module.""" - from py_code_mode.execution import protocol - - assert not hasattr(protocol, "StorageBackendAccess"), ( - "StorageBackendAccess should be deleted from protocol.py" - ) - - def test_storage_access_union_excludes_backend_access(self) -> None: - """StorageAccess type should not include StorageBackendAccess.""" - from py_code_mode.execution.protocol import StorageAccess - - # StorageAccess should only be FileStorageAccess | RedisStorageAccess - storage_access_str = str(StorageAccess) - assert "StorageBackendAccess" not in storage_access_str, ( - f"StorageAccess should not include StorageBackendAccess: {storage_access_str}" - ) - - # ============================================================================= # InProcessExecutor Tests # ============================================================================= @@ -89,36 +17,6 @@ def test_storage_access_union_excludes_backend_access(self) -> None: class TestInProcessExecutorAcceptsStorageBackend: """InProcessExecutor.start() must accept StorageBackend directly.""" - def test_start_accepts_storage_backend(self, tmp_path: Path) -> None: - """InProcessExecutor.start() accepts StorageBackend parameter.""" - from py_code_mode.execution.in_process import InProcessExecutor - from py_code_mode.storage.backends import FileStorage - - storage = FileStorage(tmp_path) - executor = InProcessExecutor() - - # This should NOT raise - executor accepts StorageBackend - # Type checker would fail if signature is still StorageAccess - import asyncio - - asyncio.run(executor.start(storage=storage)) - - def test_start_parameter_named_storage_not_storage_access(self) -> None: - """InProcessExecutor.start() parameter must be named 'storage' not 'storage_access'.""" - import inspect - - from py_code_mode.execution.in_process import InProcessExecutor - - sig = inspect.signature(InProcessExecutor.start) - param_names = list(sig.parameters.keys()) - - assert "storage" in param_names, ( - f"start() must have 'storage' parameter, got: {param_names}" - ) - assert "storage_access" not in param_names, ( - "start() should NOT have 'storage_access' parameter (old name)" - ) - @pytest.mark.asyncio async def test_uses_executor_config_for_tools(self, tmp_path: Path) -> None: """InProcessExecutor uses config.tools_path for tools. @@ -266,22 +164,6 @@ async def test_rejects_redis_storage_access(self) -> None: class TestContainerExecutorAcceptsStorageBackend: """ContainerExecutor.start() must accept StorageBackend directly.""" - def test_start_parameter_named_storage(self) -> None: - """ContainerExecutor.start() parameter must be named 'storage'.""" - import inspect - - from py_code_mode.execution.container import ContainerExecutor - - sig = inspect.signature(ContainerExecutor.start) - param_names = list(sig.parameters.keys()) - - assert "storage" in param_names, ( - f"start() must have 'storage' parameter, got: {param_names}" - ) - assert "storage_access" not in param_names, ( - "start() should NOT have 'storage_access' parameter (old name)" - ) - @pytest.mark.asyncio async def test_calls_get_serializable_access_for_file_storage(self, tmp_path: Path) -> None: """ContainerExecutor calls storage.get_serializable_access() for FileStorage.""" @@ -408,22 +290,6 @@ async def test_rejects_file_storage_access(self, tmp_path: Path) -> None: class TestSubprocessExecutorAcceptsStorageBackend: """SubprocessExecutor.start() must accept StorageBackend directly.""" - def test_start_parameter_named_storage(self) -> None: - """SubprocessExecutor.start() parameter must be named 'storage'.""" - import inspect - - from py_code_mode.execution.subprocess import SubprocessExecutor - - sig = inspect.signature(SubprocessExecutor.start) - param_names = list(sig.parameters.keys()) - - assert "storage" in param_names, ( - f"start() must have 'storage' parameter, got: {param_names}" - ) - assert "storage_access" not in param_names, ( - "start() should NOT have 'storage_access' parameter (old name)" - ) - @pytest.mark.asyncio async def test_calls_get_serializable_access(self, tmp_path: Path) -> None: """SubprocessExecutor calls storage.get_serializable_access(). @@ -513,50 +379,3 @@ async def test_rejects_file_storage_access(self, tmp_path: Path) -> None: # ============================================================================= # Cross-Executor Consistency Tests # ============================================================================= - - -class TestAllExecutorsHaveConsistentStartSignature: - """All executors must have identical start() signatures.""" - - def test_all_executors_have_storage_parameter(self) -> None: - """All executor start() methods have 'storage' parameter.""" - import inspect - - from py_code_mode.execution.container import ContainerExecutor - from py_code_mode.execution.in_process import InProcessExecutor - from py_code_mode.execution.subprocess import SubprocessExecutor - - executors = [ - ("InProcessExecutor", InProcessExecutor), - ("ContainerExecutor", ContainerExecutor), - ("SubprocessExecutor", SubprocessExecutor), - ] - - for name, executor_cls in executors: - sig = inspect.signature(executor_cls.start) - param_names = list(sig.parameters.keys()) - assert "storage" in param_names, ( - f"{name}.start() must have 'storage' parameter, got: {param_names}" - ) - - def test_all_executors_storage_defaults_to_none(self) -> None: - """All executor start() methods have storage default to None.""" - import inspect - - from py_code_mode.execution.container import ContainerExecutor - from py_code_mode.execution.in_process import InProcessExecutor - from py_code_mode.execution.subprocess import SubprocessExecutor - - executors = [ - ("InProcessExecutor", InProcessExecutor), - ("ContainerExecutor", ContainerExecutor), - ("SubprocessExecutor", SubprocessExecutor), - ] - - for name, executor_cls in executors: - sig = inspect.signature(executor_cls.start) - storage_param = sig.parameters.get("storage") - assert storage_param is not None, f"{name}.start() missing storage parameter" - assert storage_param.default is None, ( - f"{name}.start() storage must default to None, got: {storage_param.default}" - ) diff --git a/tests/test_mcp_adapter.py b/tests/test_mcp_adapter.py index 46f1827..da27bdb 100644 --- a/tests/test_mcp_adapter.py +++ b/tests/test_mcp_adapter.py @@ -195,23 +195,6 @@ async def test_call_tool_handles_error_response(self, adapter, mock_session) -> class TestMCPAdapterConnection: """Tests for MCP server connection management.""" - @pytest.mark.asyncio - async def test_connect_to_stdio_server(self) -> None: - """Can connect to MCP server via stdio.""" - from py_code_mode.tools.adapters.mcp import MCPAdapter - - # This test verifies the factory method exists - # Actual connection would require a real server - assert hasattr(MCPAdapter, "connect_stdio") - - @pytest.mark.asyncio - async def test_connect_to_sse_server(self) -> None: - """Can connect to MCP server via SSE transport.""" - from py_code_mode.tools.adapters.mcp import MCPAdapter - - # This test verifies the factory method exists - assert hasattr(MCPAdapter, "connect_sse") - @pytest.mark.asyncio async def test_close_cleans_up(self) -> None: """close() cleans up resources.""" @@ -317,14 +300,6 @@ async def test_connect_sse_initializes_session(self) -> None: mock_session.initialize.assert_called_once() - @pytest.mark.asyncio - async def test_connect_sse_method_exists(self) -> None: - """connect_sse method exists on MCPAdapter.""" - from py_code_mode.tools.adapters.mcp import MCPAdapter - - assert hasattr(MCPAdapter, "connect_sse") - assert callable(MCPAdapter.connect_sse) - class TestMCPAdapterNamespacing: """Tests for MCP tool namespacing - tools grouped under namespace like CLI tools.""" diff --git a/tests/test_rpc_errors.py b/tests/test_rpc_errors.py index 97a1918..c302d62 100644 --- a/tests/test_rpc_errors.py +++ b/tests/test_rpc_errors.py @@ -312,7 +312,7 @@ async def test_workflow_error_is_namespace_error(self, executor_with_storage) -> result = await executor_with_storage.run("issubclass(WorkflowError, NamespaceError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_tool_error_is_namespace_error(self, executor_with_storage) -> None: @@ -323,7 +323,7 @@ async def test_tool_error_is_namespace_error(self, executor_with_storage) -> Non result = await executor_with_storage.run("issubclass(ToolError, NamespaceError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_artifact_error_is_namespace_error(self, executor_with_storage) -> None: @@ -334,7 +334,7 @@ async def test_artifact_error_is_namespace_error(self, executor_with_storage) -> result = await executor_with_storage.run("issubclass(ArtifactError, NamespaceError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_error_is_namespace_error(self, executor_with_storage) -> None: @@ -345,7 +345,7 @@ async def test_deps_error_is_namespace_error(self, executor_with_storage) -> Non result = await executor_with_storage.run("issubclass(DepsError, NamespaceError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_namespace_error_is_rpc_error(self, executor_with_storage) -> None: @@ -356,7 +356,7 @@ async def test_namespace_error_is_rpc_error(self, executor_with_storage) -> None result = await executor_with_storage.run("issubclass(NamespaceError, RPCError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_rpc_error_is_exception(self, executor_with_storage) -> None: @@ -367,7 +367,7 @@ async def test_rpc_error_is_exception(self, executor_with_storage) -> None: result = await executor_with_storage.run("issubclass(RPCError, Exception)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True # ============================================================================= @@ -451,7 +451,7 @@ async def test_rpc_transport_error_is_rpc_error(self, executor_with_storage) -> result = await executor_with_storage.run("issubclass(RPCTransportError, RPCError)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_non_dict_error_raises_rpc_transport_error(self, executor_with_storage) -> None: diff --git a/tests/test_semantic.py b/tests/test_semantic.py index f59e82d..e62f0fb 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -16,19 +16,6 @@ def _make_workflow(name: str, description: str, code: str) -> PythonWorkflow: class TestEmbeddingProviderProtocol: """Tests that define the EmbeddingProvider interface.""" - def test_provider_has_embed_method(self) -> None: - """Provider must have embed() that returns vectors.""" - from py_code_mode.workflows import EmbeddingProvider - - # Protocol should define embed method - assert hasattr(EmbeddingProvider, "embed") - - def test_provider_has_dimension_property(self) -> None: - """Provider exposes embedding dimension for index allocation.""" - from py_code_mode.workflows import EmbeddingProvider - - assert hasattr(EmbeddingProvider, "dimension") - def test_embed_returns_list_of_vectors(self) -> None: """embed() takes list of strings, returns list of float vectors.""" from py_code_mode.workflows import MockEmbedder @@ -93,8 +80,6 @@ def test_batch_embedding(self, embedder) -> None: def test_detects_device(self, embedder) -> None: """Uses MPS on Apple Silicon, CUDA if available, else CPU.""" - # Just verify it has a device attribute - assert hasattr(embedder, "device") assert embedder.device in ("mps", "cuda", "cpu") diff --git a/tests/test_session.py b/tests/test_session.py index c10c417..577e655 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -98,14 +98,14 @@ def storage(self, tmp_path: Path) -> FileStorage: @pytest.mark.asyncio async def test_run_returns_execution_result(self, storage: FileStorage) -> None: - """run() returns an ExecutionResult.""" + """run() returns an ExecutionResult. + + Covered by tests that assert specific `ExecutionResult` behavior (value/stdout/error). + """ async with Session(storage=storage) as session: result = await session.run("42") - - assert hasattr(result, "is_ok") - assert hasattr(result, "value") - assert hasattr(result, "error") - assert hasattr(result, "stdout") + assert result.error is None + assert result.value == 42 @pytest.mark.asyncio async def test_run_evaluates_expression(self, storage: FileStorage) -> None: @@ -435,81 +435,16 @@ def storage(self, tmp_path: Path) -> FileStorage: async def test_session_supports_method(self, storage: FileStorage) -> None: """Session has supports() method for capability queries.""" async with Session(storage=storage) as session: - # Should have this method - assert hasattr(session, "supports") - assert callable(session.supports) + # Calls through to the active executor. + assert isinstance(session.supports("timeout"), bool) @pytest.mark.asyncio async def test_session_supported_capabilities(self, storage: FileStorage) -> None: """Session has supported_capabilities() method.""" async with Session(storage=storage) as session: - assert hasattr(session, "supported_capabilities") caps = session.supported_capabilities() assert isinstance(caps, set) - - -# ============================================================================= -# StorageAccess Type Tests -# ============================================================================= - - -class TestStorageAccessTypes: - """Tests for StorageAccess type definitions.""" - - def test_file_storage_access_exists(self) -> None: - """FileStorageAccess type is importable.""" - from py_code_mode.execution.protocol import FileStorageAccess - - assert FileStorageAccess is not None - - def test_redis_storage_access_exists(self) -> None: - """RedisStorageAccess type is importable.""" - from py_code_mode.execution.protocol import RedisStorageAccess - - assert RedisStorageAccess is not None - - def test_file_storage_access_has_paths(self) -> None: - """FileStorageAccess has workflows_path, artifacts_path. - - NOTE: tools_path and deps_path removed - tools/deps now owned by executors. - """ - from py_code_mode.execution.protocol import FileStorageAccess - - access = FileStorageAccess( - workflows_path=Path("/tmp/workflows"), - artifacts_path=Path("/tmp/artifacts"), - ) - assert access.workflows_path == Path("/tmp/workflows") - assert access.artifacts_path == Path("/tmp/artifacts") - - def test_file_storage_access_paths_optional(self) -> None: - """FileStorageAccess allows None for workflows_path. - - NOTE: tools_path and deps_path removed - tools/deps now owned by executors. - """ - from py_code_mode.execution.protocol import FileStorageAccess - - access = FileStorageAccess( - workflows_path=None, - artifacts_path=Path("/tmp/artifacts"), - ) - assert access.workflows_path is None - - def test_redis_storage_access_has_url_and_prefixes(self) -> None: - """RedisStorageAccess has redis_url and prefix fields. - - NOTE: tools_prefix and deps_prefix removed - tools/deps now owned by executors. - """ - from py_code_mode.execution.protocol import RedisStorageAccess - - access = RedisStorageAccess( - redis_url="redis://localhost:6379", - workflows_prefix="app:workflows", - artifacts_prefix="app:artifacts", - ) - assert access.redis_url == "redis://localhost:6379" - assert access.workflows_prefix == "app:workflows" - assert access.artifacts_prefix == "app:artifacts" + assert "timeout" in caps # ============================================================================= @@ -931,17 +866,17 @@ async def run(a: int, b: int) -> int: # Verify tools namespace exists result = await session.run("'tools' in dir()") assert result.is_ok, f"Failed to check tools: {result.error}" - assert result.value in (True, "True"), "tools namespace not found" + assert result.value is True, "tools namespace not found" # Verify workflows namespace exists result = await session.run("'workflows' in dir()") assert result.is_ok, f"Failed to check workflows: {result.error}" - assert result.value in (True, "True"), "workflows namespace not found" + assert result.value is True, "workflows namespace not found" # Verify artifacts namespace exists result = await session.run("'artifacts' in dir()") assert result.is_ok, f"Failed to check artifacts: {result.error}" - assert result.value in (True, "True"), "artifacts namespace not found" + assert result.value is True, "artifacts namespace not found" # Verify workflows.list() works and contains our workflow result = await session.run("workflows.list()") @@ -972,9 +907,9 @@ async def test_subprocess_capabilities_via_session(self, tmp_path: Path) -> None async with Session(storage=storage, executor=executor) as session: # SubprocessExecutor advertises PROCESS_ISOLATION - assert session.supports(Capability.PROCESS_ISOLATION), ( - "SubprocessExecutor should support PROCESS_ISOLATION" - ) + assert session.supports( + Capability.PROCESS_ISOLATION + ), "SubprocessExecutor should support PROCESS_ISOLATION" # SubprocessExecutor supports TIMEOUT assert session.supports(Capability.TIMEOUT), "SubprocessExecutor should support TIMEOUT" @@ -983,14 +918,14 @@ async def test_subprocess_capabilities_via_session(self, tmp_path: Path) -> None assert session.supports(Capability.RESET), "SubprocessExecutor should support RESET" # SubprocessExecutor does NOT support NETWORK_ISOLATION - assert not session.supports(Capability.NETWORK_ISOLATION), ( - "SubprocessExecutor should NOT support NETWORK_ISOLATION" - ) + assert not session.supports( + Capability.NETWORK_ISOLATION + ), "SubprocessExecutor should NOT support NETWORK_ISOLATION" # SubprocessExecutor does NOT support FILESYSTEM_ISOLATION - assert not session.supports(Capability.FILESYSTEM_ISOLATION), ( - "SubprocessExecutor should NOT support FILESYSTEM_ISOLATION" - ) + assert not session.supports( + Capability.FILESYSTEM_ISOLATION + ), "SubprocessExecutor should NOT support FILESYSTEM_ISOLATION" # supported_capabilities() returns the full set caps = session.supported_capabilities() @@ -1027,9 +962,9 @@ async def test_process_isolation_implies_serializable_access(self, tmp_path: Pat executor = SubprocessExecutor(config=config) # Verify the invariant: PROCESS_ISOLATION capability exists - assert executor.supports(Capability.PROCESS_ISOLATION), ( - "SubprocessExecutor must have PROCESS_ISOLATION capability" - ) + assert executor.supports( + Capability.PROCESS_ISOLATION + ), "SubprocessExecutor must have PROCESS_ISOLATION capability" # Start and verify it can access storage-based namespaces # (this implicitly verifies serializable access works) @@ -1038,6 +973,6 @@ async def test_process_isolation_implies_serializable_access(self, tmp_path: Pat # the subprocess won't be able to reconstruct the storage result = await session.run("'artifacts' in dir()") assert result.is_ok, f"Failed: {result.error}" - assert result.value in (True, "True"), ( - "artifacts namespace not available - serializable access likely broken" - ) + assert ( + result.value is True + ), "artifacts namespace not available - serializable access likely broken" diff --git a/tests/test_skill_library_vector_store.py b/tests/test_skill_library_vector_store.py index 2222c1b..50b96f8 100644 --- a/tests/test_skill_library_vector_store.py +++ b/tests/test_skill_library_vector_store.py @@ -595,19 +595,6 @@ def test_mock_vector_store_implements_protocol(self) -> None: # Should pass isinstance check assert isinstance(mock, VectorStore) - def test_mock_vector_store_has_all_required_methods(self) -> None: - """Verify MockVectorStore has all VectorStore methods.""" - mock = MockVectorStore() - - # All protocol methods should exist - assert hasattr(mock, "add") - assert hasattr(mock, "remove") - assert hasattr(mock, "search") - assert hasattr(mock, "get_content_hash") - assert hasattr(mock, "get_model_info") - assert hasattr(mock, "clear") - assert hasattr(mock, "count") - class TestWarmStartupCaching: """Test that vector_store caching works across library instances.""" diff --git a/tests/test_skill_store.py b/tests/test_skill_store.py index 798fb6a..b9f7426 100644 --- a/tests/test_skill_store.py +++ b/tests/test_skill_store.py @@ -286,9 +286,10 @@ def test_save_and_load_python_workflow( assert loaded is not None assert loaded.name == "greet" assert loaded.description == "Greet someone" - # Stored workflows have source and can invoke - duck typing - assert hasattr(loaded, "source") - assert hasattr(loaded, "invoke") + # Stored workflows have source and can invoke. + assert isinstance(loaded.source, str) + assert loaded.source + assert callable(loaded.invoke) def test_load_nonexistent_returns_none(self, redis_store: RedisWorkflowStore): """Should return None for nonexistent workflow.""" diff --git a/tests/test_storage.py b/tests/test_storage.py index ffefa41..663a022 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -263,8 +263,8 @@ def test_method_exists_on_file_storage(self, tmp_path: Path) -> None: """ storage = FileStorage(tmp_path) - assert hasattr(storage, "get_serializable_access") - assert callable(storage.get_serializable_access) + # Covered by tests that call get_serializable_access() and assert return types. + assert storage.get_serializable_access() is not None def test_method_exists_on_redis_storage(self, mock_redis: MockRedisClient) -> None: """RedisStorage has get_serializable_access method. @@ -273,8 +273,7 @@ def test_method_exists_on_redis_storage(self, mock_redis: MockRedisClient) -> No """ storage = RedisStorage(redis=mock_redis, prefix="test") - assert hasattr(storage, "get_serializable_access") - assert callable(storage.get_serializable_access) + assert storage.get_serializable_access() is not None # NOTE: TestStorageBackendExecutionMethods was removed in the executor-ownership refactor. diff --git a/tests/test_storage_vector_store.py b/tests/test_storage_vector_store.py index af93e0e..5f34200 100644 --- a/tests/test_storage_vector_store.py +++ b/tests/test_storage_vector_store.py @@ -35,15 +35,6 @@ class TestFileStorageVectorStoreIntegration: """Tests for FileStorage vector store integration.""" - def test_get_vector_store_method_exists(self, tmp_path: Path) -> None: - """FileStorage has get_vector_store() method. - - Breaks when: Method doesn't exist on FileStorage. - """ - storage = FileStorage(tmp_path) - assert hasattr(storage, "get_vector_store") - assert callable(storage.get_vector_store) - def test_get_vector_store_returns_chroma_when_available(self, tmp_path: Path) -> None: """get_vector_store() returns ChromaVectorStore when chromadb installed. @@ -55,11 +46,10 @@ def test_get_vector_store_returns_chroma_when_available(self, tmp_path: Path) -> # Should return ChromaVectorStore if chromadb available assert vector_store is not None - # Check it has VectorStore protocol methods - assert hasattr(vector_store, "add") - assert hasattr(vector_store, "remove") - assert hasattr(vector_store, "search") - assert hasattr(vector_store, "count") + # Runtime-checkable protocol: ensure it satisfies VectorStore. + from py_code_mode.workflows.vector_store import VectorStore + + assert isinstance(vector_store, VectorStore) def test_get_vector_store_uses_correct_path(self, tmp_path: Path) -> None: """get_vector_store() uses {base_path}/vectors/ directory. @@ -114,18 +104,6 @@ def test_get_vector_store_creates_embedder(self, tmp_path: Path) -> None: class TestFileStorageWorkflowLibraryVectorStoreIntegration: """Tests for WorkflowLibrary receiving vector_store from FileStorage.""" - def test_workflow_library_has_vector_store_attribute(self, tmp_path: Path) -> None: - """WorkflowLibrary created by FileStorage has vector_store attribute. - - Breaks when: create_workflow_library() not called with vector_store parameter. - """ - storage = FileStorage(tmp_path) - - library = storage.get_workflow_library() - - # Should have vector_store attribute - assert hasattr(library, "vector_store") - def test_workflow_library_vector_store_matches_get_vector_store(self, tmp_path: Path) -> None: """WorkflowLibrary.vector_store is same instance as storage.get_vector_store(). @@ -173,15 +151,6 @@ def test_workflow_library_uses_vector_store_for_search(self, tmp_path: Path) -> class TestRedisStorageVectorStorePlaceholder: """Tests for RedisStorage vector store integration (Phase 6 placeholder).""" - def test_get_vector_store_method_exists(self, mock_redis: MockRedisClient) -> None: - """RedisStorage has get_vector_store() method. - - Breaks when: Method doesn't exist on RedisStorage. - """ - storage = RedisStorage(redis=mock_redis, prefix="test") - assert hasattr(storage, "get_vector_store") - assert callable(storage.get_vector_store) - def test_get_vector_store_returns_none_for_now(self, mock_redis: MockRedisClient) -> None: """get_vector_store() returns None (RedisVectorStore not implemented yet). @@ -204,15 +173,13 @@ class TestFileStorageAccessVectorsPath: """Tests for vectors_path field in FileStorageAccess.""" def test_file_storage_access_has_vectors_path_field(self, tmp_path: Path) -> None: - """FileStorageAccess has vectors_path field. - - Breaks when: Field doesn't exist in dataclass definition. - """ + """FileStorageAccess has vectors_path field (cross-process config).""" storage = FileStorage(tmp_path) access = storage.get_serializable_access() - assert hasattr(access, "vectors_path") + # Attribute access is the behavior: this will raise AttributeError if missing. + _ = access.vectors_path def test_vectors_path_is_optional(self, tmp_path: Path) -> None: """vectors_path can be None when vector store unavailable. @@ -225,8 +192,7 @@ def test_vectors_path_is_optional(self, tmp_path: Path) -> None: with patch.object(storage, "get_vector_store", return_value=None): access = storage.get_serializable_access() - # Should be None when vector store not available - # (or set to path if chromadb is available) + # Should be None when vector store not available (or set to a Path if available). assert access.vectors_path is None or isinstance(access.vectors_path, Path) def test_vectors_path_points_to_vectors_directory(self, tmp_path: Path) -> None: @@ -266,15 +232,12 @@ class TestRedisStorageAccessVectorsPrefixPlaceholder: def test_redis_storage_access_has_vectors_prefix_field( self, mock_redis: MockRedisClient ) -> None: - """RedisStorageAccess has vectors_prefix field. - - Breaks when: Field doesn't exist in dataclass definition. - """ + """RedisStorageAccess has vectors_prefix field (placeholder for Phase 6).""" storage = RedisStorage(redis=mock_redis, prefix="test") access = storage.get_serializable_access() - assert hasattr(access, "vectors_prefix") + _ = access.vectors_prefix def test_vectors_prefix_is_optional(self, mock_redis: MockRedisClient) -> None: """vectors_prefix can be None when RedisVectorStore not implemented. @@ -285,7 +248,7 @@ def test_vectors_prefix_is_optional(self, mock_redis: MockRedisClient) -> None: access = storage.get_serializable_access() - # Should be None until Phase 6 implements RedisVectorStore + # Should be None until Phase 6 implements RedisVectorStore. assert access.vectors_prefix is None or isinstance(access.vectors_prefix, str) def test_vectors_prefix_follows_pattern_when_implemented( diff --git a/tests/test_storage_wrapper_cleanup.py b/tests/test_storage_wrapper_cleanup.py index e327995..39e4259 100644 --- a/tests/test_storage_wrapper_cleanup.py +++ b/tests/test_storage_wrapper_cleanup.py @@ -256,33 +256,6 @@ class TestStorageBackendProtocolSimplified: # NOTE: test_storage_backend_has_get_tool_registry removed # tools are now owned by executors, not storage - def test_storage_backend_has_get_workflow_library(self) -> None: - """StorageBackend protocol must have get_workflow_library method. - - Breaks when: Method is missing from protocol. - """ - from py_code_mode.storage.backends import StorageBackend - - assert hasattr(StorageBackend, "get_workflow_library") - - def test_storage_backend_has_get_artifact_store(self) -> None: - """StorageBackend protocol must have get_artifact_store method. - - Breaks when: Method is missing from protocol. - """ - from py_code_mode.storage.backends import StorageBackend - - assert hasattr(StorageBackend, "get_artifact_store") - - def test_storage_backend_has_get_serializable_access(self) -> None: - """StorageBackend protocol must have get_serializable_access method. - - Breaks when: Method is missing from protocol. - """ - from py_code_mode.storage.backends import StorageBackend - - assert hasattr(StorageBackend, "get_serializable_access") - def test_storage_backend_no_tools_property(self) -> None: """StorageBackend protocol must NOT have tools property. diff --git a/tests/test_subprocess_executor.py b/tests/test_subprocess_executor.py index bcbf47d..492176b 100644 --- a/tests/test_subprocess_executor.py +++ b/tests/test_subprocess_executor.py @@ -8,7 +8,7 @@ from py_code_mode.execution.protocol import Capability from py_code_mode.execution.subprocess.config import SubprocessConfig -from py_code_mode.execution.subprocess.venv import KernelVenv, VenvManager +from py_code_mode.execution.subprocess.venv import VenvManager class TestSubprocessConfig: @@ -214,76 +214,6 @@ def test_valid_python_version_3_10(self) -> None: # ============================================================================= -class TestKernelVenv: - """Tests for KernelVenv dataclass structure.""" - - # ========================================================================= - # Field Existence - # ========================================================================= - - def test_has_path_field(self, tmp_path: Path) -> None: - """KernelVenv has path field.""" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=tmp_path / "venv" / "bin" / "python", - kernel_spec_name="test-kernel", - ) - assert hasattr(venv, "path") - assert venv.path == tmp_path / "venv" - - def test_has_python_path_field(self, tmp_path: Path) -> None: - """KernelVenv has python_path field.""" - python_path = tmp_path / "venv" / "bin" / "python" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=python_path, - kernel_spec_name="test-kernel", - ) - assert hasattr(venv, "python_path") - assert venv.python_path == python_path - - def test_has_kernel_spec_name_field(self, tmp_path: Path) -> None: - """KernelVenv has kernel_spec_name field.""" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=tmp_path / "venv" / "bin" / "python", - kernel_spec_name="my-kernel-spec", - ) - assert hasattr(venv, "kernel_spec_name") - assert venv.kernel_spec_name == "my-kernel-spec" - - # ========================================================================= - # Type Correctness - # ========================================================================= - - def test_path_is_path_type(self, tmp_path: Path) -> None: - """path field is Path type.""" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=tmp_path / "venv" / "bin" / "python", - kernel_spec_name="test-kernel", - ) - assert isinstance(venv.path, Path) - - def test_python_path_is_path_type(self, tmp_path: Path) -> None: - """python_path field is Path type.""" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=tmp_path / "venv" / "bin" / "python", - kernel_spec_name="test-kernel", - ) - assert isinstance(venv.python_path, Path) - - def test_kernel_spec_name_is_str_type(self, tmp_path: Path) -> None: - """kernel_spec_name field is str type.""" - venv = KernelVenv( - path=tmp_path / "venv", - python_path=tmp_path / "venv" / "bin" / "python", - kernel_spec_name="test-kernel", - ) - assert isinstance(venv.kernel_spec_name, str) - - # ============================================================================= # VenvManager Initialization Tests # ============================================================================= @@ -1095,13 +1025,13 @@ async def test_start_with_storage_injects_namespaces(self, tmp_path: Path) -> No # Verify namespaces are injected result = await executor.run("'tools' in dir()") - assert result.value in (True, "True") + assert result.value is True result = await executor.run("'workflows' in dir()") - assert result.value in (True, "True") + assert result.value is True result = await executor.run("'artifacts' in dir()") - assert result.value in (True, "True") + assert result.value is True finally: await executor.close() @@ -1566,19 +1496,19 @@ async def test_reset_preserves_injected_namespaces(self, executor) -> None: """reset() preserves tools, workflows, artifacts namespaces.""" # Verify namespaces exist before reset result = await executor.run("'tools' in dir()") - assert result.value in (True, "True") + assert result.value is True await executor.reset() # Namespaces should still exist after reset result = await executor.run("'tools' in dir()") - assert result.value in (True, "True") + assert result.value is True result = await executor.run("'workflows' in dir()") - assert result.value in (True, "True") + assert result.value is True result = await executor.run("'artifacts' in dir()") - assert result.value in (True, "True") + assert result.value is True # ============================================================================= @@ -1736,7 +1666,7 @@ async def test_deps_namespace_is_available(self, executor_with_storage) -> None: result = await executor_with_storage.run("'deps' in dir()") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_list_returns_list(self, executor_with_storage) -> None: @@ -1761,7 +1691,7 @@ async def test_deps_has_add_method(self, executor_with_storage) -> None: result = await executor_with_storage.run("callable(deps.add)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_has_remove_method(self, executor_with_storage) -> None: @@ -1773,7 +1703,7 @@ async def test_deps_has_remove_method(self, executor_with_storage) -> None: result = await executor_with_storage.run("callable(deps.remove)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_has_sync_method(self, executor_with_storage) -> None: @@ -1785,7 +1715,7 @@ async def test_deps_has_sync_method(self, executor_with_storage) -> None: result = await executor_with_storage.run("callable(deps.sync)") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_repr_shows_package_count(self, executor_with_storage) -> None: @@ -1808,14 +1738,14 @@ async def test_reset_preserves_deps_namespace(self, executor_with_storage) -> No """ # Verify deps exists before reset result = await executor_with_storage.run("'deps' in dir()") - assert result.value in (True, "True") + assert result.value is True await executor_with_storage.reset() # deps should still exist after reset result = await executor_with_storage.run("'deps' in dir()") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.slow @@ -1906,7 +1836,7 @@ async def test_deps_add_persists_to_store(self, executor_allow_deps) -> None: # Verify it's in the list result = await executor_allow_deps.run("'requests' in str(deps.list())") assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_deps_add_raises_when_runtime_deps_disabled(self, executor_deny_deps) -> None: diff --git a/tests/test_subprocess_namespace_injection.py b/tests/test_subprocess_namespace_injection.py index fd4c8cb..0fc39c9 100644 --- a/tests/test_subprocess_namespace_injection.py +++ b/tests/test_subprocess_namespace_injection.py @@ -203,7 +203,7 @@ async def test_tools_list_returns_tool_objects(self, executor_with_storage) -> N # Verify it returns a list check_result = await executor_with_storage.run("isinstance(tools.list(), list)") - assert check_result.value in (True, "True") + assert check_result.value is True # Verify tools have expected keys (returned as dicts, not objects) result = await executor_with_storage.run("[t['name'] for t in tools.list()]") @@ -318,8 +318,7 @@ async def test_workflows_create_persists(self, executor_empty_storage) -> None: "'multiply' in [s['name'] if isinstance(s, dict) else s.name for s in workflows.list()]" ) assert result.error is None - # Accept True as bool or string - assert result.value in (True, "True"), f"Workflow not found in list: {result.value}" + assert result.value is True, f"Workflow not found in list: {result.value}" @pytest.mark.asyncio async def test_workflows_invoke_executes_workflow(self, executor_empty_storage) -> None: @@ -438,7 +437,7 @@ async def test_artifacts_save_persists_data(self, executor_empty_storage) -> Non # Verify exists result = await executor_empty_storage.run('artifacts.exists("test_data")') assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_artifacts_load_retrieves_data(self, executor_empty_storage) -> None: @@ -479,7 +478,7 @@ async def test_artifacts_exists_checks_presence(self, executor_empty_storage) -> # Should not exist result = await executor_empty_storage.run('artifacts.exists("nonexistent")') assert result.error is None - assert result.value in (False, "False") + assert result.value is False # Save it await executor_empty_storage.run('artifacts.save("now_exists", "data")') @@ -487,7 +486,7 @@ async def test_artifacts_exists_checks_presence(self, executor_empty_storage) -> # Should exist result = await executor_empty_storage.run('artifacts.exists("now_exists")') assert result.error is None - assert result.value in (True, "True") + assert result.value is True # ============================================================================= @@ -529,7 +528,7 @@ async def test_namespaces_use_storage_paths(self, tmp_path: Path, echo_tool_yaml # Tools should be loaded from tools_path (dicts, not objects) result = await executor.run("'echo' in [t['name'] for t in tools.list()]") assert result.error is None - assert result.value in (True, "True") + assert result.value is True finally: await executor.close() @@ -559,7 +558,7 @@ async def test_namespace_state_persists_between_runs(self, executor_empty_storag "'counter' in str(workflows.list()) and 'state_test' in str(artifacts.list())" ) assert result.error is None - assert result.value in (True, "True") + assert result.value is True @pytest.mark.asyncio async def test_namespace_state_preserved_after_reset(self, tmp_path: Path) -> None: @@ -593,14 +592,14 @@ async def test_namespace_state_preserved_after_reset(self, tmp_path: Path) -> No # Namespaces should still be accessible result = await executor.run("'tools' in dir() and 'workflows' in dir()") - assert result.value in (True, "True") + assert result.value is True # Persisted data should still be there (it's in storage, not kernel memory) result = await executor.run("'persist' in str(workflows.list())") - assert result.value in (True, "True") + assert result.value is True result = await executor.run("'persist_artifact' in str(artifacts.list())") - assert result.value in (True, "True") + assert result.value is True finally: await executor.close() @@ -637,7 +636,7 @@ async def test_namespaces_available_immediately_after_start( "'tools' in dir() and 'workflows' in dir() and 'artifacts' in dir()" ) assert result.error is None - assert result.value in (True, "True") + assert result.value is True finally: await executor.close() @@ -653,7 +652,7 @@ async def test_namespace_objects_are_stable(self, executor_empty_storage) -> Non # Check ID is same in next run result = await executor_empty_storage.run("id(tools) == _tools_id") assert result.error is None - assert result.value in (True, "True") + assert result.value is True # ============================================================================= @@ -806,9 +805,9 @@ def test_redis_storage_code_imports_redis(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "from redis import Redis" in code or "import redis" in code, ( - "Generated code must import Redis client" - ) + assert ( + "from redis import Redis" in code or "import redis" in code + ), "Generated code must import Redis client" def test_redis_storage_code_uses_provided_url(self) -> None: """Generated code should use the exact redis_url provided. @@ -855,12 +854,12 @@ def test_redis_storage_code_uses_provided_prefixes(self) -> None: code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" # NOTE: tools_prefix assertion removed - tools now owned by executors - assert workflows_prefix in code, ( - f"Generated code should contain workflows_prefix: {workflows_prefix}" - ) - assert artifacts_prefix in code, ( - f"Generated code should contain artifacts_prefix: {artifacts_prefix}" - ) + assert ( + workflows_prefix in code + ), f"Generated code should contain workflows_prefix: {workflows_prefix}" + assert ( + artifacts_prefix in code + ), f"Generated code should contain artifacts_prefix: {artifacts_prefix}" def test_redis_storage_code_sets_up_tools(self) -> None: """Generated code should set up tools namespace with empty registry. @@ -879,9 +878,9 @@ def test_redis_storage_code_sets_up_tools(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "tools = " in code or "tools=" in code, ( - "Generated code should assign tools namespace" - ) + assert ( + "tools = " in code or "tools=" in code + ), "Generated code should assign tools namespace" # Tools are now owned by executor, not storage - empty registry is created assert "ToolRegistry()" in code, "Generated code should create empty ToolRegistry" @@ -901,12 +900,12 @@ def test_redis_storage_code_sets_up_workflows(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "workflows = " in code or "workflows=" in code, ( - "Generated code should assign workflows namespace" - ) - assert "RedisWorkflowStore" in code, ( - "Generated code should use RedisWorkflowStore for workflows" - ) + assert ( + "workflows = " in code or "workflows=" in code + ), "Generated code should assign workflows namespace" + assert ( + "RedisWorkflowStore" in code + ), "Generated code should use RedisWorkflowStore for workflows" def test_redis_storage_code_sets_up_artifacts(self) -> None: """Generated code should set up artifacts namespace with RedisArtifactStore. @@ -924,12 +923,12 @@ def test_redis_storage_code_sets_up_artifacts(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "artifacts = " in code or "artifacts=" in code, ( - "Generated code should assign artifacts namespace" - ) - assert "RedisArtifactStore" in code, ( - "Generated code should use RedisArtifactStore for artifacts" - ) + assert ( + "artifacts = " in code or "artifacts=" in code + ), "Generated code should assign artifacts namespace" + assert ( + "RedisArtifactStore" in code + ), "Generated code should use RedisArtifactStore for artifacts" class TestUnknownStorageType: @@ -987,9 +986,9 @@ def test_redis_code_uses_from_url_pattern(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "from_url" in code.lower() or "Redis(" in code, ( - "Generated code should use Redis.from_url() or Redis constructor" - ) + assert ( + "from_url" in code.lower() or "Redis(" in code + ), "Generated code should use Redis.from_url() or Redis constructor" def test_redis_code_handles_nest_asyncio(self) -> None: """Generated code should apply nest_asyncio for sync wrappers. @@ -1007,9 +1006,9 @@ def test_redis_code_handles_nest_asyncio(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert "nest_asyncio" in code, ( - "Generated code should import and apply nest_asyncio for nested event loop support" - ) + assert ( + "nest_asyncio" in code + ), "Generated code should import and apply nest_asyncio for nested event loop support" def test_redis_code_imports_cli_adapter(self) -> None: """Generated code should import CLIAdapter for tool execution. @@ -1128,7 +1127,7 @@ async def test_redis_namespace_full_execution( "'tools' in dir() and 'workflows' in dir() and 'artifacts' in dir()" ) assert result.error is None, f"Namespace check failed: {result.error}" - assert result.value in (True, "True"), "Namespaces should be available" + assert result.value is True, "Namespaces should be available" # Verify artifacts work with Redis backend result = await executor.run('artifacts.save("redis_test", {"from": "redis"}) or True') diff --git a/tests/test_vector_store.py b/tests/test_vector_store.py index 188b226..07b9f9a 100644 --- a/tests/test_vector_store.py +++ b/tests/test_vector_store.py @@ -11,70 +11,6 @@ import pytest -class TestVectorStoreProtocol: - """Protocol compliance tests for VectorStore implementations.""" - - def test_vector_store_protocol_exists(self) -> None: - """VectorStore protocol should be importable from workflows module.""" - # Protocol should be runtime checkable - - from py_code_mode.workflows.vector_store import VectorStore - - assert isinstance(VectorStore, type) - - def test_protocol_has_add_method(self) -> None: - """VectorStore must define add() method signature.""" - from py_code_mode.workflows.vector_store import VectorStore - - # Protocol defines method signatures at class level - assert hasattr(VectorStore, "add") - - def test_protocol_has_remove_method(self) -> None: - """VectorStore must define remove() method signature.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "remove") - - def test_protocol_has_search_method(self) -> None: - """VectorStore must define search() method signature.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "search") - - def test_protocol_has_get_content_hash_method(self) -> None: - """VectorStore must define get_content_hash() for change detection.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "get_content_hash") - - def test_protocol_has_get_model_info_method(self) -> None: - """VectorStore must define get_model_info() for model validation.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "get_model_info") - - def test_protocol_has_clear_method(self) -> None: - """VectorStore must define clear() to reset index.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "clear") - - def test_protocol_has_count_method(self) -> None: - """VectorStore must define count() to get indexed workflow count.""" - from py_code_mode.workflows.vector_store import VectorStore - - assert hasattr(VectorStore, "count") - - def test_protocol_is_runtime_checkable(self) -> None: - """Protocol should support isinstance() checks.""" - - from py_code_mode.workflows.vector_store import VectorStore - - # Check that VectorStore has the runtime_checkable marker - # This allows isinstance(obj, VectorStore) to work - assert hasattr(VectorStore, "__protocol_attrs__") or hasattr(VectorStore, "_is_protocol") - - class TestModelInfo: """Tests for ModelInfo dataclass.""" From 73fba7f1c5f8414387150e4dfcd075b881129740 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 22:16:47 -0800 Subject: [PATCH 26/27] Mark redis testcontainers as docker and skip when Docker unavailable --- tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index a809d8a..b480a9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -801,6 +801,10 @@ def redis_container(): if not TESTCONTAINERS_AVAILABLE: pytest.skip("testcontainers[redis] not installed") if not _docker_daemon_is_available(): + if os.environ.get("CI"): + pytest.fail( + "Docker daemon not available for testcontainers Redis (CI should provide Docker)" + ) pytest.skip("Docker daemon not available for testcontainers Redis") with RedisContainer(image="redis:7-alpine") as container: From 1ea6930cfc83a62fa567d1e21241ae75fed760a7 Mon Sep 17 00:00:00 2001 From: actae0n <19864268+xpcmdshell@users.noreply.github.com> Date: Sun, 8 Feb 2026 22:23:15 -0800 Subject: [PATCH 27/27] Align pre-commit ruff with CI --- .pre-commit-config.yaml | 4 +- tests/test_error_handling.py | 6 +- tests/test_session.py | 30 +++++----- tests/test_subprocess_namespace_injection.py | 60 ++++++++++---------- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c83d99a..4fa6715 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.0 + # Keep in sync with the ruff version pinned in uv.lock / used in CI. + rev: v0.14.9 hooks: - id: ruff args: [--fix] - id: ruff-format - diff --git a/tests/test_error_handling.py b/tests/test_error_handling.py index b19e0b0..8fa23cc 100644 --- a/tests/test_error_handling.py +++ b/tests/test_error_handling.py @@ -727,9 +727,9 @@ async def test_developer_discovers_workflow_parse_error_through_logs( log_messages = " ".join(r.message for r in log_capture.records) assert "syntax_error" in log_messages, "Log should mention which file had errors" # Ideally also shows the error type - assert ( - "syntax" in log_messages.lower() or "error" in log_messages.lower() - ), "Log should indicate type of error" + assert "syntax" in log_messages.lower() or "error" in log_messages.lower(), ( + "Log should indicate type of error" + ) # ============================================================================= diff --git a/tests/test_session.py b/tests/test_session.py index 577e655..d024450 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -907,9 +907,9 @@ async def test_subprocess_capabilities_via_session(self, tmp_path: Path) -> None async with Session(storage=storage, executor=executor) as session: # SubprocessExecutor advertises PROCESS_ISOLATION - assert session.supports( - Capability.PROCESS_ISOLATION - ), "SubprocessExecutor should support PROCESS_ISOLATION" + assert session.supports(Capability.PROCESS_ISOLATION), ( + "SubprocessExecutor should support PROCESS_ISOLATION" + ) # SubprocessExecutor supports TIMEOUT assert session.supports(Capability.TIMEOUT), "SubprocessExecutor should support TIMEOUT" @@ -918,14 +918,14 @@ async def test_subprocess_capabilities_via_session(self, tmp_path: Path) -> None assert session.supports(Capability.RESET), "SubprocessExecutor should support RESET" # SubprocessExecutor does NOT support NETWORK_ISOLATION - assert not session.supports( - Capability.NETWORK_ISOLATION - ), "SubprocessExecutor should NOT support NETWORK_ISOLATION" + assert not session.supports(Capability.NETWORK_ISOLATION), ( + "SubprocessExecutor should NOT support NETWORK_ISOLATION" + ) # SubprocessExecutor does NOT support FILESYSTEM_ISOLATION - assert not session.supports( - Capability.FILESYSTEM_ISOLATION - ), "SubprocessExecutor should NOT support FILESYSTEM_ISOLATION" + assert not session.supports(Capability.FILESYSTEM_ISOLATION), ( + "SubprocessExecutor should NOT support FILESYSTEM_ISOLATION" + ) # supported_capabilities() returns the full set caps = session.supported_capabilities() @@ -962,9 +962,9 @@ async def test_process_isolation_implies_serializable_access(self, tmp_path: Pat executor = SubprocessExecutor(config=config) # Verify the invariant: PROCESS_ISOLATION capability exists - assert executor.supports( - Capability.PROCESS_ISOLATION - ), "SubprocessExecutor must have PROCESS_ISOLATION capability" + assert executor.supports(Capability.PROCESS_ISOLATION), ( + "SubprocessExecutor must have PROCESS_ISOLATION capability" + ) # Start and verify it can access storage-based namespaces # (this implicitly verifies serializable access works) @@ -973,6 +973,6 @@ async def test_process_isolation_implies_serializable_access(self, tmp_path: Pat # the subprocess won't be able to reconstruct the storage result = await session.run("'artifacts' in dir()") assert result.is_ok, f"Failed: {result.error}" - assert ( - result.value is True - ), "artifacts namespace not available - serializable access likely broken" + assert result.value is True, ( + "artifacts namespace not available - serializable access likely broken" + ) diff --git a/tests/test_subprocess_namespace_injection.py b/tests/test_subprocess_namespace_injection.py index 0fc39c9..5e956f0 100644 --- a/tests/test_subprocess_namespace_injection.py +++ b/tests/test_subprocess_namespace_injection.py @@ -805,9 +805,9 @@ def test_redis_storage_code_imports_redis(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "from redis import Redis" in code or "import redis" in code - ), "Generated code must import Redis client" + assert "from redis import Redis" in code or "import redis" in code, ( + "Generated code must import Redis client" + ) def test_redis_storage_code_uses_provided_url(self) -> None: """Generated code should use the exact redis_url provided. @@ -854,12 +854,12 @@ def test_redis_storage_code_uses_provided_prefixes(self) -> None: code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" # NOTE: tools_prefix assertion removed - tools now owned by executors - assert ( - workflows_prefix in code - ), f"Generated code should contain workflows_prefix: {workflows_prefix}" - assert ( - artifacts_prefix in code - ), f"Generated code should contain artifacts_prefix: {artifacts_prefix}" + assert workflows_prefix in code, ( + f"Generated code should contain workflows_prefix: {workflows_prefix}" + ) + assert artifacts_prefix in code, ( + f"Generated code should contain artifacts_prefix: {artifacts_prefix}" + ) def test_redis_storage_code_sets_up_tools(self) -> None: """Generated code should set up tools namespace with empty registry. @@ -878,9 +878,9 @@ def test_redis_storage_code_sets_up_tools(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "tools = " in code or "tools=" in code - ), "Generated code should assign tools namespace" + assert "tools = " in code or "tools=" in code, ( + "Generated code should assign tools namespace" + ) # Tools are now owned by executor, not storage - empty registry is created assert "ToolRegistry()" in code, "Generated code should create empty ToolRegistry" @@ -900,12 +900,12 @@ def test_redis_storage_code_sets_up_workflows(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "workflows = " in code or "workflows=" in code - ), "Generated code should assign workflows namespace" - assert ( - "RedisWorkflowStore" in code - ), "Generated code should use RedisWorkflowStore for workflows" + assert "workflows = " in code or "workflows=" in code, ( + "Generated code should assign workflows namespace" + ) + assert "RedisWorkflowStore" in code, ( + "Generated code should use RedisWorkflowStore for workflows" + ) def test_redis_storage_code_sets_up_artifacts(self) -> None: """Generated code should set up artifacts namespace with RedisArtifactStore. @@ -923,12 +923,12 @@ def test_redis_storage_code_sets_up_artifacts(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "artifacts = " in code or "artifacts=" in code - ), "Generated code should assign artifacts namespace" - assert ( - "RedisArtifactStore" in code - ), "Generated code should use RedisArtifactStore for artifacts" + assert "artifacts = " in code or "artifacts=" in code, ( + "Generated code should assign artifacts namespace" + ) + assert "RedisArtifactStore" in code, ( + "Generated code should use RedisArtifactStore for artifacts" + ) class TestUnknownStorageType: @@ -986,9 +986,9 @@ def test_redis_code_uses_from_url_pattern(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "from_url" in code.lower() or "Redis(" in code - ), "Generated code should use Redis.from_url() or Redis constructor" + assert "from_url" in code.lower() or "Redis(" in code, ( + "Generated code should use Redis.from_url() or Redis constructor" + ) def test_redis_code_handles_nest_asyncio(self) -> None: """Generated code should apply nest_asyncio for sync wrappers. @@ -1006,9 +1006,9 @@ def test_redis_code_handles_nest_asyncio(self) -> None: ) code = build_namespace_setup_code(storage_access) assert code, "Code must be generated first" - assert ( - "nest_asyncio" in code - ), "Generated code should import and apply nest_asyncio for nested event loop support" + assert "nest_asyncio" in code, ( + "Generated code should import and apply nest_asyncio for nested event loop support" + ) def test_redis_code_imports_cli_adapter(self) -> None: """Generated code should import CLIAdapter for tool execution.