From 0296020d4aaecbf7aec9f5b4446a9dd103c6c84e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 20 Jun 2026 00:11:01 +0800 Subject: [PATCH] Add S3-compatible artifact store (optional boto3, injectable client) --- README.md | 7 ++ README/README_zh-CN.md | 7 ++ README/README_zh-TW.md | 7 ++ .../Eng/doc/new_features/v41_features_doc.rst | 55 ++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v41_features_doc.rst | 50 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 ++ .../gui/script_builder/command_schema.py | 26 ++++ .../utils/artifact_store/__init__.py | 10 ++ .../utils/artifact_store/s3_store.py | 104 ++++++++++++++++ .../utils/executor/action_executor.py | 28 +++++ .../utils/mcp_server/tools/_factories.py | 44 ++++++- .../utils/mcp_server/tools/_handlers.py | 20 +++ pyproject.toml | 1 + .../unit_test/headless/test_s3_store_batch.py | 117 ++++++++++++++++++ 16 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v41_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v41_features_doc.rst create mode 100644 je_auto_control/utils/artifact_store/__init__.py create mode 100644 je_auto_control/utils/artifact_store/s3_store.py create mode 100644 test/unit_test/headless/test_s3_store_batch.py diff --git a/README.md b/README.md index b232208..44455da 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ ## Table of Contents +- [What's new (2026-06-20) — S3-Compatible Artifact Store](#whats-new-2026-06-20--s3-compatible-artifact-store) - [What's new (2026-06-20) — Fuzzy String Matching & Dedupe](#whats-new-2026-06-20--fuzzy-string-matching--dedupe) - [What's new (2026-06-19) — Video Step-Overlay Report](#whats-new-2026-06-19--video-step-overlay-report) - [What's new (2026-06-19) — Agent Observability (GenAI OpenTelemetry Spans)](#whats-new-2026-06-19--agent-observability-genai-opentelemetry-spans) @@ -93,6 +94,12 @@ --- +## What's new (2026-06-20) — S3-Compatible Artifact Store + +Push run artifacts to object storage. Full reference: [`docs/source/Eng/doc/new_features/v41_features_doc.rst`](docs/source/Eng/doc/new_features/v41_features_doc.rst). + +- **`S3ArtifactStore`** (`AC_s3_upload` / `AC_s3_download` / `AC_s3_list` / `AC_s3_delete`, `ac_*`): upload/download/list/delete reports, screenshots, and recordings against any S3-compatible bucket (AWS S3, MinIO, R2). `boto3` is an **optional** `[s3]` extra and the client is **injectable**, so the store's logic — and the executor path — are fully unit-tested with a fake client (no boto3/network); the live AWS path is honestly noted as CI-unverifiable. The whole API is relative to the store `prefix`. A module-level default store backs the commands. + ## What's new (2026-06-20) — Fuzzy String Matching & Dedupe Match noisy OCR/UI text robustly. Full reference: [`docs/source/Eng/doc/new_features/v40_features_doc.rst`](docs/source/Eng/doc/new_features/v40_features_doc.rst). diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index de8a39a..53da234 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -12,6 +12,7 @@ ## 目录 +- [本次更新 (2026-06-20) — S3 兼容成品存储](#本次更新-2026-06-20--s3-兼容成品存储) - [本次更新 (2026-06-20) — 模糊字符串匹配与去重](#本次更新-2026-06-20--模糊字符串匹配与去重) - [本次更新 (2026-06-19) — 视频步骤叠加报告](#本次更新-2026-06-19--视频步骤叠加报告) - [本次更新 (2026-06-19) — Agent 可观测性(GenAI OpenTelemetry Spans)](#本次更新-2026-06-19--agent-可观测性genai-opentelemetry-spans) @@ -92,6 +93,12 @@ --- +## 本次更新 (2026-06-20) — S3 兼容成品存储 + +将运行成品推送到对象存储。完整参考:[`docs/source/Zh/doc/new_features/v41_features_doc.rst`](../docs/source/Zh/doc/new_features/v41_features_doc.rst)。 + +- **`S3ArtifactStore`**(`AC_s3_upload` / `AC_s3_download` / `AC_s3_list` / `AC_s3_delete`、`ac_*`):对任何 S3 兼容存储桶(AWS S3、MinIO、R2)上传/下载/列出/删除报告、屏幕截图与录像。`boto3` 为**可选** `[s3]` extra,且 client **可注入**,因此存储体逻辑(含 executor 路径)以假 client 完整单元测试(无 boto3/网络);实际 AWS 路径诚实标注为 CI 无法验证。整个 API 相对于存储体 `prefix`。模块级的默认存储体支撑这些指令。 + ## 本次更新 (2026-06-20) — 模糊字符串匹配与去重 稳健匹配含噪声的 OCR/UI 文本。完整参考:[`docs/source/Zh/doc/new_features/v40_features_doc.rst`](../docs/source/Zh/doc/new_features/v40_features_doc.rst)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 4a5c857..a23d552 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -12,6 +12,7 @@ ## 目錄 +- [本次更新 (2026-06-20) — S3 相容成品儲存](#本次更新-2026-06-20--s3-相容成品儲存) - [本次更新 (2026-06-20) — 模糊字串比對與去重](#本次更新-2026-06-20--模糊字串比對與去重) - [本次更新 (2026-06-19) — 影片步驟疊加報告](#本次更新-2026-06-19--影片步驟疊加報告) - [本次更新 (2026-06-19) — Agent 可觀測性(GenAI OpenTelemetry Spans)](#本次更新-2026-06-19--agent-可觀測性genai-opentelemetry-spans) @@ -92,6 +93,12 @@ --- +## 本次更新 (2026-06-20) — S3 相容成品儲存 + +將執行成品推送到物件儲存。完整參考:[`docs/source/Zh/doc/new_features/v41_features_doc.rst`](../docs/source/Zh/doc/new_features/v41_features_doc.rst)。 + +- **`S3ArtifactStore`**(`AC_s3_upload` / `AC_s3_download` / `AC_s3_list` / `AC_s3_delete`、`ac_*`):對任何 S3 相容儲存桶(AWS S3、MinIO、R2)上傳/下載/列出/刪除報告、螢幕截圖與錄影。`boto3` 為**選用** `[s3]` extra,且 client **可注入**,因此儲存體邏輯(含 executor 路徑)以假 client 完整單元測試(無 boto3/網路);實際 AWS 路徑誠實標註為 CI 無法驗證。整個 API 相對於儲存體 `prefix`。模組層級的預設儲存體支撐這些指令。 + ## 本次更新 (2026-06-20) — 模糊字串比對與去重 穩健比對含雜訊的 OCR/UI 文字。完整參考:[`docs/source/Zh/doc/new_features/v40_features_doc.rst`](../docs/source/Zh/doc/new_features/v40_features_doc.rst)。 diff --git a/docs/source/Eng/doc/new_features/v41_features_doc.rst b/docs/source/Eng/doc/new_features/v41_features_doc.rst new file mode 100644 index 0000000..b92fd75 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v41_features_doc.rst @@ -0,0 +1,55 @@ +S3-Compatible Artifact Store +============================ + +Reports, screenshots, and screen recordings produced by a run are usually worth +keeping off the runner. ``S3ArtifactStore`` uploads, downloads, lists, and +deletes them against any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, …). + +``boto3`` is an **optional** dependency (``pip install je_auto_control[s3]``): +the S3 client is *injectable*, so the store's logic is fully unit-testable with a +fake client and ``boto3`` is imported only when no client is supplied. The whole +API is relative to the store's configured ``prefix`` — ``upload`` returns a +store-relative key that ``download`` / ``delete`` / ``url`` accept unchanged, and +``list`` strips the prefix back off. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import S3ArtifactStore + + store = S3ArtifactStore("my-bucket", prefix="runs/42") # boto3 client lazily built + key = store.upload("report.html") # -> "report.html" (relative) + store.url(key) # -> "s3://my-bucket/runs/42/report.html" + store.download(key, "local/report.html") + store.list() # -> ["report.html", ...] + store.delete(key) + +For tests or non-AWS backends, pass your own client: +``S3ArtifactStore("bucket", client=my_client)``. + +.. note:: + + The live AWS path requires ``boto3`` plus credentials and is therefore not + exercised in CI; the store's logic is validated against a fake S3 client. + +Executor commands +----------------- + +A module-level default store — configured once with +``configure_default_store(bucket, client=None, prefix="")`` — backs the +executor/MCP commands: + +================================ =================================================== +Command Effect +================================ =================================================== +``AC_s3_upload`` Upload a local artifact; returns ``{key}``. +``AC_s3_download`` Download an object to a local path. +``AC_s3_list`` List object keys (optional extra ``prefix``). +``AC_s3_delete`` Delete an object. +================================ =================================================== + +The same operations are exposed as MCP tools (``ac_s3_upload`` / +``ac_s3_download`` / ``ac_s3_list`` / ``ac_s3_delete``) and as Script Builder +commands under **Tools**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 27b3fa9..1e0597b 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -63,6 +63,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v38_features_doc doc/new_features/v39_features_doc doc/new_features/v40_features_doc + doc/new_features/v41_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v41_features_doc.rst b/docs/source/Zh/doc/new_features/v41_features_doc.rst new file mode 100644 index 0000000..5f49728 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v41_features_doc.rst @@ -0,0 +1,50 @@ +S3 相容成品儲存 +=============== + +一次執行產生的報告、螢幕截圖與螢幕錄影,通常值得存放到 runner 之外。 +``S3ArtifactStore`` 可對任何 S3 相容儲存桶(AWS S3、MinIO、Cloudflare R2…)上傳、下 +載、列出與刪除這些成品。 + +``boto3`` 為**選用**相依(``pip install je_auto_control[s3]``):S3 client *可注入*,因 +此儲存體的邏輯可用假 client 完整單元測試,且僅在未提供 client 時才匯入 ``boto3``。整個 +API 皆相對於儲存體設定的 ``prefix`` —— ``upload`` 回傳儲存體相對鍵,``download`` / +``delete`` / ``url`` 原樣接受,而 ``list`` 則會把 prefix 去除。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import S3ArtifactStore + + store = S3ArtifactStore("my-bucket", prefix="runs/42") # boto3 client 延遲建立 + key = store.upload("report.html") # -> "report.html"(相對) + store.url(key) # -> "s3://my-bucket/runs/42/report.html" + store.download(key, "local/report.html") + store.list() # -> ["report.html", ...] + store.delete(key) + +測試或非 AWS 後端可傳入自己的 client:``S3ArtifactStore("bucket", client=my_client)``。 + +.. note:: + + 實際的 AWS 路徑需要 ``boto3`` 與憑證,因此不會在 CI 中執行;儲存體邏輯以假 S3 + client 驗證。 + +執行器指令 +---------- + +模組層級的預設儲存體 —— 以 ``configure_default_store(bucket, client=None, prefix="")`` +設定一次 —— 支撐 executor/MCP 指令: + +================================ =================================================== +指令 效果 +================================ =================================================== +``AC_s3_upload`` 上傳本機成品;回傳 ``{key}``。 +``AC_s3_download`` 將物件下載到本機路徑。 +``AC_s3_list`` 列出物件鍵(可加 ``prefix``)。 +``AC_s3_delete`` 刪除物件。 +================================ =================================================== + +相同操作亦提供為 MCP 工具(``ac_s3_upload`` / ``ac_s3_download`` / ``ac_s3_list`` / +``ac_s3_delete``),以及 Script Builder 中 **Tools** 分類下的指令。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index eb68bf6..3260833 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -63,6 +63,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v38_features_doc doc/new_features/v39_features_doc doc/new_features/v40_features_doc + doc/new_features/v41_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index f8c18a8..748aa16 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -234,6 +234,11 @@ from je_auto_control.utils.fuzzy import ( fuzzy_best_match, fuzzy_dedupe, fuzzy_matches, fuzzy_ratio, ) +# S3-compatible artifact store (optional boto3, injectable client) +from je_auto_control.utils.artifact_store import ( + S3ArtifactStore, configure_default_store, get_default_store, + set_default_store, +) # Background popup/interrupt watchdog (unattended automation) from je_auto_control.utils.watchdog import ( PopupWatchdog, WatchdogRule, default_popup_watchdog, @@ -681,6 +686,8 @@ def start_autocontrol_gui(*args, **kwargs): "VideoStep", "build_overlay_plan", "render_overlay_frame", "write_step_video", "fuzzy_best_match", "fuzzy_dedupe", "fuzzy_matches", "fuzzy_ratio", + "S3ArtifactStore", "configure_default_store", "get_default_store", + "set_default_store", # MCP server "AuditLogger", "HttpMCPServer", "MCPContent", "MCPPrompt", "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a726cbf..4411b35 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -902,6 +902,32 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="Collapse near-duplicate strings (JSON list).", )) + specs.append(CommandSpec( + "AC_s3_upload", "Tools", "S3: Upload Artifact", + fields=( + FieldSpec("local_path", FieldType.FILE_PATH), + FieldSpec("key", FieldType.STRING, optional=True), + ), + description="Upload a file to the configured default S3 store.", + )) + specs.append(CommandSpec( + "AC_s3_download", "Tools", "S3: Download Artifact", + fields=( + FieldSpec("key", FieldType.STRING), + FieldSpec("local_path", FieldType.STRING), + ), + description="Download an object from the default S3 store.", + )) + specs.append(CommandSpec( + "AC_s3_list", "Tools", "S3: List Artifacts", + fields=(FieldSpec("prefix", FieldType.STRING, optional=True),), + description="List object keys in the default S3 store.", + )) + specs.append(CommandSpec( + "AC_s3_delete", "Tools", "S3: Delete Artifact", + fields=(FieldSpec("key", FieldType.STRING),), + description="Delete an object from the default S3 store.", + )) specs.append(CommandSpec( "AC_generate_sop", "Report", "Generate SOP Document", fields=( diff --git a/je_auto_control/utils/artifact_store/__init__.py b/je_auto_control/utils/artifact_store/__init__.py new file mode 100644 index 0000000..794abca --- /dev/null +++ b/je_auto_control/utils/artifact_store/__init__.py @@ -0,0 +1,10 @@ +"""S3-compatible artifact store for reports, screenshots, and recordings.""" +from je_auto_control.utils.artifact_store.s3_store import ( + S3ArtifactStore, configure_default_store, get_default_store, + set_default_store, +) + +__all__ = [ + "S3ArtifactStore", "configure_default_store", "get_default_store", + "set_default_store", +] diff --git a/je_auto_control/utils/artifact_store/s3_store.py b/je_auto_control/utils/artifact_store/s3_store.py new file mode 100644 index 0000000..ce2e711 --- /dev/null +++ b/je_auto_control/utils/artifact_store/s3_store.py @@ -0,0 +1,104 @@ +"""Push automation artifacts to S3-compatible object storage. + +Reports, screenshots, and screen recordings produced by a run are usually worth +keeping off the runner. ``S3ArtifactStore`` uploads/downloads/lists/deletes them +against any S3-compatible bucket (AWS S3, MinIO, R2, …). ``boto3`` is an +**optional** dependency (``pip install je_auto_control[s3]``): the S3 client is +injectable, so the store's logic is fully unit-testable with a fake client and +``boto3`` is imported only when no client is supplied. + +A module-level default store (configured once via :func:`configure_default_store` +or :func:`set_default_store`) backs the executor/MCP commands. Object keys are +prefixed and normalised to forward slashes. Imports no ``PySide6``. +""" +from pathlib import Path +from typing import Any, List, Optional + + +def _build_boto3_client() -> Any: # pragma: no cover - requires boto3 + creds + import boto3 + return boto3.client("s3") + + +class S3ArtifactStore: + """Upload/download/list/delete artifacts in one S3-compatible bucket.""" + + def __init__(self, bucket: str, *, client: Optional[Any] = None, + prefix: str = "") -> None: + """``client`` is an S3 client (boto3 or a fake); built lazily if None.""" + self._bucket = bucket + self._prefix = prefix.strip("/") + self._client = client + + @property + def client(self) -> Any: + """The S3 client, lazily built via boto3 on first use.""" + if self._client is None: + self._client = _build_boto3_client() + return self._client + + def _key(self, name: str) -> str: + """Map a store-relative name to its full (prefixed) object key.""" + name = name.replace("\\", "/").lstrip("/") + return f"{self._prefix}/{name}" if self._prefix else name + + def _relative(self, full_key: str) -> str: + """Strip the store prefix from a full object key.""" + if self._prefix and full_key.startswith(self._prefix + "/"): + return full_key[len(self._prefix) + 1:] + return full_key + + def upload(self, local_path: str, key: Optional[str] = None) -> str: + """Upload ``local_path``; return the store-relative key.""" + relative = key or Path(local_path).name + self.client.upload_file(local_path, self._bucket, self._key(relative)) + return relative + + def download(self, key: str, local_path: str) -> str: + """Download store-relative object ``key`` to ``local_path``.""" + target = Path(local_path) + target.parent.mkdir(parents=True, exist_ok=True) + self.client.download_file(self._bucket, self._key(key), str(target)) + return str(target) + + def list(self, prefix: Optional[str] = None) -> List[str]: + """List store-relative object keys, optionally under extra ``prefix``.""" + response = self.client.list_objects_v2( + Bucket=self._bucket, Prefix=self._key(prefix) if prefix + else self._prefix) + return [self._relative(item["Key"]) + for item in response.get("Contents", [])] + + def delete(self, key: str) -> bool: + """Delete store-relative object ``key``; return ``True``.""" + self.client.delete_object(Bucket=self._bucket, Key=self._key(key)) + return True + + def url(self, key: str) -> str: + """Return the ``s3://`` URL for store-relative object ``key``.""" + return f"s3://{self._bucket}/{self._key(key)}" + + +_STATE: dict = {"store": None} + + +def set_default_store(store: Optional[S3ArtifactStore]) -> None: + """Install (or clear) the module-level default artifact store.""" + _STATE["store"] = store + + +def configure_default_store(bucket: str, *, client: Optional[Any] = None, + prefix: str = "") -> S3ArtifactStore: + """Build and install the default store; return it.""" + store = S3ArtifactStore(bucket, client=client, prefix=prefix) + set_default_store(store) + return store + + +def get_default_store() -> S3ArtifactStore: + """Return the default store or raise if it has not been configured.""" + store = _STATE["store"] + if store is None: + raise RuntimeError( + "no default artifact store; call configure_default_store(bucket)") + return store diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 41d6f0a..73db389 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3098,6 +3098,30 @@ def _fuzzy_dedupe(items: Any, threshold: float = 0.9, ignore_case=ignore_case)} +def _s3_upload(local_path: str, key: Optional[str] = None) -> Dict[str, Any]: + """Adapter: upload an artifact to the default S3 store; return the key.""" + from je_auto_control.utils.artifact_store import get_default_store + return {"key": get_default_store().upload(local_path, key)} + + +def _s3_download(key: str, local_path: str) -> Dict[str, Any]: + """Adapter: download an artifact from the default S3 store.""" + from je_auto_control.utils.artifact_store import get_default_store + return {"path": get_default_store().download(key, local_path)} + + +def _s3_list(prefix: Optional[str] = None) -> Dict[str, Any]: + """Adapter: list artifact keys in the default S3 store.""" + from je_auto_control.utils.artifact_store import get_default_store + return {"keys": get_default_store().list(prefix)} + + +def _s3_delete(key: str) -> Dict[str, Any]: + """Adapter: delete an artifact from the default S3 store.""" + from je_auto_control.utils.artifact_store import get_default_store + return {"deleted": get_default_store().delete(key)} + + class Executor: """ Executor @@ -3353,6 +3377,10 @@ def __init__(self): "AC_fuzzy_ratio": _fuzzy_ratio, "AC_fuzzy_best_match": _fuzzy_best_match, "AC_fuzzy_dedupe": _fuzzy_dedupe, + "AC_s3_upload": _s3_upload, + "AC_s3_download": _s3_download, + "AC_s3_list": _s3_list, + "AC_s3_delete": _s3_delete, "AC_a11y_record_start": _a11y_record_start, "AC_a11y_record_stop": _a11y_record_stop, "AC_a11y_record_events": _a11y_record_events, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 0069d94..820b40e 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2885,6 +2885,48 @@ def fuzzy_tools() -> List[MCPTool]: ] +def artifact_store_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_s3_upload", + description=("Upload a local artifact to the configured default " + "S3-compatible store. Optional 'key' (defaults to the " + "file name). Returns {key}."), + input_schema=schema( + {"local_path": {"type": "string"}, "key": {"type": "string"}}, + ["local_path"]), + handler=h.s3_upload, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_s3_download", + description=("Download object 'key' from the default S3 store to " + "'local_path'. Returns {path}."), + input_schema=schema( + {"key": {"type": "string"}, + "local_path": {"type": "string"}}, ["key", "local_path"]), + handler=h.s3_download, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_s3_list", + description=("List object keys in the default S3 store, optionally " + "under an extra 'prefix'. Returns {keys}."), + input_schema=schema({"prefix": {"type": "string"}}), + handler=h.s3_list, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_s3_delete", + description="Delete object 'key' from the default S3 store. " + "Returns {deleted}.", + input_schema=schema({"key": {"type": "string"}}, ["key"]), + handler=h.s3_delete, + annotations=SIDE_EFFECT_ONLY, + ), + ] + + def unattended_tools() -> List[MCPTool]: return [ MCPTool( @@ -3945,7 +3987,7 @@ def media_assert_tools() -> List[MCPTool]: process_doc_tools, tween_drag_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, - video_report_tools, fuzzy_tools, + video_report_tools, fuzzy_tools, artifact_store_tools, screen_record_tools, process_and_shell_tools, remote_desktop_tools, gamepad_tools, usb_passthrough_tools, assertion_tools, data_source_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 2621783..b5ed80c 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -1392,6 +1392,26 @@ def fuzzy_dedupe(items, threshold=0.9, ignore_case=True): ignore_case=ignore_case)} +def s3_upload(local_path, key=None): + from je_auto_control.utils.artifact_store import get_default_store + return {"key": get_default_store().upload(local_path, key)} + + +def s3_download(key, local_path): + from je_auto_control.utils.artifact_store import get_default_store + return {"path": get_default_store().download(key, local_path)} + + +def s3_list(prefix=None): + from je_auto_control.utils.artifact_store import get_default_store + return {"keys": get_default_store().list(prefix)} + + +def s3_delete(key): + from je_auto_control.utils.artifact_store import get_default_store + return {"deleted": get_default_store().delete(key)} + + def vlm_locate(description: str, screen_region: Optional[List[int]] = None, model: Optional[str] = None) -> Optional[List[int]]: diff --git a/pyproject.toml b/pyproject.toml index a7136a5..9c23220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ discovery = ["zeroconf>=0.130"] pdf = ["pypdf>=4.0"] office = ["openpyxl>=3.1", "python-docx>=1.1", "python-pptx>=0.6"] fuzzy = ["rapidfuzz>=3.0"] +s3 = ["boto3>=1.34"] [tool.bandit] exclude_dirs = [ diff --git a/test/unit_test/headless/test_s3_store_batch.py b/test/unit_test/headless/test_s3_store_batch.py new file mode 100644 index 0000000..8be3854 --- /dev/null +++ b/test/unit_test/headless/test_s3_store_batch.py @@ -0,0 +1,117 @@ +"""Headless tests for the S3 artifact store. A fake S3 client implements the +four boto3 methods used, so the store's logic (and the executor path) is fully +tested without boto3 or network. Pure stdlib, no Qt imports.""" +from pathlib import Path + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.artifact_store import ( + S3ArtifactStore, configure_default_store, get_default_store, + set_default_store) + + +class _FakeS3: + """Minimal stand-in for a boto3 S3 client (in-memory object store).""" + + def __init__(self): + self.objects = {} + + def upload_file(self, filename, bucket, key): + self.objects[(bucket, key)] = Path(filename).read_bytes() + + def download_file(self, bucket, key, filename): + Path(filename).write_bytes(self.objects[(bucket, key)]) + + def list_objects_v2(self, Bucket, Prefix=""): + keys = [k for (b, k) in self.objects if b == Bucket + and k.startswith(Prefix)] + return {"Contents": [{"Key": k} for k in sorted(keys)]} + + def delete_object(self, Bucket, Key): + self.objects.pop((Bucket, Key), None) + + +@pytest.fixture +def store(): + return S3ArtifactStore("my-bucket", client=_FakeS3(), prefix="runs/42") + + +def test_upload_returns_relative_key_and_prefixed_url(tmp_path, store): + art = tmp_path / "report.html" + art.write_text("", encoding="utf-8") + key = store.upload(str(art)) + assert key == "report.html" # store-relative + assert store.url(key) == "s3://my-bucket/runs/42/report.html" + + +def test_upload_explicit_key(tmp_path, store): + art = tmp_path / "a.txt" + art.write_text("x", encoding="utf-8") + assert store.upload(str(art), key="nested/out.txt") == "nested/out.txt" + + +def test_round_trip_download(tmp_path, store): + art = tmp_path / "data.bin" + art.write_bytes(b"\x00\x01\x02") + store.upload(str(art), key="data.bin") + dest = tmp_path / "out" / "data.bin" + store.download("data.bin", str(dest)) + assert dest.read_bytes() == b"\x00\x01\x02" + + +def test_list_and_delete(tmp_path, store): + for name in ("a.txt", "b.txt"): + p = tmp_path / name + p.write_text(name, encoding="utf-8") + store.upload(str(p)) + assert sorted(store.list()) == ["a.txt", "b.txt"] + assert store.delete("a.txt") is True + assert store.list() == ["b.txt"] + + +def test_get_default_store_unconfigured_raises(): + set_default_store(None) + with pytest.raises(RuntimeError): + get_default_store() + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip_with_fake_client(tmp_path): + configure_default_store("bkt", client=_FakeS3(), prefix="ci") + try: + art = tmp_path / "log.txt" + art.write_text("hello", encoding="utf-8") + rec = ac.execute_action([ + ["AC_s3_upload", {"local_path": str(art), "key": "log.txt"}], + ]) + assert next(v for v in rec.values() + if isinstance(v, dict))["key"] == "log.txt" + rec2 = ac.execute_action([["AC_s3_list", {}]]) + assert "log.txt" in next(v for v in rec2.values() + if isinstance(v, dict))["keys"] + finally: + set_default_store(None) # leave global state clean + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_s3_upload", "AC_s3_download", "AC_s3_list", + "AC_s3_delete"} <= known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_s3_upload", "ac_s3_download", "ac_s3_list", + "ac_s3_delete"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + cmds = {s.command for s in _build_specs()} + assert {"AC_s3_upload", "AC_s3_download", "AC_s3_list", + "AC_s3_delete"} <= cmds + + +def test_facade_exports(): + for attr in ("S3ArtifactStore", "configure_default_store", + "get_default_store", "set_default_store"): + assert hasattr(ac, attr) + assert attr in ac.__all__