pyproject.toml

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "openadapt-evals"
version = "0.87.0"
description = "Evaluation infrastructure for GUI agent benchmarks"
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
authors = [
    {name = "Richard Abrich", email = "richard@openadapt.ai"}
]
maintainers = [
    {name = "OpenAdaptAI", email = "contact@openadapt.ai"}
]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Testing",
]
keywords = ["gui", "automation", "evaluation", "benchmark", "agent", "ai"]

dependencies = [
    "pillow>=10.0.0",
    "pydantic-settings>=2.0.0",
    "python-dotenv>=1.2.1",
    "tenacity>=8.2.0",
    "requests>=2.28.0",
    "httpx>=0.25.0",
    "openai>=1.0.0",
    "anthropic>=0.76.0",
    "pyyaml>=6.0",
    "openadapt-consilium>=0.3.2",
    "openadapt-telemetry>=0.2.0",
    "openadapt-types>=0.1.0",
]

[project.optional-dependencies]
ml = [
    # VLM adapters, schema, baselines from openadapt-ml
    # Required for: PolicyAgent, BaselineAgent, trace_export, Qwen3VLAgent
    "openadapt-ml>=0.11.0",
]
dev = [
    "pytest>=8.0.0",
    "pytest-mock>=3.12.0",
    "ruff>=0.1.0",
    "flask>=3.0.0",
    "requests-toolbelt>=1.0.0",
    "fire>=0.5.0",
    # Constrained decoding tests need outlines
    "outlines>=0.1.0",
    # Required by test_workflow_models and workflow pipeline
    "numpy>=1.24.0",
]
waa = [
    # Windows Agent Arena dependencies
    "requests>=2.28.0",
]
azure = [
    # Azure ML dependencies for distributed WAA evaluation
    "azure-ai-ml>=1.12.0",
    "azure-identity>=1.15.0",
    # Azure VM management (SDK path)
    "azure-mgmt-compute>=33.0.0",
    "azure-mgmt-network>=27.0.0",
    "azure-mgmt-resource>=23.2.0",
]
aws = [
    # AWS EC2 management for VM pool operations
    "boto3>=1.34.0",
]
ocr = [
    # OCR for Tier 1.5a text anchoring in grounding cascade
    # GLM-OCR (VLM-based, better accuracy): pip install glmocr
    "glmocr>=0.1.0",
]
retrieval = [
    # For RetrievalAugmentedAgent with automatic demo selection
    "openadapt-retrieval>=0.1.0",
]
viewer = [
    # For live benchmark viewer API server
    "flask>=3.0.0",
    "flask-cors>=4.0.0",
]
wandb = [
    # Weights & Biases for experiment tracking + Weave agent tracing
    "wandb>=0.16.0",
    # Weave: LLM/agent tracing with auto-patched OpenAI/Anthropic clients
    "weave>=0.50.0",
]
training = [
    # TRL for GRPO/PPO training with rollout_func
    "trl>=0.17.0",
    # Unsloth: 90% VRAM reduction for VLM RL training
    "unsloth",
    # HF datasets for TRL training dataset
    "datasets>=2.14.0",
    # Constrained decoding for GRPO (forces valid action format output)
    "outlines>=0.1.0",
    # Perceptual hashing for planner cache (falls back to MD5 if not installed)
    "imagehash>=4.3.0",
    # CLIP embeddings for visual similarity
    "open-clip-torch>=2.20.0",
    # Heavy ML deps for openadapt-ml training
    "openadapt-ml[training]>=0.11.0",
]
verl = [
    # veRL: Volcano Engine Reinforcement Learning for LLMs
    # https://github.com/volcengine/verl
    "verl>=0.3.0",
    # NOTE: VAGEN (Visual Agent with Environment Grounding) is not published
    # on PyPI. Install from source: https://github.com/RAGEN-AI/VAGEN
]
all = [
    "openadapt-evals[dev,waa,azure,aws,retrieval,viewer,wandb]",
]
test = [
    "anthropic>=0.76.0",
    "flask>=3.0.0",
]

[project.scripts]
oa = "openadapt_evals.cli.main:main"
# Legacy entry point (kept for backward compatibility)
openadapt-evals = "openadapt_evals.benchmarks.cli:main"
# VM/pool infrastructure CLI (migrated from openadapt-ml)
oa-vm = "openadapt_evals.benchmarks.vm_cli:main"
# Training, evaluation, collection, and analysis
openadapt-train-grpo = "scripts.train_trl_grpo:main"
openadapt-eval = "scripts.run_full_eval:main"
openadapt-collect = "scripts.collect_distillation_data:main"
openadapt-analyze = "openadapt_evals.analysis.cli:main"
# GPU instance lifecycle (launch, serve, terminate)
openadapt-gpu = "scripts.gpu_cli:main"

[project.urls]
Homepage = "https://github.com/OpenAdaptAI/openadapt-evals"
Repository = "https://github.com/OpenAdaptAI/openadapt-evals"
Documentation = "https://github.com/OpenAdaptAI/openadapt-evals#readme"
"Bug Tracker" = "https://github.com/OpenAdaptAI/openadapt-evals/issues"

[tool.uv.sources]
openadapt-consilium = { git = "https://github.com/OpenAdaptAI/openadapt-consilium.git" }
openadapt-ml = { path = "../openadapt-ml", editable = true }
openadapt-types = { path = "../openadapt-types", editable = true }

[tool.hatch.build.targets.wheel]
packages = ["openadapt_evals", "scripts"]

[tool.pytest.ini_options]
markers = [
    "heavy: requires ML deps (torch, transformers, trl, openadapt-ml) — skipped in light CI",
    "gpu: requires GPU hardware",
    "vm: requires a running WAA VM",
]

[tool.ruff]
line-length = 100

[tool.semantic_release]
version_toml = ["pyproject.toml:project.version"]
commit_message = "chore: release {version}"
major_on_zero = false

[tool.semantic_release.branches.main]
match = "main"

[tool.semantic_release.commit_parser_options]
allowed_tags = ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "style", "test"]
minor_tags = ["feat"]
patch_tags = ["fix", "perf"]