test_toolkit.py

#!/usr/bin/env python3
"""
test_toolkit -- Automated smoke tests for all developer tools. Zero deps.

Validates every tool in the toolkit: syntax check, --help runs, basic smoke tests.
Run after changes to catch regressions. CI/CD friendly (exit 1 on failure).

Usage:
    py test_toolkit.py                    # Run all tests
    py test_toolkit.py --quick            # Syntax + help only (fast)
    py test_toolkit.py --tool serve       # Test specific tool
    py test_toolkit.py --verbose          # Show command output
"""

import argparse
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path

RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
GREEN = "\033[32m"
YELLOW = "\033[33m"
RED = "\033[31m"
CYAN = "\033[36m"

PYTHON = sys.executable
TOOL_DIR = str(Path(__file__).parent)

# Tools that have --help
TOOLS_WITH_HELP = [
    "serve", "mockapi", "httpprobe", "catchhook", "httpfile",
    "dataconv", "csvtool", "textpipe", "smartdiff", "sqlq", "fakegen",
    "multirun", "watchrun", "taskrun", "todo", "quickbench",
    "codebase_analyzer", "readmegen", "mdtool", "diskuse", "scaffold", "changelog",
    "secretscan", "envtool",
    "logtail",
    "renamer", "snap", "tmpl",
    "devutils", "portman", "crontool", "colortool", "cheat", "retest", "timer",
    "toolkit",
    "mcp_schema_auditor", "mcp_tool_selector", "claude_config_linter",
    "ai_devtool", "mcp_security_scanner", "mcp_health_check",
    "test_toolkit",
]

# Skip files that aren't standalone tools
SKIP_FILES = {
    "experiment_harness.py", "build_full_corpus.py",
    "generate_figures.py", "build_verbosity_dataset.py",
}


def color_supported() -> bool:
    if os.environ.get("NO_COLOR"):
        return False
    if sys.platform == "win32":
        return bool(os.environ.get("TERM") or os.environ.get("WT_SESSION"))
    return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()


USE_COLOR = color_supported()


def c(code: str, text: str) -> str:
    return f"{code}{text}{RESET}" if USE_COLOR else text


def run(cmd: list[str], timeout: int = 15, stdin_data: str = None) -> tuple[int, str, str]:
    """Run a command and return (exit_code, stdout, stderr)."""
    try:
        result = subprocess.run(
            cmd, capture_output=True, text=True, timeout=timeout,
            input=stdin_data, cwd=TOOL_DIR,
        )
        return result.returncode, result.stdout, result.stderr
    except subprocess.TimeoutExpired:
        return -1, "", "TIMEOUT"
    except Exception as e:
        return -1, "", str(e)


class TestRunner:
    def __init__(self, verbose: bool = False):
        self.passed = 0
        self.failed = 0
        self.skipped = 0
        self.failures: list[str] = []
        self.verbose = verbose

    def ok(self, name: str, detail: str = ""):
        self.passed += 1
        d = f" {c(DIM, detail)}" if detail else ""
        print(f"  {c(GREEN, 'PASS')}  {name}{d}")

    def fail(self, name: str, detail: str = ""):
        self.failed += 1
        self.failures.append(name)
        d = f" {c(DIM, detail)}" if detail else ""
        print(f"  {c(RED, 'FAIL')}  {name}{d}")

    def skip(self, name: str, reason: str = ""):
        self.skipped += 1
        print(f"  {c(YELLOW, 'SKIP')}  {name} {c(DIM, reason)}")

    def log(self, msg: str):
        if self.verbose:
            for line in msg.strip().splitlines()[:5]:
                print(f"         {c(DIM, line)}")

    def summary(self):
        total = self.passed + self.failed + self.skipped
        print(f"\n  {c(BOLD, 'Results:')} {c(GREEN, f'{self.passed} passed')}", end="")
        if self.failed:
            print(f", {c(RED, f'{self.failed} failed')}", end="")
        if self.skipped:
            print(f", {c(YELLOW, f'{self.skipped} skipped')}", end="")
        print(f" ({total} total)")
        if self.failures:
            print(f"\n  {c(RED, 'Failures:')}")
            for f in self.failures:
                print(f"    - {f}")
        print()


def test_syntax(t: TestRunner):
    """Check all .py files for syntax errors."""
    print(f"\n  {c(BOLD, '--- Syntax Check ---')}\n")
    for pyfile in sorted(Path(TOOL_DIR).glob("*.py")):
        if pyfile.name in SKIP_FILES:
            continue
        code, out, err = run([PYTHON, "-m", "py_compile", str(pyfile)])
        if code == 0:
            t.ok(f"syntax:{pyfile.name}")
        else:
            t.fail(f"syntax:{pyfile.name}", err.strip().splitlines()[-1] if err else "")
            t.log(err)


def test_help(t: TestRunner):
    """Check --help runs without error for all tools."""
    print(f"\n  {c(BOLD, '--- Help Check ---')}\n")
    for tool in TOOLS_WITH_HELP:
        pyfile = Path(TOOL_DIR) / f"{tool}.py"
        if not pyfile.exists():
            t.skip(f"help:{tool}", "(file missing)")
            continue
        code, out, err = run([PYTHON, str(pyfile), "--help"])
        # Some tools use -h, some --help, some show help on no args
        if code == 0 and (out.strip() or err.strip()):
            t.ok(f"help:{tool}")
        else:
            # Try with no args (some tools show help by default)
            code2, out2, err2 = run([PYTHON, str(pyfile)])
            if code2 == 0:
                t.ok(f"help:{tool}", "(default output)")
            else:
                t.fail(f"help:{tool}", f"exit {code}")
                t.log(err or err2)


def test_smoke(t: TestRunner):
    """Run basic smoke tests on key tools."""
    print(f"\n  {c(BOLD, '--- Smoke Tests ---')}\n")
    tmpdir = tempfile.mkdtemp(prefix="toolkit_test_")

    try:
        # devutils: uuid
        code, out, _ = run([PYTHON, "devutils.py", "uuid"])
        if code == 0 and len(out.strip()) == 36:
            t.ok("smoke:devutils:uuid")
        else:
            t.fail("smoke:devutils:uuid")

        # fakegen: name 3
        code, out, _ = run([PYTHON, "fakegen.py", "name", "3"])
        if code == 0 and len(out.strip().splitlines()) == 3:
            t.ok("smoke:fakegen:name")
        else:
            t.fail("smoke:fakegen:name")

        # textpipe: stdin upper
        code, out, _ = run([PYTHON, "textpipe.py", "upper"], stdin_data="hello world\n")
        if code == 0 and "HELLO WORLD" in out:
            t.ok("smoke:textpipe:upper")
        else:
            t.fail("smoke:textpipe:upper")

        # crontool: explain
        code, out, _ = run([PYTHON, "crontool.py", "explain", "0 9 * * 1-5"])
        if code == 0 and "09:00" in out:
            t.ok("smoke:crontool:explain")
        else:
            t.fail("smoke:crontool:explain")

        # colortool: show
        code, out, _ = run([PYTHON, "colortool.py", "show", "#ff0000"])
        if code == 0 and "rgb" in out.lower():
            t.ok("smoke:colortool:show")
        else:
            t.fail("smoke:colortool:show")

        # retest: match
        code, out, _ = run([PYTHON, "retest.py", r"\d+", "abc 123 def"])
        if code == 0 and "123" in out:
            t.ok("smoke:retest:match")
        else:
            t.fail("smoke:retest:match")

        # csvtool: stdin tomd
        code, out, _ = run([PYTHON, "csvtool.py", "tomd"], stdin_data="a,b\n1,2\n3,4\n")
        if code == 0 and "|" in out:
            t.ok("smoke:csvtool:tomd")
        else:
            t.fail("smoke:csvtool:tomd")

        # diskuse: current dir
        code, out, _ = run([PYTHON, "diskuse.py", ".", "--top", "3"])
        if code == 0 and "Total size" in out:
            t.ok("smoke:diskuse:top")
        else:
            t.fail("smoke:diskuse:top")

        # mdtool: headings on a test file
        test_md = Path(tmpdir) / "test.md"
        test_md.write_text("# Title\n## Section\n### Sub\nText\n")
        code, out, _ = run([PYTHON, "mdtool.py", "headings", str(test_md)])
        if code == 0 and "Title" in out and "Section" in out:
            t.ok("smoke:mdtool:headings")
        else:
            t.fail("smoke:mdtool:headings")

        # scaffold: list
        code, out, _ = run([PYTHON, "scaffold.py", "--list"])
        if code == 0 and "python" in out.lower():
            t.ok("smoke:scaffold:list")
        else:
            t.fail("smoke:scaffold:list")

        # secretscan: clean file
        clean = Path(tmpdir) / "clean.py"
        clean.write_text("x = 42\nname = 'Alice'\n")
        code, out, _ = run([PYTHON, "secretscan.py", str(clean), "--no-entropy"])
        if code == 0:
            t.ok("smoke:secretscan:clean")
        else:
            t.fail("smoke:secretscan:clean")

        # tmpl: render stdin
        code, out, _ = run([PYTHON, "tmpl.py", "render", "-v", "x=hello"], stdin_data="say {{x}}\n")
        if code == 0 and "say hello" in out:
            t.ok("smoke:tmpl:render")
        else:
            t.fail("smoke:tmpl:render")

        # toolkit: stats
        code, out, _ = run([PYTHON, "toolkit.py", "stats"])
        if code == 0 and "Total tools" in out:
            t.ok("smoke:toolkit:stats")
        else:
            t.fail("smoke:toolkit:stats")

    finally:
        # Cleanup
        import shutil
        shutil.rmtree(tmpdir, ignore_errors=True)


def main():
    parser = argparse.ArgumentParser(description="test_toolkit -- smoke tests for all tools")
    parser.add_argument("--quick", action="store_true", help="Syntax + help only")
    parser.add_argument("--tool", help="Test specific tool only")
    parser.add_argument("-v", "--verbose", action="store_true", help="Show command output")
    args = parser.parse_args()

    t = TestRunner(verbose=args.verbose)
    start = time.perf_counter()

    print(f"\n  {c(BOLD, 'Toolkit Test Suite')}")
    print(c(DIM, f"  Directory: {TOOL_DIR}\n"))

    if args.tool:
        # Test specific tool
        pyfile = Path(TOOL_DIR) / f"{args.tool}.py"
        if not pyfile.exists():
            print(c(RED, f"  Tool not found: {args.tool}"))
            sys.exit(1)
        code, out, err = run([PYTHON, "-m", "py_compile", str(pyfile)])
        if code == 0:
            t.ok(f"syntax:{args.tool}")
        else:
            t.fail(f"syntax:{args.tool}", err)
        code, out, err = run([PYTHON, str(pyfile), "--help"])
        if code == 0:
            t.ok(f"help:{args.tool}")
        else:
            t.fail(f"help:{args.tool}")
    else:
        test_syntax(t)
        test_help(t)
        if not args.quick:
            test_smoke(t)

    elapsed = time.perf_counter() - start
    t.summary()
    print(c(DIM, f"  Completed in {elapsed:.1f}s\n"))

    sys.exit(1 if t.failed else 0)


if __name__ == "__main__":
    main()