toolkit.py

#!/usr/bin/env python3
"""
toolkit -- Discover, search, and run all developer tools. The meta-launcher.

With 30+ tools in the toolkit, this helps you find the right one.
Auto-discovers .py tools in the same directory, parses their docstrings.

Usage:
    py toolkit.py                        # List all tools by category
    py toolkit.py search json            # Find tools related to JSON
    py toolkit.py search api             # Find API-related tools
    py toolkit.py info serve             # Show detailed help for serve.py
    py toolkit.py run serve -p 3000      # Run serve.py with arguments
    py toolkit.py stats                  # Toolkit summary statistics
    py toolkit.py categories             # List categories
"""

import argparse
import os
import re
import subprocess
import sys
from pathlib import Path

RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
GREEN = "\033[32m"
YELLOW = "\033[33m"
CYAN = "\033[36m"
RED = "\033[31m"
MAGENTA = "\033[35m"

# Tool categories (tool_name -> category)
CATEGORIES = {
    "mcp_schema_auditor": "MCP/AI",
    "mcp_tool_selector": "MCP/AI",
    "claude_config_linter": "MCP/AI",
    "ai_devtool": "MCP/AI",
    "mcp_security_scanner": "MCP/AI",
    "mcp_health_check": "MCP/AI",
    "serve": "Web/API",
    "mockapi": "Web/API",
    "httpprobe": "Web/API",
    "catchhook": "Web/API",
    "dataconv": "Data",
    "csvtool": "Data",
    "textpipe": "Data",
    "smartdiff": "Data",
    "sqlq": "Data",
    "fakegen": "Data",
    "multirun": "Process",
    "watchrun": "Process",
    "taskrun": "Process",
    "todo": "Process",
    "quickbench": "Process",
    "logtail": "Logging",
    "codebase_analyzer": "Code/Project",
    "readmegen": "Code/Project",
    "mdtool": "Code/Project",
    "diskuse": "Code/Project",
    "secretscan": "Security",
    "envtool": "Security",
    "devutils": "Utilities",
    "portman": "Utilities",
    "crontool": "Utilities",
    "colortool": "Utilities",
    "renamer": "Files",
    "toolkit": "Meta",
}

CATEGORY_ORDER = [
    "Web/API", "Data", "Process", "Code/Project",
    "Security", "Logging", "Files", "Utilities", "MCP/AI", "Meta",
]

CATEGORY_ICONS = {
    "Web/API": "@", "Data": "#", "Process": ">", "Code/Project": "/",
    "Security": "!", "Logging": "~", "Files": "+", "Utilities": "*",
    "MCP/AI": "^", "Meta": "=",
}

# Keywords for search (beyond docstring parsing)
EXTRA_KEYWORDS = {
    "serve": "http server localhost cors spa upload development",
    "mockapi": "mock api fake server routes json stub",
    "httpprobe": "health check endpoint status ssl certificate url",
    "catchhook": "webhook receiver inspect request capture log",
    "dataconv": "json yaml toml csv xml convert transform jq query",
    "csvtool": "csv query filter sort aggregate groupby column table",
    "textpipe": "grep awk sort uniq text filter pipeline chain",
    "smartdiff": "diff compare json yaml directory structure",
    "sqlq": "sqlite database query sql table schema export",
    "fakegen": "fake data generator mock test user email name random",
    "multirun": "parallel process foreman procfile simultaneous",
    "watchrun": "watch file change reload auto restart trigger",
    "taskrun": "task runner make just dependency build",
    "todo": "task list manager checklist priority tag",
    "quickbench": "benchmark time measure performance compare hyperfine",
    "logtail": "log tail follow filter json level grep highlight",
    "codebase_analyzer": "project stats language lines code analysis",
    "readmegen": "readme generate documentation markdown",
    "mdtool": "markdown toc table format links heading",
    "diskuse": "disk space usage size directory file ncdu du",
    "secretscan": "secret leak api key password token scan security",
    "envtool": "env environment variable dotenv secrets",
    "devutils": "base64 uuid hash jwt url encode decode password",
    "portman": "port process pid kill listen network",
    "crontool": "cron schedule expression explain next run validate",
    "colortool": "color hex rgb hsl palette contrast wcag css",
    "renamer": "rename file batch regex pattern sequential",
    "mcp_schema_auditor": "mcp schema token bloat optimize compress",
    "mcp_tool_selector": "mcp tool select bm25 relevance",
    "claude_config_linter": "claude gemini config lint instruction",
    "ai_devtool": "ai dev unified cli check mcp",
    "mcp_security_scanner": "mcp security vulnerability auth",
    "mcp_health_check": "mcp health report score",
}


def color_supported() -> bool:
    if os.environ.get("NO_COLOR"):
        return False
    if sys.platform == "win32":
        return bool(os.environ.get("TERM") or os.environ.get("WT_SESSION"))
    return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()


USE_COLOR = color_supported()


def c(code: str, text: str) -> str:
    return f"{code}{text}{RESET}" if USE_COLOR else text


def discover_tools(directory: str) -> list[dict]:
    """Discover all .py tools and parse their docstrings."""
    tools = []
    skip = {"toolkit.py", "__pycache__", "experiment_harness.py",
            "build_full_corpus.py", "generate_figures.py", "build_verbosity_dataset.py"}

    for path in sorted(Path(directory).glob("*.py")):
        if path.name in skip:
            continue

        try:
            content = path.read_text(encoding="utf-8", errors="replace")
        except Exception:
            continue

        # Extract module docstring (may be after shebang/comments)
        m = re.search(r'"""(.*?)"""', content, re.DOTALL)
        if not m:
            m = re.search(r"'''(.*?)'''", content, re.DOTALL)
        if not m:
            continue

        docstring = m.group(1).strip()
        lines = docstring.splitlines()

        # First line is the title/tagline
        title_line = lines[0] if lines else path.stem
        # Extract just the tagline (after " -- ")
        if " -- " in title_line:
            name_part, tagline = title_line.split(" -- ", 1)
        elif " - " in title_line:
            name_part, tagline = title_line.split(" - ", 1)
        else:
            tagline = title_line

        # Extract usage examples
        usage_lines = []
        in_usage = False
        for line in lines:
            if line.strip().lower().startswith("usage:"):
                in_usage = True
                continue
            if in_usage:
                if line.strip() and (line.startswith("    ") or line.startswith("\t")):
                    usage_lines.append(line.strip())
                elif not line.strip():
                    continue
                else:
                    break

        tool_name = path.stem
        category = CATEGORIES.get(tool_name, "Other")

        tools.append({
            "name": tool_name,
            "file": path.name,
            "path": str(path),
            "tagline": tagline.strip().rstrip("."),
            "docstring": docstring,
            "usage": usage_lines[:6],
            "category": category,
            "keywords": EXTRA_KEYWORDS.get(tool_name, "") + " " + tagline.lower(),
            "size": path.stat().st_size,
        })

    return tools


def cmd_list(tools: list[dict], category_filter: str = None):
    """List all tools by category."""
    by_cat: dict[str, list] = {}
    for t in tools:
        cat = t["category"]
        if category_filter and cat.lower() != category_filter.lower():
            continue
        by_cat.setdefault(cat, []).append(t)

    total = len(tools)
    print(f"\n  {c(BOLD, f'Developer Toolkit')} ({total} tools)\n")

    for cat in CATEGORY_ORDER:
        if cat not in by_cat:
            continue
        icon = CATEGORY_ICONS.get(cat, "-")
        print(f"  {c(BOLD + YELLOW, f'[{icon}] {cat}')}")
        for t in sorted(by_cat[cat], key=lambda x: x["name"]):
            name_str = t['name'].ljust(24)
            print(f"      {c(CYAN, name_str)} {t['tagline']}")
        print()

    # Any uncategorized
    for cat, cat_tools in by_cat.items():
        if cat not in CATEGORY_ORDER:
            print(f"  {c(BOLD + YELLOW, f'[-] {cat}')}")
            for t in cat_tools:
                name_str = t['name'].ljust(24)
            print(f"      {c(CYAN, name_str)} {t['tagline']}")
            print()


def cmd_search(tools: list[dict], query: str):
    """Search tools by keyword."""
    query_lower = query.lower()
    terms = query_lower.split()

    scored = []
    for t in tools:
        searchable = f"{t['name']} {t['tagline']} {t['keywords']} {t['category']}".lower()
        score = sum(2 if term in t["name"].lower() else 1 for term in terms if term in searchable)
        if score > 0:
            scored.append((score, t))

    scored.sort(key=lambda x: -x[0])

    if not scored:
        print(f"\n  No tools matching '{query}'.\n")
        return

    print(f"\n  {c(BOLD, f'Search: \"{query}\"')} ({len(scored)} results)\n")
    for score, t in scored:
        icon = CATEGORY_ICONS.get(t["category"], "-")
        nm = t['name'].ljust(22)
        print(f"  {c(CYAN, nm)} [{icon}] {t['tagline']}")
        if t["usage"]:
            print(f"  {c(DIM, '  ' + t['usage'][0])}")
    print()


def cmd_info(tools: list[dict], name: str):
    """Show detailed info about a tool."""
    tool = None
    for t in tools:
        if t["name"] == name or t["file"] == name or t["name"] == name.replace(".py", ""):
            tool = t
            break

    if not tool:
        print(f"\n  Tool '{name}' not found. Use 'py toolkit.py search {name}' to search.\n")
        return

    icon = CATEGORY_ICONS.get(tool["category"], "-")
    print(f"\n  {c(BOLD, tool['name'])}  [{icon}] {tool['category']}")
    print(f"  {tool['tagline']}")
    print(f"  File: {c(DIM, tool['file'])} ({tool['size']:,} bytes)")

    if tool["usage"]:
        print(f"\n  {c(BOLD, 'Usage:')}")
        for line in tool["usage"]:
            print(f"    {c(DIM, line)}")

    help_cmd = f"Full help: py {tool['file']} --help"
    print(f"\n  {c(DIM, help_cmd)}\n")


def cmd_run(tools: list[dict], name: str, run_args: list[str]):
    """Run a tool with arguments."""
    tool = None
    for t in tools:
        if t["name"] == name or t["file"] == name or t["name"] == name.replace(".py", ""):
            tool = t
            break

    if not tool:
        print(f"  Error: tool '{name}' not found.", file=sys.stderr)
        sys.exit(1)

    cmd = [sys.executable, tool["path"]] + run_args
    result = subprocess.run(cmd)
    sys.exit(result.returncode)


def cmd_stats(tools: list[dict]):
    """Show toolkit summary."""
    total_size = sum(t["size"] for t in tools)
    by_cat = {}
    for t in tools:
        by_cat.setdefault(t["category"], []).append(t)

    print(f"\n  {c(BOLD, 'Toolkit Statistics')}\n")
    print(f"  Total tools:  {c(BOLD, str(len(tools)))}")
    print(f"  Total size:   {total_size / 1024:.1f} KB")
    print(f"  Categories:   {len(by_cat)}")
    print(f"  Avg size:     {total_size / len(tools) / 1024:.1f} KB per tool")
    print()

    print(f"  {c(BOLD, 'By Category:')}")
    for cat in CATEGORY_ORDER:
        if cat in by_cat:
            cnt = len(by_cat[cat])
            bar = "#" * cnt
            print(f"    {cat:<16} {cnt:>2}  {bar}")
    for cat in by_cat:
        if cat not in CATEGORY_ORDER:
            cnt = len(by_cat[cat])
            print(f"    {cat:<16} {cnt:>2}")

    # Top 5 largest
    by_size = sorted(tools, key=lambda t: t["size"], reverse=True)[:5]
    print(f"\n  {c(BOLD, 'Largest tools:')}")
    for t in by_size:
        nm = t['name'].ljust(22)
        sz = t['size'] / 1024
        print(f"    {nm} {sz:.1f} KB")
    print()


def cmd_categories(tools: list[dict]):
    """List categories with tool counts."""
    by_cat = {}
    for t in tools:
        by_cat.setdefault(t["category"], []).append(t)

    print(f"\n  {c(BOLD, 'Categories')}\n")
    for cat in CATEGORY_ORDER:
        if cat in by_cat:
            icon = CATEGORY_ICONS.get(cat, "-")
            names = ", ".join(t["name"] for t in sorted(by_cat[cat], key=lambda x: x["name"]))
            print(f"  {c(YELLOW, f'[{icon}] {cat}')} ({len(by_cat[cat])})")
            print(f"      {c(DIM, names)}")
    print()


def main():
    # Discover tools in same directory
    script_dir = str(Path(__file__).parent)
    tools = discover_tools(script_dir)

    parser = argparse.ArgumentParser(
        description="toolkit -- discover, search, and run developer tools",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    sub = parser.add_subparsers(dest="command")

    sub.add_parser("list", help="List all tools by category")

    p = sub.add_parser("search", help="Search tools by keyword")
    p.add_argument("query", help="Search term")

    p = sub.add_parser("info", help="Show detailed tool info")
    p.add_argument("name", help="Tool name")

    p = sub.add_parser("run", help="Run a tool")
    p.add_argument("name", help="Tool name")
    p.add_argument("args", nargs=argparse.REMAINDER, help="Tool arguments")

    sub.add_parser("stats", help="Toolkit summary statistics")
    sub.add_parser("categories", help="List categories")

    args = parser.parse_args()

    if args.command == "search":
        cmd_search(tools, args.query)
    elif args.command == "info":
        cmd_info(tools, args.name)
    elif args.command == "run":
        cmd_run(tools, args.name, args.args)
    elif args.command == "stats":
        cmd_stats(tools)
    elif args.command == "categories":
        cmd_categories(tools)
    else:
        cmd_list(tools)


if __name__ == "__main__":
    main()