diff --git a/.agentready-config.example.yaml b/.agentready-config.example.yaml index f8db860b..f4c4ae1f 100644 --- a/.agentready-config.example.yaml +++ b/.agentready-config.example.yaml @@ -14,7 +14,7 @@ weights: # Tier 1 (Essential) - 50% total test_execution: 0.10 # Test execution & coverage (highest priority) type_annotations: 0.08 # Type hints in code - claude_md_file: 0.07 # CLAUDE.md/AGENTS.md configuration files + agent_instructions: 0.07 # Agent instruction files (CLAUDE.md/AGENTS.md) ci_quality_gates: 0.05 # CI quality gates (lint + type-check + tests) single_file_verification: 0.05 # Single-file lint/type-check commands readme_structure: 0.05 # README structure and content @@ -87,5 +87,5 @@ report_theme: default # Other attributes are automatically rescaled to maintain sum of 1.0 # # weights: -# claude_md_file: 0.15 +# agent_instructions: 0.15 # test_execution: 0.15 diff --git a/CLAUDE.md b/CLAUDE.md index 2c7f3a24..4a4073cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ Assess repositories against evidence-based attributes for AI-assisted developmen |------|-------| | **Python** | >=3.12 | | **Entry Point** | `agentready.cli.main:cli` (or `python -m agentready`) | -| **Self-Score** | 80.0/100 (Silver) | +| **Self-Score** | 74.5/100 (Silver) | | **Test Coverage** | 37% (target: >80%) | ## Commands @@ -69,7 +69,7 @@ src/agentready/ 4. Add tests in `tests/unit/test_assessors_*.py` **Reference implementations**: -- Simple: `CLAUDEmdAssessor` at `assessors/documentation.py` +- Simple: `AgentInstructionsAssessor` at `assessors/documentation.py` - Complex: `TypeAnnotationsAssessor` at `assessors/code_quality.py` - Conditional: `ContainerSetupAssessor` at `assessors/containers.py` diff --git a/README.md b/README.md index 2beaf90d..948d1323 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ Create `.agentready-config.yaml` to customize weights: ```yaml weights: - claude_md_file: 0.15 # Increase importance (default: 0.07) + agent_instructions: 0.15 # Increase importance (default: 0.07) test_execution: 0.15 # Increase importance (default: 0.10) conventional_commits: 0.01 # Decrease importance (default: 0.03) # Other attributes use defaults, rescaled to sum to 1.0 diff --git a/docs/api-reference.md b/docs/api-reference.md index 0e48e04b..084bb6ed 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -81,7 +81,7 @@ from agentready.models import Attribute class Attribute: """Immutable attribute definition.""" - id: str # Unique identifier (e.g., "claude_md_file") + id: str # Unique identifier (e.g., "agent_instructions") name: str # Display name tier: int # Tier 1-4 (1 = most important) weight: float # Weight in scoring (0.0-1.0, sum to 1.0 across all) @@ -108,12 +108,12 @@ Attribute( ```python attribute = Attribute( - id="claude_md_file", - name="CLAUDE.md File", + id="agent_instructions", + name="Agent Instruction Files", tier=1, weight=0.10, category="Context Window Optimization", - description="CLAUDE.md file at repository root", + description="Agent instruction file (CLAUDE.md, AGENTS.md, or .claude/CLAUDE.md)", rationale="Provides immediate project context to AI agents" ) ``` @@ -1053,7 +1053,7 @@ from agentready.models import Repository # Override default weights programmatically custom_weights = { - "claude_md_file": 0.15, # Increase from 0.10 + "agent_instructions": 0.15, # Increase from 0.10 "readme_structure": 0.12, # Increase from 0.10 "type_annotations": 0.08, # Decrease from 0.10 # ... other attributes diff --git a/docs/attributes.md b/docs/attributes.md index a63b4087..8ba93c73 100644 --- a/docs/attributes.md +++ b/docs/attributes.md @@ -3,7 +3,7 @@ layout: page title: Attributes Reference --- -Complete reference for all 28 agent-ready attributes assessed by AgentReady. +Complete reference for all 25 agent-ready attributes assessed by AgentReady.

šŸ¤– Bootstrap Automation

@@ -51,9 +51,9 @@ Missing a Tier 1 attribute (up to 12% weight) has up to 12x the score impact of *Fundamentals that enable basic AI agent functionality ā€” 55% of total score* -### 1. CLAUDE.md Configuration File +### 1. Agent Instruction Files -**ID**: `claude_md_file` +**ID**: `agent_instructions` **Weight**: 7% **Category**: Context Window Optimization **Status**: āœ… Implemented @@ -68,12 +68,21 @@ Claude Code reads `CLAUDE.md` at the start of every session. Without it, agents #### Measurable Criteria -**Passes if**: +**Two-phase scoring:** + +1. **Presence (up to 70 points)**: File exists at `CLAUDE.md`, `.claude/CLAUDE.md`, or `AGENTS.md` with at least 50 bytes (direct, symlink, or `@` file reference). `AGENTS.md` is a fully equivalent alternative. + +2. **Length (up to 30 points)**: Context file line count determines length credit: + +| Lines | Length Credit | Total Score | +|-------|-------------|-------------| +| <=150 | 30 (full) | 100 | +| 151-300 | 15 (partial) | 85 | +| >300 | 0 (none) | 70 | -- File exists at `CLAUDE.md`, `.claude/CLAUDE.md`, or `AGENTS.md` (all treated as fully equivalent) -- File is at least 50 bytes, OR is a valid symlink, OR contains an `@` file reference +For symlinks and `@` references, line count is measured on the resolved target file. -`AGENTS.md` is a fully equivalent alternative and scores identically to `CLAUDE.md`. +**Agent access documentation** (substantiating evidence): The assessor also checks for agent access sections documenting platform, CLI tool, and authentication requirements. This is recorded as evidence but does not affect the score. #### Example: Good CLAUDE.md @@ -1019,7 +1028,7 @@ setup: **Inline Documentation** (`inline_documentation`, 3%) ā€” Comments and docstrings for functions, classes, modules **File Size Limits** (`file_size_limits`, 3%) ā€” Files under threshold to keep context manageable **Separation of Concerns** (`separation_of_concerns`, 3%) ā€” Clean module boundaries and single-responsibility -**Pattern References** (`pattern_references`, 3%) ā€” Documented patterns for common changes +**Pattern References** (`pattern_references`, 3%) ā€” Documented patterns for common changes. Skills scoring is tiered: 1-2 SKILL.md files earn partial credit (30 pts), 3+ earn full credit (60 pts). Context files >150 lines without skills trigger a warning **Design Intent Documentation** (`design_intent`, 3%) ā€” Preconditions, invariants, and rationale in design docs (moved from T3) *Full details for each attribute available in the [research document](https://github.com/ambient-code/agentready/blob/main/RESEARCH_REPORT.md).* diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 4617c062..8744abd7 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -249,7 +249,7 @@ Orchestration and core algorithms: Strategy pattern implementations for each attribute: - **BaseAssessor**: Abstract base class defining interface -- Concrete assessors: `CLAUDEmdAssessor`, `READMEAssessor`, etc. +- Concrete assessors: `AgentInstructionsAssessor`, `READMEAssessor`, etc. **Design Principles**: diff --git a/docs/user-guide.md b/docs/user-guide.md index 0bb7b19c..d3e9a56a 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -291,7 +291,7 @@ Create `.agentready-config.yaml` to customize: ```yaml # Custom attribute weights (must sum to 1.0) weights: - claude_md_file: 0.15 # Increase from default 0.10 + agent_instructions: 0.15 # Increase from default 0.10 readme_structure: 0.12 type_annotations: 0.08 diff --git a/experiments/configs/claude-md.yaml b/experiments/configs/claude-md.yaml index 2ae7fd0e..bc0b69d1 100644 --- a/experiments/configs/claude-md.yaml +++ b/experiments/configs/claude-md.yaml @@ -4,4 +4,4 @@ agentready_changes: align: enabled: true attributes: - - claude_md_file + - agent_instructions diff --git a/experiments/configs/tier1.yaml b/experiments/configs/tier1.yaml index ef1d201e..196daaae 100644 --- a/experiments/configs/tier1.yaml +++ b/experiments/configs/tier1.yaml @@ -4,7 +4,7 @@ agentready_changes: align: enabled: true attributes: - - claude_md_file + - agent_instructions - readme_structure - type_annotations - standard_layout diff --git a/scripts/generate_slides.py b/scripts/generate_slides.py index 51d837d2..44bdef1e 100755 --- a/scripts/generate_slides.py +++ b/scripts/generate_slides.py @@ -146,7 +146,7 @@ def create_presentation_slides() -> List[Dict[str, str]]: .agentready/eval_harness/ ā”œā”€ā”€ baseline/summary.json ā”œā”€ā”€ assessors/ -ā”‚ ā””ā”€ā”€ claude_md_file/ +ā”‚ ā””ā”€ā”€ agent_instructions/ ā”‚ ā”œā”€ā”€ impact.json ā† Delta, p-value, effect size ā”‚ ā””ā”€ā”€ run_*.json ā””ā”€ā”€ summary.json ā† Ranked results @@ -192,7 +192,7 @@ def create_presentation_slides() -> List[Dict[str, str]]: # 2. Test single assessor agentready eval-harness test-assessor \\ - --assessor-id claude_md_file --iterations 3 + --assessor-id agent_instructions --iterations 3 # 3. Aggregate all results agentready eval-harness summarize diff --git a/src/agentready/assessors/__init__.py b/src/agentready/assessors/__init__.py index 857b7a75..904e5763 100644 --- a/src/agentready/assessors/__init__.py +++ b/src/agentready/assessors/__init__.py @@ -20,8 +20,8 @@ DbtProjectStructureAssessor, ) from .documentation import ( + AgentInstructionsAssessor, ArchitectureDecisionsAssessor, - CLAUDEmdAssessor, InlineDocumentationAssessor, OpenAPISpecsAssessor, READMEAssessor, @@ -69,7 +69,7 @@ def create_all_assessors() -> list[BaseAssessor]: # Tier 1 Essential ā€” 59% total (9 attributes) TestExecutionAssessor(), # 12% TypeAnnotationsAssessor(), # 10% - CLAUDEmdAssessor(), # 7% + AgentInstructionsAssessor(), # 7% CIQualityGatesAssessor(), # 5% SingleFileVerificationAssessor(), # 5% READMEAssessor(), # 5% diff --git a/src/agentready/assessors/base.py b/src/agentready/assessors/base.py index 0883671e..fc6c2966 100644 --- a/src/agentready/assessors/base.py +++ b/src/agentready/assessors/base.py @@ -20,7 +20,7 @@ class BaseAssessor(ABC): @property @abstractmethod def attribute_id(self) -> str: - """Unique attribute identifier (e.g., 'claude_md_file'). + """Unique attribute identifier (e.g., 'agent_instructions'). Must be lowercase snake_case matching the attribute ID in the research report and default-weights.yaml. diff --git a/src/agentready/assessors/documentation.py b/src/agentready/assessors/documentation.py index 70757b95..fee3fb83 100644 --- a/src/agentready/assessors/documentation.py +++ b/src/agentready/assessors/documentation.py @@ -14,8 +14,8 @@ from .base import BaseAssessor -class CLAUDEmdAssessor(BaseAssessor): - """Assesses presence and quality of CLAUDE.md/AGENTS.md configuration file. +class AgentInstructionsAssessor(BaseAssessor): + """Assesses presence and quality of agent instruction files (CLAUDE.md/AGENTS.md). Tier 1 Essential (7% weight). Context files help agents understand project conventions, but ETH Zurich (Feb 2026) found: auto-generated files hurt @@ -25,7 +25,7 @@ class CLAUDEmdAssessor(BaseAssessor): @property def attribute_id(self) -> str: - return "claude_md_file" + return "agent_instructions" @property def tier(self) -> int: @@ -35,7 +35,7 @@ def tier(self) -> int: def attribute(self) -> Attribute: return Attribute( id=self.attribute_id, - name="CLAUDE.md Configuration Files", + name="Agent Instruction Files", category="Context Window Optimization", tier=self.tier, description="Project-specific configuration for AI coding agents", @@ -46,39 +46,48 @@ def attribute(self) -> Attribute: def assess(self, repository: Repository) -> Finding: """Check for CLAUDE.md file in repository root. - Pass criteria: - - CLAUDE.md exists with >50 bytes, OR - - CLAUDE.md is a symlink to a file with >50 bytes, OR - - CLAUDE.md contains @ reference to a file with >50 bytes, OR - - AGENTS.md exists with >50 bytes (alternative) + Two-phase scoring: + + Phase 1 - Presence (up to 70 points): + - CLAUDE.md exists with >50 bytes (direct, symlink, or @ reference) + - .claude/CLAUDE.md exists with >50 bytes + - AGENTS.md exists with >50 bytes (cross-tool alternative) + - 25 if file exists but is minimal, 0 if missing + + Phase 2 - Length (up to 30 points, only if presence passes): + - <=150 lines: 30 (full credit) + - 151-300 lines: 15 (partial credit) + - >300 lines: 0 (exceeds recommended limit) + + Also checks for agent access documentation (substantiating evidence). Security: - @ references are restricted to relative paths within repository - Path traversal attempts (../) and absolute paths are rejected - - Scoring: - - 100 if CLAUDE.md passes (direct, symlink, or @ reference) - - 90 if AGENTS.md exists without CLAUDE.md - - 25 if CLAUDE.md exists but is minimal without valid references - - 0 if neither file exists """ claude_md_path = repository.path / "CLAUDE.md" agents_md_path = repository.path / "AGENTS.md" - # Check for CLAUDE.md first + # Phase 1: Determine presence and read content + resolved_content = None + base_score = 0.0 + evidence = [] + measured_value = "missing" + try: - # Resolve symlinks if CLAUDE.md is a symlink resolved_path = claude_md_path.resolve(strict=True) is_symlink = claude_md_path.is_symlink() with open(resolved_path, "r", encoding="utf-8") as f: - content = f.read() + file_content = f.read() - size = len(content) + size = len(file_content) - # Check if file has sufficient content if size >= 50: - evidence = [f"CLAUDE.md found at {claude_md_path}"] + resolved_content = file_content + base_score = 100.0 + measured_value = "present" + evidence.append(f"CLAUDE.md found at {claude_md_path}") if is_symlink: target = ( resolved_path.relative_to(repository.path) @@ -86,140 +95,124 @@ def assess(self, repository: Repository) -> Finding: else resolved_path ) evidence.append(f"Symlink to {target} ({size} bytes)") - - # Bonus: Check if AGENTS.md also exists if self._check_agents_md_exists(agents_md_path): evidence.append("AGENTS.md also present (cross-tool compatibility)") - - return Finding( - attribute=self.attribute, - status="pass", - score=100.0, - measured_value="present", - threshold="present", - evidence=evidence, - remediation=None, - error_message=None, - ) - - # File is small - check for @ references - referenced_file = self._extract_at_reference(content) - if referenced_file: - ref_path = repository.path / referenced_file - ref_content, ref_size = self._read_referenced_file(ref_path) - - if ref_content and ref_size >= 50: - evidence = [ - f"CLAUDE.md found with @ reference to {referenced_file}", - f"Referenced file contains {ref_size} bytes", - ] - - # Bonus: Check if AGENTS.md also exists - if self._check_agents_md_exists(agents_md_path): + else: + referenced_file = self._extract_at_reference(file_content) + if referenced_file: + ref_path = repository.path / referenced_file + ref_content, ref_size = self._read_referenced_file(ref_path) + if ref_content and ref_size >= 50: + resolved_content = ref_content + base_score = 100.0 + measured_value = f"@ reference to {referenced_file}" evidence.append( - "AGENTS.md also present (cross-tool compatibility)" + f"CLAUDE.md found with @ reference to {referenced_file}" + ) + evidence.append(f"Referenced file contains {ref_size} bytes") + if self._check_agents_md_exists(agents_md_path): + evidence.append( + "AGENTS.md also present (cross-tool compatibility)" + ) + else: + base_score = 25.0 + measured_value = f"{size} bytes, invalid @ reference" + evidence.append( + f"CLAUDE.md exists but is minimal ({size} bytes)" + ) + evidence.append( + f"@ reference to {referenced_file} but file is missing or too small" ) - - return Finding( - attribute=self.attribute, - status="pass", - score=100.0, - measured_value=f"@ reference to {referenced_file}", - threshold="present", - evidence=evidence, - remediation=None, - error_message=None, - ) else: - # Referenced file doesn't exist or is too small - return Finding( - attribute=self.attribute, - status="fail", - score=25.0, - measured_value=f"{size} bytes, invalid @ reference", - threshold=">50 bytes or valid @ reference", - evidence=[ - f"CLAUDE.md exists but is minimal ({size} bytes)", - f"@ reference to {referenced_file} but file is missing or too small", - ], - remediation=self._create_remediation(), - error_message=None, - ) - - # File is small and no valid @ reference - return Finding( - attribute=self.attribute, - status="fail", - score=25.0, - measured_value=f"{size} bytes", - threshold=">50 bytes", - evidence=[f"CLAUDE.md exists but is minimal ({size} bytes)"], - remediation=self._create_remediation(), - error_message=None, - ) + base_score = 25.0 + measured_value = f"{size} bytes" + evidence.append(f"CLAUDE.md exists but is minimal ({size} bytes)") except FileNotFoundError: - # CLAUDE.md not found at root - check .claude/CLAUDE.md dotclaude_md_path = repository.path / ".claude" / "CLAUDE.md" dotclaude_content, dotclaude_size = self._read_referenced_file( dotclaude_md_path ) if dotclaude_content and dotclaude_size >= 50: - evidence = [ - "CLAUDE.md not found at repository root", - f".claude/CLAUDE.md found with {dotclaude_size} bytes", - ] + resolved_content = dotclaude_content + base_score = 100.0 + measured_value = ".claude/CLAUDE.md present" + evidence.append("CLAUDE.md not found at repository root") + evidence.append(f".claude/CLAUDE.md found with {dotclaude_size} bytes") if self._check_agents_md_exists(agents_md_path): evidence.append("AGENTS.md also present (cross-tool compatibility)") - return Finding( - attribute=self.attribute, - status="pass", - score=100.0, - measured_value=".claude/CLAUDE.md present", - threshold="CLAUDE.md or AGENTS.md", - evidence=evidence, - remediation=None, - error_message=None, - ) - - # Check for AGENTS.md as alternative - agents_content, agents_size = self._read_referenced_file(agents_md_path) - - if agents_content and agents_size >= 50: - return Finding( - attribute=self.attribute, - status="pass", - score=100.0, # AGENTS.md is the cross-tool standard (60k+ repos) - measured_value="AGENTS.md present", - threshold="CLAUDE.md or AGENTS.md", - evidence=[ - "CLAUDE.md not found", - f"AGENTS.md found with {agents_size} bytes", - "AGENTS.md is the cross-tool standard supported by Claude Code, Copilot, Cursor, Codex, and Gemini CLI", - ], - remediation=None, - error_message=None, - ) + else: + agents_content, agents_size = self._read_referenced_file(agents_md_path) + if agents_content and agents_size >= 50: + resolved_content = agents_content + base_score = 100.0 + measured_value = "AGENTS.md present" + evidence.extend( + [ + "CLAUDE.md not found", + f"AGENTS.md found with {agents_size} bytes", + "AGENTS.md is the cross-tool standard supported by Claude Code, Copilot, Cursor, Codex, and Gemini CLI", + ] + ) + else: + base_score = 0.0 + measured_value = "missing" + evidence.extend( + [ + "CLAUDE.md not found in repository root", + "AGENTS.md not found (alternative)", + ] + ) + except OSError as e: + return Finding.error( + self.attribute, reason=f"Could not read CLAUDE.md file: {e}" + ) - # Neither file exists + # File missing or minimal: return early without quality checks + if base_score < 50: return Finding( attribute=self.attribute, status="fail", - score=0.0, - measured_value="missing", - threshold="present", - evidence=[ - "CLAUDE.md not found in repository root", - "AGENTS.md not found (alternative)", - ], + score=base_score, + measured_value=measured_value, + threshold=">50 bytes, <=150 lines recommended", + evidence=evidence, remediation=self._create_remediation(), error_message=None, ) - except OSError as e: - return Finding.error( - self.attribute, reason=f"Could not read CLAUDE.md file: {e}" + + # Phase 2: Length validation (applies to all repos) + line_count = len(resolved_content.splitlines()) if resolved_content else 0 + if line_count <= 150: + length_score = 30.0 + evidence.append(f"Context file is {line_count} lines (good: <=150)") + elif line_count <= 300: + length_score = 15.0 + evidence.append( + f"Context file is {line_count} lines (partial credit: <=300)" ) + else: + length_score = 0.0 + evidence.append( + f"Context file is {line_count} lines (exceeds 300 line limit, consider splitting into .claude/skills/)" + ) + + final_score = min(70.0 + length_score, 100.0) + + # Phase 3: Agent access documentation (substantiating evidence only) + self._check_agent_access(resolved_content, evidence) + + return Finding( + attribute=self.attribute, + status="pass", + score=final_score, + measured_value=measured_value, + threshold=">50 bytes, <=150 lines recommended", + evidence=evidence, + remediation=None, + error_message=None, + ) def _extract_at_reference(self, content: str) -> str | None: """Extract @ reference from CLAUDE.md content. @@ -263,6 +256,43 @@ def _check_agents_md_exists(self, agents_md_path: Path) -> bool: content, size = self._read_referenced_file(agents_md_path) return content is not None and size >= 50 + def _check_agent_access(self, content: str, evidence: list[str]) -> None: + """Check for agent access documentation in context file content. + + Substantiating evidence only, not a hard gate. Looks for access-related + headings or co-occurrence of platform and tool/auth keywords. + """ + if not content: + return + + access_heading = re.compile( + r"^#{1,3}\s.*(agent\s+access|repository\s+access|repo\s+access|platform)", + re.IGNORECASE | re.MULTILINE, + ) + if access_heading.search(content): + evidence.append("Agent access documentation found") + return + + content_lower = content.lower() + platform_keywords = ["github", "gitlab", "bitbucket", "azure devops"] + tool_auth_keywords = [ + "gh ", + "glab", + "authentication", + "token", + "credential", + "vpn", + "cli tool", + ] + + has_platform = any(kw in content_lower for kw in platform_keywords) + has_tool_auth = any(kw in content_lower for kw in tool_auth_keywords) + + if has_platform and has_tool_auth: + evidence.append( + "Agent access documentation found (platform and tool/auth references)" + ) + def _create_remediation(self) -> Remediation: """Create remediation guidance for missing/inadequate CLAUDE.md.""" return Remediation( @@ -272,10 +302,12 @@ def _create_remediation(self) -> Remediation: " Option 1: Create standalone CLAUDE.md (>50 bytes) with project context", " Option 2: Create AGENTS.md and symlink CLAUDE.md to it (cross-tool compatibility)", " Option 3: Create AGENTS.md and reference it with @AGENTS.md in minimal CLAUDE.md", + "Keep the file under 150 lines (hard cap: 300 lines)", "Add project overview and purpose", "Document key architectural patterns", "Specify coding standards and conventions", "Include build/test/deployment commands", + "Add agent access info (platform, CLI tool, auth requirements) if applicable", "Add any project-specific context that helps AI assistants", ], tools=[], diff --git a/src/agentready/assessors/patterns.py b/src/agentready/assessors/patterns.py index 2be78e15..406c254d 100644 --- a/src/agentready/assessors/patterns.py +++ b/src/agentready/assessors/patterns.py @@ -39,15 +39,24 @@ def assess(self, repository: Repository) -> Finding: """Check for pattern references in skills directories and context files.""" score = 0.0 evidence = [] + has_skills = False - # Check for .claude/skills/ directory + # Check for .claude/skills/ directory with tiered scoring skills_dir = repository.path / ".claude" / "skills" if skills_dir.exists() and skills_dir.is_dir(): skill_files = list(skills_dir.rglob("SKILL.md")) - if skill_files: + skill_count = len(skill_files) + if skill_count >= 3: + has_skills = True score += 60.0 evidence.append( - f".claude/skills/ directory with {len(skill_files)} SKILL.md file(s)" + f".claude/skills/ directory with {skill_count} SKILL.md file(s)" + ) + elif skill_count > 0: + has_skills = True + score += 30.0 + evidence.append( + f".claude/skills/ directory with {skill_count} SKILL.md file(s) (3+ recommended for full credit)" ) # Check for pattern references in context files @@ -94,6 +103,22 @@ def assess(self, repository: Repository) -> Finding: score = min(score, 100.0) + # Context file length correlation: warn if >150 lines with no skills + if not has_skills: + for ctx_name in ["CLAUDE.md", "AGENTS.md"]: + ctx_path = repository.path / ctx_name + if ctx_path.exists(): + try: + ctx_lines = len( + ctx_path.read_text(encoding="utf-8").splitlines() + ) + if ctx_lines > 150: + evidence.append( + f"{ctx_name} is {ctx_lines} lines with no skills; consider extracting patterns into .claude/skills/" + ) + except (OSError, UnicodeDecodeError): + pass + if score >= 40: return Finding( attribute=self.attribute, diff --git a/src/agentready/cli/align.py b/src/agentready/cli/align.py index e8222bb7..02f1b7ed 100644 --- a/src/agentready/cli/align.py +++ b/src/agentready/cli/align.py @@ -149,11 +149,20 @@ def align(repository, dry_run, attributes, interactive): failing_ids = { f.attribute.id for f in assessment.findings if f.status == "fail" } - if "claude_md_file" in failing_ids: - click.echo( - "\nšŸ’” Tip: Install the Claude CLI and set ANTHROPIC_API_KEY to " - "enable automatic CLAUDE.md generation." + if "agent_instructions" in failing_ids: + instruction_finding = next( + ( + f + for f in assessment.findings + if f.attribute.id == "agent_instructions" and f.status == "fail" + ), + None, ) + if instruction_finding and instruction_finding.measured_value == "missing": + click.echo( + "\nšŸ’” Tip: Install the Claude CLI and set ANTHROPIC_API_KEY to " + "enable automatic CLAUDE.md generation." + ) sys.exit(0) # Show fix plan @@ -201,7 +210,7 @@ def align(repository, dry_run, attributes, interactive): click.echo(f"\nšŸ”Ø Applying {len(fixes_to_apply)} fixes...\n") def progress_callback(fix, phase: str, success: bool | None) -> None: - if fix.attribute_id == "claude_md_file" and phase == "before": + if fix.attribute_id == "agent_instructions" and phase == "before": click.echo(" Generating CLAUDE.md file...") results = fixer_service.apply_fixes( diff --git a/src/agentready/data/.agentready-config.example.yaml b/src/agentready/data/.agentready-config.example.yaml index ca1e2ae0..1cd91bd0 100644 --- a/src/agentready/data/.agentready-config.example.yaml +++ b/src/agentready/data/.agentready-config.example.yaml @@ -14,7 +14,7 @@ weights: # Tier 1 (Essential) - 59% total test_execution: 0.12 # Test execution & coverage (highest priority) type_annotations: 0.10 # Type hints in code - claude_md_file: 0.07 # CLAUDE.md/AGENTS.md configuration files + agent_instructions: 0.07 # Agent instruction files (CLAUDE.md/AGENTS.md) ci_quality_gates: 0.05 # CI quality gates (lint + type-check + tests) single_file_verification: 0.05 # Single-file lint/type-check commands readme_structure: 0.05 # README structure and content @@ -84,5 +84,5 @@ report_theme: default # Other attributes are automatically rescaled to maintain sum of 1.0 # # weights: -# claude_md_file: 0.15 +# agent_instructions: 0.15 # test_execution: 0.15 diff --git a/src/agentready/data/default-weights.yaml b/src/agentready/data/default-weights.yaml index 18e4afde..ee985a03 100644 --- a/src/agentready/data/default-weights.yaml +++ b/src/agentready/data/default-weights.yaml @@ -28,7 +28,7 @@ # Tier 1 (Essential) - 59% total weight test_execution: 0.12 # 5.1 - Test Execution & Coverage type_annotations: 0.10 # 3.3 - Type Annotations (structural signals for agents) -claude_md_file: 0.07 # 1.1 - CLAUDE.md/AGENTS.md Configuration Files +agent_instructions: 0.07 # 1.1 - Agent Instruction Files (CLAUDE.md/AGENTS.md) ci_quality_gates: 0.05 # 16.2 - CI Quality Gates (lint + type-check + tests on PR) single_file_verification: 0.05 # 16.1 - Single-File Verification (fast feedback loops) readme_structure: 0.05 # 2.1 - README Structure diff --git a/src/agentready/fixers/base.py b/src/agentready/fixers/base.py index b03a7547..7bee1f3c 100644 --- a/src/agentready/fixers/base.py +++ b/src/agentready/fixers/base.py @@ -20,7 +20,7 @@ class BaseFixer(ABC): @property @abstractmethod def attribute_id(self) -> str: - """Unique attribute identifier (e.g., 'claude_md_file'). + """Unique attribute identifier (e.g., 'agent_instructions'). Must match the attribute ID from assessors. """ diff --git a/src/agentready/fixers/documentation.py b/src/agentready/fixers/documentation.py index 80a7615a..874509af 100644 --- a/src/agentready/fixers/documentation.py +++ b/src/agentready/fixers/documentation.py @@ -112,7 +112,7 @@ class CLAUDEmdFixer(BaseFixer): @property def attribute_id(self) -> str: """Return attribute ID.""" - return "claude_md_file" + return "agent_instructions" def can_fix(self, finding: Finding) -> bool: """Check if CLAUDE.md is missing.""" diff --git a/src/agentready/github/review_formatter.py b/src/agentready/github/review_formatter.py index 011074c0..cd5ae48e 100644 --- a/src/agentready/github/review_formatter.py +++ b/src/agentready/github/review_formatter.py @@ -107,7 +107,7 @@ def map_finding_to_attribute( "test coverage": "test_execution", "pytest": "test_execution", "missing test": "test_execution", - "claude.md": "claude_md_file", + "claude.md": "agent_instructions", "documentation": "readme_file", "readme": "readme_file", "conventional commit": "conventional_commits", diff --git a/src/agentready/models/attribute.py b/src/agentready/models/attribute.py index 9c8612ad..d520f774 100644 --- a/src/agentready/models/attribute.py +++ b/src/agentready/models/attribute.py @@ -8,8 +8,8 @@ class Attribute: """Defines an agent-ready quality attribute from the research report. Attributes: - id: Unique identifier (e.g., "claude_md_file", "test_execution") - name: Human-readable name (e.g., "CLAUDE.md Configuration Files") + id: Unique identifier (e.g., "agent_instructions", "test_execution") + name: Human-readable name (e.g., "Agent Instruction Files") category: Research report section (e.g., "Context Window Optimization") tier: Priority tier 1-4 (1=Essential, 4=Advanced) description: What this attribute measures diff --git a/src/agentready/models/eval_harness.py b/src/agentready/models/eval_harness.py index 0619dc75..9bbf30d3 100644 --- a/src/agentready/models/eval_harness.py +++ b/src/agentready/models/eval_harness.py @@ -144,7 +144,7 @@ class AssessorImpact: with statistical significance testing. Attributes: - assessor_id: Attribute ID (e.g., 'claude_md_file') + assessor_id: Attribute ID (e.g., 'agent_instructions') assessor_name: Human-readable name tier: Tier 1-4 from research report diff --git a/src/agentready/services/llm_cache.py b/src/agentready/services/llm_cache.py index 31d0235f..ff7809cc 100644 --- a/src/agentready/services/llm_cache.py +++ b/src/agentready/services/llm_cache.py @@ -122,7 +122,7 @@ def generate_key(attribute_id: str, score: float, evidence_hash: str) -> str: """Generate cache key from finding attributes. Args: - attribute_id: Attribute ID (e.g., "claude_md_file") + attribute_id: Attribute ID (e.g., "agent_instructions") score: Finding score evidence_hash: Hash of evidence list diff --git a/tests/e2e/test_critical_paths.py b/tests/e2e/test_critical_paths.py index 5b578173..415f2fa9 100644 --- a/tests/e2e/test_critical_paths.py +++ b/tests/e2e/test_critical_paths.py @@ -279,7 +279,7 @@ def test_assess_with_valid_config(self): config_file = Path(tmp_dir) / "config.yaml" config_file.write_text(""" weights: - claude_md_file: 2.0 + agent_instructions: 2.0 excluded_attributes: - openapi_specs """) diff --git a/tests/integration/test_scan_workflow.py b/tests/integration/test_scan_workflow.py index 7eeb2627..ce01a7f7 100644 --- a/tests/integration/test_scan_workflow.py +++ b/tests/integration/test_scan_workflow.py @@ -2,7 +2,7 @@ from pathlib import Path -from agentready.assessors.documentation import CLAUDEmdAssessor, READMEAssessor +from agentready.assessors.documentation import AgentInstructionsAssessor, READMEAssessor from agentready.models.config import Config from agentready.models.theme import Theme from agentready.reporters.html import HTMLReporter @@ -21,7 +21,7 @@ def test_scan_current_repository(self): scanner = Scanner(repo_path, config=None) # Use minimal assessors for faster test - assessors = [CLAUDEmdAssessor(), READMEAssessor()] + assessors = [AgentInstructionsAssessor(), READMEAssessor()] # Run scan assessment = scanner.scan(assessors, verbose=False) @@ -44,7 +44,7 @@ def test_html_report_generation(self, tmp_path): repo_path = Path(__file__).parent.parent.parent scanner = Scanner(repo_path, config=None) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report @@ -67,7 +67,7 @@ def test_markdown_report_generation(self, tmp_path): repo_path = Path(__file__).parent.parent.parent scanner = Scanner(repo_path, config=None) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate Markdown report @@ -100,7 +100,7 @@ def test_html_report_with_light_theme(self, tmp_path): ) scanner = Scanner(repo_path, config=config) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report @@ -131,7 +131,7 @@ def test_html_report_with_dark_theme(self, tmp_path): ) scanner = Scanner(repo_path, config=config) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report @@ -180,7 +180,7 @@ def test_html_report_with_custom_theme(self, tmp_path): ) scanner = Scanner(repo_path, config=config) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report @@ -202,7 +202,7 @@ def test_html_report_theme_switcher_present(self, tmp_path): repo_path = Path(__file__).parent.parent.parent scanner = Scanner(repo_path, config=None) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report @@ -224,7 +224,7 @@ def test_html_report_all_themes_embedded(self, tmp_path): repo_path = Path(__file__).parent.parent.parent scanner = Scanner(repo_path, config=None) - assessors = [CLAUDEmdAssessor()] + assessors = [AgentInstructionsAssessor()] assessment = scanner.scan(assessors, verbose=False) # Generate HTML report diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py index 46d79f26..ddc3b983 100644 --- a/tests/unit/cli/test_main.py +++ b/tests/unit/cli/test_main.py @@ -250,7 +250,7 @@ def test_assess_with_config_file(self, runner, test_repo, mock_assessment): """Test assess with custom config file.""" # Create config file config_file = test_repo / "test-config.yaml" - config_file.write_text("weights:\n claude_md_file: 1.0\n") + config_file.write_text("weights:\n agent_instructions: 1.0\n") with patch("agentready.cli.main.Scanner") as mock_scanner_class: mock_scanner = MagicMock() @@ -357,7 +357,7 @@ def test_load_config_valid_yaml(self, tmp_path): config_file = tmp_path / "config.yaml" config_file.write_text(""" weights: - claude_md_file: 2.0 + agent_instructions: 2.0 excluded_attributes: - test_attribute """) @@ -365,7 +365,7 @@ def test_load_config_valid_yaml(self, tmp_path): config = load_config(config_file) assert isinstance(config, Config) - assert config.weights["claude_md_file"] == 2.0 + assert config.weights["agent_instructions"] == 2.0 assert "test_attribute" in config.excluded_attributes def test_load_config_empty(self, tmp_path): diff --git a/tests/unit/test_assessors_documentation.py b/tests/unit/test_assessors_documentation.py index 4b0a7d2f..9f81b17c 100644 --- a/tests/unit/test_assessors_documentation.py +++ b/tests/unit/test_assessors_documentation.py @@ -1,11 +1,11 @@ """Tests for documentation assessors.""" -from agentready.assessors.documentation import CLAUDEmdAssessor +from agentready.assessors.documentation import AgentInstructionsAssessor from agentready.models.repository import Repository -class TestCLAUDEmdAssessor: - """Test CLAUDEmdAssessor.""" +class TestAgentInstructionsAssessor: + """Test AgentInstructionsAssessor.""" def test_passes_with_sufficient_claude_md(self, tmp_path): """Test that assessor passes with CLAUDE.md file >50 bytes.""" @@ -29,7 +29,7 @@ def test_passes_with_sufficient_claude_md(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -62,7 +62,7 @@ def test_passes_with_claude_md_symlink(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -95,7 +95,7 @@ def test_passes_with_at_reference_to_agents_md(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -128,7 +128,7 @@ def test_passes_with_at_reference_with_space(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -163,7 +163,7 @@ def test_passes_with_at_reference_in_subdirectory(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -190,7 +190,7 @@ def test_fails_with_invalid_at_reference(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "fail" @@ -218,7 +218,7 @@ def test_fails_with_minimal_claude_md_no_reference(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "fail" @@ -248,7 +248,7 @@ def test_passes_with_agents_md_only(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -281,7 +281,7 @@ def test_passes_with_dotclaude_claude_md(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -306,7 +306,7 @@ def test_fails_with_no_files(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "fail" @@ -340,7 +340,7 @@ def test_bonus_points_for_both_files(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -349,7 +349,7 @@ def test_bonus_points_for_both_files(self, tmp_path): def test_at_reference_extraction_various_formats(self): """Test _extract_at_reference method with various formats.""" - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() # Test basic @ reference assert assessor._extract_at_reference("@AGENTS.md") == "AGENTS.md" @@ -407,7 +407,7 @@ def test_at_reference_with_at_reference_and_agents_md(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) assert finding.status == "pass" @@ -435,7 +435,7 @@ def test_rejects_path_traversal_attempts(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) # Should fail with score 25 (minimal CLAUDE.md, no valid reference) @@ -463,10 +463,226 @@ def test_rejects_absolute_path_references(self, tmp_path): total_lines=100, ) - assessor = CLAUDEmdAssessor() + assessor = AgentInstructionsAssessor() finding = assessor.assess(repo) # Should fail with score 25 (minimal CLAUDE.md, no valid reference) assert finding.status == "fail" assert finding.score == 25.0 assert finding.remediation is not None + + # --- Length validation tests (ADR A.1) --- + + def test_length_full_credit_under_150_lines(self, tmp_path): + """Test full length credit for context file <=150 lines.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + lines = ["# Project Config"] + [f"Line {i}" for i in range(100)] + (tmp_path / "CLAUDE.md").write_text("\n".join(lines)) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + assert any("good: <=150" in e for e in finding.evidence) + + def test_length_partial_credit_150_to_300_lines(self, tmp_path): + """Test partial length credit for context file 151-300 lines.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + lines = ["# Project Config"] + [f"Line {i}" for i in range(200)] + (tmp_path / "CLAUDE.md").write_text("\n".join(lines)) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 85.0 + assert any("partial credit: <=300" in e for e in finding.evidence) + + def test_length_no_credit_over_300_lines(self, tmp_path): + """Test no length credit for context file >300 lines.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + lines = ["# Project Config"] + [f"Line {i}" for i in range(400)] + (tmp_path / "CLAUDE.md").write_text("\n".join(lines)) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 70.0 + assert any("exceeds 300 line limit" in e for e in finding.evidence) + + def test_length_check_follows_symlink(self, tmp_path): + """Test that length check counts lines of the resolved symlink target.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + lines = ["# Agent Config"] + [f"Line {i}" for i in range(250)] + (tmp_path / "AGENTS.md").write_text("\n".join(lines)) + (tmp_path / "CLAUDE.md").symlink_to("AGENTS.md") + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 85.0 + assert any("partial credit" in e for e in finding.evidence) + + def test_length_check_follows_at_reference(self, tmp_path): + """Test that length check counts lines of the @ referenced file.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + lines = ["# Agent Config"] + [f"Line {i}" for i in range(350)] + (tmp_path / "AGENTS.md").write_text("\n".join(lines)) + (tmp_path / "CLAUDE.md").write_text("@AGENTS.md") + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 70.0 + assert any("exceeds 300 line limit" in e for e in finding.evidence) + + # --- Agent access documentation tests (ADR A.9) --- + + def test_agent_access_heading_in_evidence(self, tmp_path): + """Test that agent access heading is detected as evidence.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + content = ( + "# Project\n\n## Agent Access\n\nUse `gh` CLI for GitHub operations.\n" + ) + (tmp_path / "CLAUDE.md").write_text(content) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert any("Agent access documentation found" in e for e in finding.evidence) + + def test_agent_access_keywords_in_evidence(self, tmp_path): + """Test that platform + tool/auth keyword co-occurrence is detected.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + content = ( + "# Project\n\nHosted on GitHub. Use `gh ` CLI with token authentication.\n" + ) + (tmp_path / "CLAUDE.md").write_text(content) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert any("Agent access documentation found" in e for e in finding.evidence) + + def test_no_agent_access_no_penalty(self, tmp_path): + """Test that missing agent access documentation does not affect score.""" + git_dir = tmp_path / ".git" + git_dir.mkdir() + + content = "# Project\n\nThis is a Python project using pytest for tests.\n" + (tmp_path / "CLAUDE.md").write_text(content) + + repo = Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=10, + total_lines=100, + ) + + assessor = AgentInstructionsAssessor() + finding = assessor.assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + assert not any("Agent access" in e for e in finding.evidence) diff --git a/tests/unit/test_assessors_patterns.py b/tests/unit/test_assessors_patterns.py index 801e393d..5f66c6af 100644 --- a/tests/unit/test_assessors_patterns.py +++ b/tests/unit/test_assessors_patterns.py @@ -52,8 +52,8 @@ def test_fails_with_no_patterns(self, tmp_path): assert finding.status == "fail" assert finding.score == 0.0 - def test_passes_with_skills_directory(self, tmp_path): - """Test that .claude/skills/ with SKILL.md files passes.""" + def test_partial_credit_with_single_skill(self, tmp_path): + """Test that 1 skill gets partial credit (30 pts) below pass threshold.""" skills_dir = tmp_path / ".claude" / "skills" / "add-endpoint" skills_dir.mkdir(parents=True) (skills_dir / "SKILL.md").write_text( @@ -64,8 +64,8 @@ def test_passes_with_skills_directory(self, tmp_path): assessor = PatternReferencesAssessor() finding = assessor.assess(repo) - assert finding.status == "pass" - assert finding.score >= 60.0 + assert finding.status == "fail" + assert finding.score == 30.0 assert any(".claude/skills/" in e for e in finding.evidence) def test_passes_with_pattern_refs_in_claude_md(self, tmp_path): @@ -110,14 +110,14 @@ def test_examples_directory_adds_score(self, tmp_path): # examples/ alone gives 20 points, which is below pass threshold (40) assert finding.score == 20.0 - def test_skills_plus_examples_caps_at_100(self, tmp_path): - """Test that combined score caps at 100.""" - # Skills dir + def test_skills_plus_examples_combined_score(self, tmp_path): + """Test combined score from skills + examples.""" + # Skills dir (1 skill = 30 pts partial credit) skills_dir = tmp_path / ".claude" / "skills" / "add-endpoint" skills_dir.mkdir(parents=True) (skills_dir / "SKILL.md").write_text("# Pattern\n") - # Examples dir + # Examples dir (+20 pts) examples_dir = tmp_path / "examples" examples_dir.mkdir() (examples_dir / "example.py").write_text("# Example\n") @@ -126,7 +126,7 @@ def test_skills_plus_examples_caps_at_100(self, tmp_path): assessor = PatternReferencesAssessor() finding = assessor.assess(repo) - assert finding.score == 80.0 # 60 + 20 + assert finding.score == 50.0 # 30 + 20 def test_remediation_on_fail(self, tmp_path): """Test that remediation is provided on failure.""" @@ -152,6 +152,89 @@ def test_agents_md_keyword_match(self, tmp_path): assert finding.status == "pass" assert finding.score >= 40.0 + # --- Skill depth tiering tests (ADR A.4) --- + + def test_skills_partial_credit_1_skill(self, tmp_path): + """Test that 1 skill gets partial credit (30 pts, not 60).""" + skills_dir = tmp_path / ".claude" / "skills" / "build" + skills_dir.mkdir(parents=True) + (skills_dir / "SKILL.md").write_text("# Build\n") + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert finding.score == 30.0 + assert any("3+ recommended" in e for e in finding.evidence) + + def test_skills_partial_credit_2_skills(self, tmp_path): + """Test that 2 skills get partial credit (30 pts).""" + for name in ["build", "test"]: + skill_dir = tmp_path / ".claude" / "skills" / name + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text(f"# {name.title()}\n") + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert finding.score == 30.0 + assert any("3+ recommended" in e for e in finding.evidence) + + def test_skills_full_credit_3_plus_skills(self, tmp_path): + """Test that 3+ skills get full credit (60 pts).""" + for name in ["build", "test", "deploy"]: + skill_dir = tmp_path / ".claude" / "skills" / name + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text(f"# {name.title()}\n") + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert finding.score == 60.0 + assert not any("3+ recommended" in e for e in finding.evidence) + + def test_context_file_length_warning_no_skills(self, tmp_path): + """Test warning when context file >150 lines with no skills.""" + lines = ["# Project Config"] + [f"Line {i}" for i in range(200)] + (tmp_path / "CLAUDE.md").write_text("\n".join(lines)) + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert any( + "no skills" in e and "consider extracting" in e for e in finding.evidence + ) + + def test_context_file_length_no_warning_with_skills(self, tmp_path): + """Test no length warning when skills exist.""" + lines = ["# Project Config"] + [f"Line {i}" for i in range(200)] + (tmp_path / "CLAUDE.md").write_text("\n".join(lines)) + + skills_dir = tmp_path / ".claude" / "skills" / "build" + skills_dir.mkdir(parents=True) + (skills_dir / "SKILL.md").write_text("# Build\n") + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert not any( + "no skills" in e and "consider extracting" in e for e in finding.evidence + ) + + def test_context_file_length_no_warning_under_150(self, tmp_path): + """Test no length warning when context file is short.""" + (tmp_path / "CLAUDE.md").write_text("# Short file\nJust a few lines.\n") + + repo = _make_repo(tmp_path) + assessor = PatternReferencesAssessor() + finding = assessor.assess(repo) + + assert not any("consider extracting" in e for e in finding.evidence) + class TestDesignIntentAssessor: """Test DesignIntentAssessor.""" diff --git a/tests/unit/test_batch_assessment.py b/tests/unit/test_batch_assessment.py index 9586b07c..9df99a08 100644 --- a/tests/unit/test_batch_assessment.py +++ b/tests/unit/test_batch_assessment.py @@ -41,7 +41,7 @@ def sample_repository(): def sample_assessment(sample_repository): """Create a sample assessment for testing.""" attribute = Attribute( - id="claude_md_file", + id="agent_instructions", name="CLAUDE.md File", description="Repository has CLAUDE.md", category="Documentation", diff --git a/tests/unit/test_cli_align.py b/tests/unit/test_cli_align.py index 2b750283..5b644929 100644 --- a/tests/unit/test_cli_align.py +++ b/tests/unit/test_cli_align.py @@ -161,7 +161,7 @@ def test_align_with_specific_attributes( mock_fixer.return_value.generate_fix_plan.return_value = mock_fix_plan result = runner.invoke( - align, [str(temp_repo), "--attributes", "claude_md_file,gitignore_file"] + align, [str(temp_repo), "--attributes", "agent_instructions,gitignore_file"] ) # Should succeed @@ -427,7 +427,7 @@ def test_align_fixer_service_error( class TestAlignClaudeMdFileFeatures: - """Test align command features specific to claude_md_file attribute. + """Test align command features specific to agent_instructions attribute. These tests verify the tip message when CLAUDE.md fix is skipped and the progress callback logging for CLAUDE.md generation. @@ -437,15 +437,16 @@ class TestAlignClaudeMdFileFeatures: @patch("agentready.cli.align.Scanner") @patch("agentready.cli.align.Config") @patch("agentready.cli.main.create_all_assessors") - def test_align_echoes_tip_when_no_fixes_and_claude_md_file_failing( + def test_align_echoes_tip_when_no_fixes_and_agent_instructions_failing( self, mock_assessors, mock_config, mock_scanner, mock_fixer, runner, temp_repo ): - """Test that align shows tip when claude_md_file fails but no fix is available.""" - # Setup mock finding with claude_md_file failing + """Test that align shows tip when agent_instructions fails but no fix is available.""" + # Setup mock finding with agent_instructions failing mock_finding = MagicMock() - mock_finding.attribute.id = "claude_md_file" + mock_finding.attribute.id = "agent_instructions" mock_finding.status = "fail" mock_finding.score = 0.0 + mock_finding.measured_value = "missing" mock_assessment = MagicMock() mock_assessment.overall_score = 65.0 @@ -472,13 +473,13 @@ def test_align_echoes_tip_when_no_fixes_and_claude_md_file_failing( @patch("agentready.cli.align.Scanner") @patch("agentready.cli.align.Config") @patch("agentready.cli.main.create_all_assessors") - def test_align_does_not_show_tip_when_claude_md_file_passes( + def test_align_does_not_show_tip_when_agent_instructions_passes( self, mock_assessors, mock_config, mock_scanner, mock_fixer, runner, temp_repo ): - """Test that align does not show tip when claude_md_file passes.""" - # Setup mock finding with claude_md_file passing + """Test that align does not show tip when agent_instructions passes.""" + # Setup mock finding with agent_instructions passing mock_finding = MagicMock() - mock_finding.attribute.id = "claude_md_file" + mock_finding.attribute.id = "agent_instructions" mock_finding.status = "pass" mock_finding.score = 100.0 @@ -518,7 +519,7 @@ def test_align_echoes_generating_claude_md_when_fix_applies( """Test that align echoes 'Generating CLAUDE.md file...' when applying fix.""" # Setup mock finding mock_finding = MagicMock() - mock_finding.attribute.id = "claude_md_file" + mock_finding.attribute.id = "agent_instructions" mock_finding.status = "fail" mock_finding.score = 0.0 @@ -528,9 +529,9 @@ def test_align_echoes_generating_claude_md_when_fix_applies( mock_assessment.repository = MagicMock() mock_scanner.return_value.scan.return_value = mock_assessment - # Setup mock fix for claude_md_file + # Setup mock fix for agent_instructions mock_fix = MagicMock() - mock_fix.attribute_id = "claude_md_file" + mock_fix.attribute_id = "agent_instructions" mock_fix.description = "Run Claude CLI to create CLAUDE.md" mock_fix.preview.return_value = "RUN claude -p ..." mock_fix.points_gained = 10.0 @@ -593,7 +594,7 @@ def test_multiline_preview_indentation( """ # Setup mock assessment mock_finding = MagicMock() - mock_finding.attribute.id = "claude_md_file" + mock_finding.attribute.id = "agent_instructions" mock_finding.status = "fail" mock_finding.score = 0.0 @@ -605,7 +606,7 @@ def test_multiline_preview_indentation( # Create a mock fix with multi-line preview (simulating MultiStepFix) mock_fix = MagicMock() - mock_fix.attribute_id = "claude_md_file" + mock_fix.attribute_id = "agent_instructions" mock_fix.description = ( "Run Claude CLI to create CLAUDE.md, then move content to AGENTS.md" ) @@ -632,7 +633,7 @@ def test_multiline_preview_indentation( assert result.exit_code == 0 # The fix header should be indented with 2 spaces + "1. " - assert " 1. [claude_md_file]" in result.output + assert " 1. [agent_instructions]" in result.output # The "MULTI-STEP FIX" header should be indented with 5 spaces assert " MULTI-STEP FIX (2 steps):" in result.output diff --git a/tests/unit/test_fixers.py b/tests/unit/test_fixers.py index fa664e25..cf195864 100644 --- a/tests/unit/test_fixers.py +++ b/tests/unit/test_fixers.py @@ -44,7 +44,7 @@ def temp_repo(): def claude_md_failing_finding(): """Create a failing finding for CLAUDE.md.""" attribute = Attribute( - id="claude_md_file", + id="agent_instructions", name="CLAUDE.md File", description="Repository has CLAUDE.md", category="Documentation", @@ -122,7 +122,7 @@ class TestCLAUDEmdFixer: def test_attribute_id(self): """Test attribute ID matches.""" fixer = CLAUDEmdFixer() - assert fixer.attribute_id == "claude_md_file" + assert fixer.attribute_id == "agent_instructions" def test_can_fix_failing_finding(self, claude_md_failing_finding): """Test can fix failing CLAUDE.md finding.""" @@ -154,7 +154,7 @@ def test_generate_fix_when_agent_md_missing( assert fix.steps[0].command == _claude_md_command() assert fix.steps[0].working_dir == temp_repo.path assert fix.steps[0].capture_output is False - assert fix.attribute_id == "claude_md_file" + assert fix.attribute_id == "agent_instructions" assert fix.points_gained > 0 assert ( "Move" in fix.steps[1].preview() and "AGENTS.md" in fix.steps[1].preview() @@ -172,7 +172,7 @@ def test_generate_fix_when_agent_md_exists_returns_redirect_only_fix( assert fix is not None assert isinstance(fix, Fix) assert not isinstance(fix, MultiStepFix) - assert fix.attribute_id == "claude_md_file" + assert fix.attribute_id == "agent_instructions" assert fix.points_gained > 0 # Applying the fix should create CLAUDE.md with redirect only result = fix.apply(dry_run=False)