"""Pytest tests for Devin skill validation. Validates that all skills in .devin/skills/ are well-formed: H1 title, "when to invoke" section, prerequisites when commands are referenced, make-target references that exist, and file references that exist. Run with: make pytest TEST=tests/test_skills_validation.py """ from __future__ import annotations import re from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parents[2] # Sections required for every skill REQUIRED_SECTIONS = ["when to invoke"] # Sections required only for skills that reference commands/tools COMMAND_REQUIRED_SECTIONS = ["prerequisites"] # Markers indicating a skill references commands/tools COMMAND_MARKERS = ("`make ", "```bash", "```sh", "curl ", "python ", "python3 ", "ssh ") EXPECTED_SKILLS = [ "dependency-graph", "deployment-coordination", "devx-workflow", "pr-review", "skill-creation", "spec-driven-development", "testing-and-debugging", "vikunja-tasks", ] def _find_skills() -> dict[str, Path]: skills_dir = REPO_ROOT / ".devin" / "skills" assert skills_dir.exists(), ".devin/skills/ directory not found" return {d.name: d / "SKILL.md" for d in skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()} # Skills shared with other repos — file-path references are only checked # in the owning repo (infra), where the referenced files live. SHARED_SKILLS = {"cross-repo-sync", "branch-hygiene", "dependency-graph", "skill-creation"} def _make_targets() -> set[str]: """Collect make targets from Makefile plus included devx .mak files.""" targets: set[str] = set() makefile = REPO_ROOT / "Makefile" if makefile.exists(): targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", makefile.read_text(), re.MULTILINE)) for mak in REPO_ROOT.glob(".venv/lib/python*/site-packages/devx/make/*.mak"): targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", mak.read_text(), re.MULTILINE)) # devx repo: devx.mak lives in the package source (editable install) for mak in REPO_ROOT.glob("src/devx/make/*.mak"): targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", mak.read_text(), re.MULTILINE)) return targets def _validate_skill(skill_name: str, skill_path: Path, make_targets: set[str]) -> list[str]: """Validate a single skill file. Returns list of error messages.""" errors: list[str] = [] content = skill_path.read_text() if not re.search(r"^# ", content, re.MULTILINE): errors.append(f"{skill_name}: missing H1 title") lower = content.lower() for section in REQUIRED_SECTIONS: if f"## {section}" not in lower: errors.append(f"{skill_name}: missing '## {section.title()}' section") references_commands = any(marker in content for marker in COMMAND_MARKERS) if references_commands: for section in COMMAND_REQUIRED_SECTIONS: if f"## {section}" not in lower: errors.append( f"{skill_name}: missing '## {section.title()}' section " "(required because skill references commands/tools)" ) for target in re.findall(r"`make ([a-zA-Z][a-zA-Z0-9_-]*)`", content): if target not in make_targets: errors.append(f"{skill_name}: references `make {target}` but target does not exist") # File-path checks: skip shared skills (checked in infra) and # placeholder paths containing <...> templates. if skill_name not in SHARED_SKILLS: for match in re.findall(r"`((?:scripts|src|ansible|docs|tests|environments)/[^`\s]+)`", content): if "<" in match: continue if not (REPO_ROOT / match).exists(): errors.append(f"{skill_name}: references `{match}` but file does not exist") return errors @pytest.mark.parametrize("skill_name", EXPECTED_SKILLS) def test_skill_exists(skill_name: str) -> None: """Each expected skill must have a SKILL.md.""" skill = REPO_ROOT / ".devin" / "skills" / skill_name / "SKILL.md" assert skill.exists(), f"{skill_name}/SKILL.md not found" def test_minimum_skill_count() -> None: """The repo should carry a working set of skills, not a stub.""" assert len(_find_skills()) >= 7, "expected >=10 skills" def test_all_skills_validate() -> None: """All skills must pass structure/reference validation.""" make_targets = _make_targets() errors: list[str] = [] for skill_name, skill_path in _find_skills().items(): errors.extend(_validate_skill(skill_name, skill_path, make_targets)) assert not errors, "Skill validation failed:\n" + "\n".join(f" - {e}" for e in errors)