Public Access
122 lines
4.6 KiB
Python
122 lines
4.6 KiB
Python
"""Pytest tests for Devin skill validation.
|
|
|
|
Validates that all skills in .devin/skills/ are well-formed: H1 title,
|
|
"when to invoke" section, prerequisites when commands are referenced,
|
|
make-target references that exist, and file references that exist.
|
|
|
|
Run with: make pytest TEST=tests/test_skills_validation.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
# Sections required for every skill
|
|
REQUIRED_SECTIONS = ["when to invoke"]
|
|
|
|
# Sections required only for skills that reference commands/tools
|
|
COMMAND_REQUIRED_SECTIONS = ["prerequisites"]
|
|
|
|
# Markers indicating a skill references commands/tools
|
|
COMMAND_MARKERS = ("`make ", "```bash", "```sh", "curl ", "python ", "python3 ", "ssh ")
|
|
|
|
EXPECTED_SKILLS = [
|
|
"dependency-graph",
|
|
"deployment-coordination",
|
|
"devx-workflow",
|
|
"pr-review",
|
|
"skill-creation",
|
|
"spec-driven-development",
|
|
"testing-and-debugging",
|
|
"vikunja-tasks",
|
|
]
|
|
|
|
|
|
def _find_skills() -> dict[str, Path]:
|
|
skills_dir = REPO_ROOT / ".devin" / "skills"
|
|
assert skills_dir.exists(), ".devin/skills/ directory not found"
|
|
return {d.name: d / "SKILL.md" for d in skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()}
|
|
|
|
|
|
# Skills shared with other repos — file-path references are only checked
|
|
# in the owning repo (infra), where the referenced files live.
|
|
SHARED_SKILLS = {"cross-repo-sync", "branch-hygiene", "dependency-graph", "skill-creation"}
|
|
|
|
|
|
def _make_targets() -> set[str]:
|
|
"""Collect make targets from Makefile plus included devx .mak files."""
|
|
targets: set[str] = set()
|
|
makefile = REPO_ROOT / "Makefile"
|
|
if makefile.exists():
|
|
targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", makefile.read_text(), re.MULTILINE))
|
|
for mak in REPO_ROOT.glob(".venv/lib/python*/site-packages/devx/make/*.mak"):
|
|
targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", mak.read_text(), re.MULTILINE))
|
|
# devx repo: devx.mak lives in the package source (editable install)
|
|
for mak in REPO_ROOT.glob("src/devx/make/*.mak"):
|
|
targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", mak.read_text(), re.MULTILINE))
|
|
return targets
|
|
|
|
|
|
def _validate_skill(skill_name: str, skill_path: Path, make_targets: set[str]) -> list[str]:
|
|
"""Validate a single skill file. Returns list of error messages."""
|
|
errors: list[str] = []
|
|
content = skill_path.read_text()
|
|
|
|
if not re.search(r"^# ", content, re.MULTILINE):
|
|
errors.append(f"{skill_name}: missing H1 title")
|
|
|
|
lower = content.lower()
|
|
for section in REQUIRED_SECTIONS:
|
|
if f"## {section}" not in lower:
|
|
errors.append(f"{skill_name}: missing '## {section.title()}' section")
|
|
|
|
references_commands = any(marker in content for marker in COMMAND_MARKERS)
|
|
if references_commands:
|
|
for section in COMMAND_REQUIRED_SECTIONS:
|
|
if f"## {section}" not in lower:
|
|
errors.append(
|
|
f"{skill_name}: missing '## {section.title()}' section "
|
|
"(required because skill references commands/tools)"
|
|
)
|
|
|
|
for target in re.findall(r"`make ([a-zA-Z][a-zA-Z0-9_-]*)`", content):
|
|
if target not in make_targets:
|
|
errors.append(f"{skill_name}: references `make {target}` but target does not exist")
|
|
|
|
# File-path checks: skip shared skills (checked in infra) and
|
|
# placeholder paths containing <...> templates.
|
|
if skill_name not in SHARED_SKILLS:
|
|
for match in re.findall(r"`((?:scripts|src|ansible|docs|tests|environments)/[^`\s]+)`", content):
|
|
if "<" in match:
|
|
continue
|
|
if not (REPO_ROOT / match).exists():
|
|
errors.append(f"{skill_name}: references `{match}` but file does not exist")
|
|
|
|
return errors
|
|
|
|
|
|
@pytest.mark.parametrize("skill_name", EXPECTED_SKILLS)
|
|
def test_skill_exists(skill_name: str) -> None:
|
|
"""Each expected skill must have a SKILL.md."""
|
|
skill = REPO_ROOT / ".devin" / "skills" / skill_name / "SKILL.md"
|
|
assert skill.exists(), f"{skill_name}/SKILL.md not found"
|
|
|
|
|
|
def test_minimum_skill_count() -> None:
|
|
"""The repo should carry a working set of skills, not a stub."""
|
|
assert len(_find_skills()) >= 7, "expected >=10 skills"
|
|
|
|
|
|
def test_all_skills_validate() -> None:
|
|
"""All skills must pass structure/reference validation."""
|
|
make_targets = _make_targets()
|
|
errors: list[str] = []
|
|
for skill_name, skill_path in _find_skills().items():
|
|
errors.extend(_validate_skill(skill_name, skill_path, make_targets))
|
|
assert not errors, "Skill validation failed:\n" + "\n".join(f" - {e}" for e in errors)
|