GRM-171: docs: add runner-ops, molecule-testing, vikunja-tasks skills, fix create-task docs

tests/unit/test_skills_validation.py

@@ -0,0 +1,120 @@
+"""Pytest tests for Devin skill validation.
+
+Validates that all skills in .devin/skills/ are well-formed: H1 title,
+"when to invoke" section, prerequisites when commands are referenced,
+make-target references that exist, and file references that exist.
+
+Run with: make pytest TEST=tests/test_skills_validation.py
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+# Sections required for every skill
+REQUIRED_SECTIONS = ["when to invoke"]
+
+# Sections required only for skills that reference commands/tools
+COMMAND_REQUIRED_SECTIONS = ["prerequisites"]
+
+# Markers indicating a skill references commands/tools
+COMMAND_MARKERS = ("`make ", "```bash", "```sh", "curl ", "python ", "python3 ", "ssh ")
+
+EXPECTED_SKILLS = [
+    "dependency-graph",
+    "deployment-coordination",
+    "devx-workflow",
+    "molecule-testing",
+    "pr-review",
+    "runner-ops",
+    "skill-creation",
+    "spec-driven-development",
+    "testing-and-debugging",
+    "vikunja-tasks",
+]
+
+
+def _find_skills() -> dict[str, Path]:
+    skills_dir = REPO_ROOT / ".devin" / "skills"
+    assert skills_dir.exists(), ".devin/skills/ directory not found"
+    return {d.name: d / "SKILL.md" for d in skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()}
+
+
+# Skills shared with other repos — file-path references are only checked
+# in the owning repo (infra), where the referenced files live.
+SHARED_SKILLS = {"cross-repo-sync", "branch-hygiene", "dependency-graph", "skill-creation"}
+
+
+def _make_targets() -> set[str]:
+    """Collect make targets from Makefile plus included devx .mak files."""
+    targets: set[str] = set()
+    makefile = REPO_ROOT / "Makefile"
+    if makefile.exists():
+        targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", makefile.read_text(), re.MULTILINE))
+    for mak in REPO_ROOT.glob(".venv/lib/python*/site-packages/devx/make/*.mak"):
+        targets.update(re.findall(r"^([a-zA-Z][a-zA-Z0-9_-]*):", mak.read_text(), re.MULTILINE))
+    return targets
+
+
+def _validate_skill(skill_name: str, skill_path: Path, make_targets: set[str]) -> list[str]:
+    """Validate a single skill file. Returns list of error messages."""
+    errors: list[str] = []
+    content = skill_path.read_text()
+
+    if not re.search(r"^# ", content, re.MULTILINE):
+        errors.append(f"{skill_name}: missing H1 title")
+
+    lower = content.lower()
+    for section in REQUIRED_SECTIONS:
+        if f"## {section}" not in lower:
+            errors.append(f"{skill_name}: missing '## {section.title()}' section")
+
+    references_commands = any(marker in content for marker in COMMAND_MARKERS)
+    if references_commands:
+        for section in COMMAND_REQUIRED_SECTIONS:
+            if f"## {section}" not in lower:
+                errors.append(
+                    f"{skill_name}: missing '## {section.title()}' section "
+                    "(required because skill references commands/tools)"
+                )
+
+    for target in re.findall(r"`make ([a-zA-Z][a-zA-Z0-9_-]*)`", content):
+        if target not in make_targets:
+            errors.append(f"{skill_name}: references `make {target}` but target does not exist")
+
+    # File-path checks: skip shared skills (checked in infra) and
+    # placeholder paths containing <...> templates.
+    if skill_name not in SHARED_SKILLS:
+        for match in re.findall(r"`((?:scripts|src|ansible|docs|tests|environments)/[^`\s]+)`", content):
+            if "<" in match:
+                continue
+            if not (REPO_ROOT / match).exists():
+                errors.append(f"{skill_name}: references `{match}` but file does not exist")
+
+    return errors
+
+
+@pytest.mark.parametrize("skill_name", EXPECTED_SKILLS)
+def test_skill_exists(skill_name: str) -> None:
+    """Each expected skill must have a SKILL.md."""
+    skill = REPO_ROOT / ".devin" / "skills" / skill_name / "SKILL.md"
+    assert skill.exists(), f"{skill_name}/SKILL.md not found"
+
+
+def test_minimum_skill_count() -> None:
+    """The repo should carry a working set of skills, not a stub."""
+    assert len(_find_skills()) >= 8, "expected >=10 skills"
+
+
+def test_all_skills_validate() -> None:
+    """All skills must pass structure/reference validation."""
+    make_targets = _make_targets()
+    errors: list[str] = []
+    for skill_name, skill_path in _find_skills().items():
+        errors.extend(_validate_skill(skill_name, skill_path, make_targets))
+    assert not errors, "Skill validation failed:\n" + "\n".join(f"  - {e}" for e in errors)