Public Access
358 lines
15 KiB
Python
358 lines
15 KiB
Python
"""Structural tests for spec-driven development workflows and skills in devx.
|
|
|
|
These tests parse devx's own workflow YAML files and assert that the
|
|
spec-driven development steps, jobs, and env vars are present and
|
|
correctly wired. They also validate that the skills exist in devx's
|
|
own .devin/skills/ directory with required sections.
|
|
|
|
devx must NOT be aware of other repos (infra, grm, sso-bridge). Those
|
|
repos consume devx; devx does not test them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
# devx repo root
|
|
# __file__ = .../devx/tests/unit/test_spec_driven_workflows.py
|
|
# parents[2] = .../devx
|
|
_DEVX = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _load_workflow(filename: str) -> dict:
|
|
"""Load a devx workflow YAML file and return parsed dict."""
|
|
path = _DEVX / ".gitea" / "workflows" / filename
|
|
if not path.exists():
|
|
pytest.skip(f"Workflow {filename} not found in devx")
|
|
with open(path, encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def _read_skill(skill_name: str) -> str:
|
|
"""Read a skill file from devx's .devin/skills/ directory."""
|
|
skill_path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
|
|
if not skill_path.exists():
|
|
pytest.fail(f"SKILL.md not found for {skill_name} in devx")
|
|
return skill_path.read_text(encoding="utf-8")
|
|
|
|
|
|
def _get_step_names(job: dict) -> list[str]:
|
|
"""Extract step names from a job dict."""
|
|
names = []
|
|
for step in job.get("steps", []):
|
|
if "name" in step:
|
|
names.append(step["name"])
|
|
return names
|
|
|
|
|
|
def _find_step(job: dict, name_part: str) -> dict | None:
|
|
"""Find a step by partial name match."""
|
|
for step in job.get("steps", []):
|
|
if "name" in step and name_part.lower() in step["name"].lower():
|
|
return step
|
|
return None
|
|
|
|
|
|
def _get_run_commands(step: dict) -> str:
|
|
"""Get the run command from a step."""
|
|
return step.get("run", "")
|
|
|
|
|
|
# ============================================================================
|
|
# devx ci.yml — spec validation + PR size
|
|
# ============================================================================
|
|
|
|
|
|
class TestDevxCiWorkflow:
|
|
@pytest.fixture
|
|
def workflow(self) -> dict:
|
|
return _load_workflow("ci.yml")
|
|
|
|
def test_validate_job_exists(self, workflow: dict) -> None:
|
|
assert "validate" in workflow["jobs"]
|
|
|
|
def test_has_spec_validation_step(self, workflow: dict) -> None:
|
|
steps = _get_step_names(workflow["jobs"]["validate"])
|
|
assert any("Validate spec file" in s for s in steps), "validate job must have 'Validate spec file' step"
|
|
|
|
def test_has_pr_size_check_step(self, workflow: dict) -> None:
|
|
steps = _get_step_names(workflow["jobs"]["validate"])
|
|
assert any("Check PR size" in s for s in steps), "validate job must have 'Check PR size' step"
|
|
|
|
def test_spec_validation_uses_correct_module(self, workflow: dict) -> None:
|
|
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
|
|
assert step is not None
|
|
cmd = _get_run_commands(step)
|
|
assert "devx.ci.validate_spec" in cmd
|
|
assert "--github-output" in cmd
|
|
|
|
def test_pr_size_uses_correct_module(self, workflow: dict) -> None:
|
|
step = _find_step(workflow["jobs"]["validate"], "Check PR size")
|
|
assert step is not None
|
|
cmd = _get_run_commands(step)
|
|
assert "devx.ci.check_pr_size" in cmd
|
|
assert "--github-output" in cmd
|
|
|
|
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
|
|
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
|
|
assert step is not None
|
|
env = step.get("env", {})
|
|
assert env.get("DEVX_TASK_PREFIX") == "DEVX"
|
|
|
|
def test_has_auto_merge_job(self, workflow: dict) -> None:
|
|
assert "auto-merge" in workflow["jobs"], "ci.yml must have 'auto-merge' job"
|
|
|
|
|
|
# ============================================================================
|
|
# devx post-merge.yml — release + publish
|
|
# ============================================================================
|
|
|
|
|
|
class TestDevxPostMergeWorkflow:
|
|
@pytest.fixture
|
|
def workflow(self) -> dict:
|
|
return _load_workflow("post-merge.yml")
|
|
|
|
def test_post_merge_workflow_exists(self, workflow: dict) -> None:
|
|
assert workflow is not None
|
|
|
|
def test_has_release_and_maintain_job(self, workflow: dict) -> None:
|
|
assert "release-and-maintain" in workflow["jobs"], "post-merge must have 'release-and-maintain' job"
|
|
|
|
def test_has_publish_step(self, workflow: dict) -> None:
|
|
job = workflow["jobs"].get("release-and-maintain", {})
|
|
steps = _get_step_names(job)
|
|
assert any("publish" in s.lower() for s in steps), "post-merge must have a publish step"
|
|
|
|
|
|
# ============================================================================
|
|
# Skill files — spec-driven-development SKILL.md in devx
|
|
# ============================================================================
|
|
|
|
|
|
class TestSpecDrivenDevelopmentSkill:
|
|
REQUIRED_SECTIONS = [
|
|
"## Overview",
|
|
"## Workflow",
|
|
"## Spec Template",
|
|
"## CI Validation",
|
|
"## Acceptance Criteria",
|
|
]
|
|
|
|
def test_skill_exists_in_repo(self) -> None:
|
|
skill_path = _DEVX / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
|
|
assert skill_path.exists(), "SKILL.md not found in devx"
|
|
|
|
def test_skill_has_required_sections(self) -> None:
|
|
content = _read_skill("spec-driven-development")
|
|
for section in self.REQUIRED_SECTIONS:
|
|
assert section in content, f"SKILL.md missing section: {section}"
|
|
|
|
def test_skill_mentions_req_ids(self) -> None:
|
|
content = _read_skill("spec-driven-development")
|
|
assert "REQ-" in content, "SKILL.md must mention REQ-ID format"
|
|
|
|
def test_skill_mentions_pr_size_limit(self) -> None:
|
|
content = _read_skill("spec-driven-development")
|
|
assert "500" in content, "SKILL.md must mention 500 line PR size limit"
|
|
|
|
def test_skill_mentions_nightly_gate(self) -> None:
|
|
content = _read_skill("spec-driven-development")
|
|
assert "nightly" in content.lower(), "SKILL.md must mention nightly gate"
|
|
|
|
|
|
# ============================================================================
|
|
# devx-workflow skill — exists in devx, mentions spec gates
|
|
# ============================================================================
|
|
|
|
|
|
class TestDevxWorkflowSkill:
|
|
def test_skill_exists(self) -> None:
|
|
path = _DEVX / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
|
|
assert path.exists(), "devx-workflow SKILL.md not found in devx"
|
|
|
|
def test_mentions_spec_validation(self) -> None:
|
|
content = _read_skill("devx-workflow")
|
|
assert "validate_spec" in content, "devx-workflow skill must mention validate_spec"
|
|
|
|
def test_mentions_pr_size_check(self) -> None:
|
|
content = _read_skill("devx-workflow")
|
|
assert "check_pr_size" in content, "devx-workflow skill must mention check_pr_size"
|
|
|
|
def test_mentions_pr_workflow_commands(self) -> None:
|
|
content = _read_skill("devx-workflow")
|
|
assert "make create-pr" in content or "make push-with-pr" in content, (
|
|
"devx-workflow skill must mention PR creation commands"
|
|
)
|
|
|
|
def test_mentions_auto_merge(self) -> None:
|
|
content = _read_skill("devx-workflow")
|
|
assert "auto-merge" in content.lower() or "ready-to-merge" in content, (
|
|
"devx-workflow skill must mention auto-merge"
|
|
)
|
|
|
|
def test_has_correct_task_prefix(self) -> None:
|
|
content = _read_skill("devx-workflow")
|
|
assert "DEVX" in content, "devx-workflow skill must mention task prefix DEVX"
|
|
|
|
|
|
# ============================================================================
|
|
# testing-and-debugging skill — exists in devx, mentions spec workflow
|
|
# ============================================================================
|
|
|
|
|
|
class TestTestingAndDebuggingSkill:
|
|
def test_skill_exists(self) -> None:
|
|
path = _DEVX / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
|
|
assert path.exists(), "testing-and-debugging SKILL.md not found in devx"
|
|
|
|
def test_has_required_sections(self) -> None:
|
|
content = _read_skill("testing-and-debugging")
|
|
assert "CI Failure Investigation" in content or "CI failure" in content, (
|
|
"testing-and-debugging skill must have CI failure section"
|
|
)
|
|
|
|
def test_mentions_spec_driven_workflow(self) -> None:
|
|
content = _read_skill("testing-and-debugging")
|
|
assert "spec" in content.lower(), "testing-and-debugging skill must mention spec-driven workflow"
|
|
|
|
|
|
# ============================================================================
|
|
# pr-review skill — deep review with auto-fix, exists in devx
|
|
# ============================================================================
|
|
|
|
|
|
class TestPrReviewSkill:
|
|
def test_skill_exists(self) -> None:
|
|
path = _DEVX / ".devin" / "skills" / "pr-review" / "SKILL.md"
|
|
assert path.exists(), "pr-review SKILL.md not found in devx"
|
|
|
|
def test_mentions_all_review_categories(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
required_categories = [
|
|
"Functional Correctness",
|
|
"Completeness",
|
|
"Architecture",
|
|
"Reliability",
|
|
"Robustness",
|
|
"Security",
|
|
"Technical Excellence",
|
|
"Test Quality",
|
|
]
|
|
for cat in required_categories:
|
|
assert cat in content, f"pr-review skill missing category: {cat}"
|
|
|
|
def test_mentions_auto_fix(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "auto-fix" in content.lower() or "auto fix" in content.lower(), "pr-review skill must mention auto-fix"
|
|
|
|
def test_mentions_gitea_mcp(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "mcp" in content.lower(), "pr-review skill must mention Gitea MCP"
|
|
|
|
def test_mentions_inline_comments(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "inline" in content.lower(), "pr-review skill must mention inline comments"
|
|
|
|
def test_mentions_ready_to_merge(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "ready-to-merge" in content, "pr-review skill must mention ready-to-merge label"
|
|
|
|
def test_mentions_resolve_discussion(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "resolve" in content.lower(), "pr-review skill must mention resolving discussions"
|
|
|
|
def test_mentions_summary(self) -> None:
|
|
content = _read_skill("pr-review")
|
|
assert "summary" in content.lower(), "pr-review skill must mention posting a summary"
|
|
|
|
def test_no_pr_review_module_remains(self) -> None:
|
|
"""The old devx.ci.pr_review module should be deleted."""
|
|
path = _DEVX / "src" / "devx" / "ci" / "pr_review.py"
|
|
assert not path.exists(), "devx.ci.pr_review module should be deleted (replaced by pr-review skill)"
|
|
|
|
def test_no_pr_review_test_remains(self) -> None:
|
|
"""The old test_pr_review.py should be deleted."""
|
|
path = _DEVX / "tests" / "unit" / "test_pr_review.py"
|
|
assert not path.exists(), "tests/unit/test_pr_review.py should be deleted"
|
|
|
|
def test_no_pr_review_in_workflows(self) -> None:
|
|
"""No devx CI workflow should reference devx.ci.pr_review."""
|
|
wf_dir = _DEVX / ".gitea" / "workflows"
|
|
if not wf_dir.exists():
|
|
pytest.skip("No workflows directory")
|
|
for wf_file in wf_dir.glob("*.yml"):
|
|
content = wf_file.read_text(encoding="utf-8")
|
|
assert "devx.ci.pr_review" not in content, f"{wf_file.name} still references devx.ci.pr_review"
|
|
|
|
|
|
# ============================================================================
|
|
# Skill consistency — all devx skills have proper structure
|
|
# ============================================================================
|
|
|
|
|
|
class TestSkillConsistency:
|
|
DEVX_SKILLS = [
|
|
"devx-workflow",
|
|
"testing-and-debugging",
|
|
"spec-driven-development",
|
|
"pr-review",
|
|
]
|
|
|
|
@pytest.mark.parametrize("skill_name", DEVX_SKILLS)
|
|
def test_skill_has_title(self, skill_name: str) -> None:
|
|
path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
|
|
assert path.exists(), f"SKILL.md not found for {skill_name}"
|
|
content = path.read_text(encoding="utf-8")
|
|
first_line = content.strip().split("\n")[0]
|
|
assert first_line.startswith("# "), f"{skill_name}: SKILL.md must start with a # title"
|
|
|
|
@pytest.mark.parametrize("skill_name", DEVX_SKILLS)
|
|
def test_skill_not_empty(self, skill_name: str) -> None:
|
|
path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
|
|
assert path.exists(), f"SKILL.md not found for {skill_name}"
|
|
content = path.read_text(encoding="utf-8").strip()
|
|
assert len(content) > 100, f"{skill_name}: SKILL.md is too short ({len(content)} chars)"
|
|
|
|
@pytest.mark.parametrize("skill_name", DEVX_SKILLS)
|
|
def test_skill_has_sections(self, skill_name: str) -> None:
|
|
path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
|
|
assert path.exists(), f"SKILL.md not found for {skill_name}"
|
|
content = path.read_text(encoding="utf-8")
|
|
section_count = content.count("\n## ")
|
|
assert section_count >= 2, f"{skill_name}: SKILL.md must have at least 2 sections (found {section_count})"
|
|
|
|
|
|
# ============================================================================
|
|
# AGENTS.md — spec-driven development section in devx
|
|
# ============================================================================
|
|
|
|
|
|
class TestAgentsMdSpecSection:
|
|
def test_agents_md_has_spec_driven_section(self) -> None:
|
|
path = _DEVX / "AGENTS.md"
|
|
if not path.exists():
|
|
pytest.skip("AGENTS.md not found in devx")
|
|
content = path.read_text(encoding="utf-8")
|
|
assert "## Spec-Driven Development" in content, "AGENTS.md must have '## Spec-Driven Development' section"
|
|
|
|
def test_agents_md_mentions_validate_spec(self) -> None:
|
|
path = _DEVX / "AGENTS.md"
|
|
if not path.exists():
|
|
pytest.skip("AGENTS.md not found in devx")
|
|
content = path.read_text(encoding="utf-8")
|
|
assert "validate_spec" in content or "devx.ci.validate_spec" in content, (
|
|
"AGENTS.md must mention devx.ci.validate_spec"
|
|
)
|
|
|
|
def test_agents_md_pr_workflow_section_intact(self) -> None:
|
|
"""Ensure the PR Workflow section wasn't accidentally deleted."""
|
|
path = _DEVX / "AGENTS.md"
|
|
if not path.exists():
|
|
pytest.skip("AGENTS.md not found in devx")
|
|
content = path.read_text(encoding="utf-8")
|
|
assert "## PR Workflow" in content, "AGENTS.md must still have '## PR Workflow' section"
|