"""Structural tests for spec-driven development workflows and skills in devx. These tests parse devx's own workflow YAML files and assert that the spec-driven development steps, jobs, and env vars are present and correctly wired. They also validate that the skills exist in devx's own .devin/skills/ directory with required sections. devx must NOT be aware of other repos (infra, grm, sso-bridge). Those repos consume devx; devx does not test them. """ from __future__ import annotations from pathlib import Path import pytest import yaml # devx repo root # __file__ = .../devx/tests/unit/test_spec_driven_workflows.py # parents[2] = .../devx _DEVX = Path(__file__).resolve().parents[2] def _load_workflow(filename: str) -> dict: """Load a devx workflow YAML file and return parsed dict.""" path = _DEVX / ".gitea" / "workflows" / filename if not path.exists(): pytest.skip(f"Workflow {filename} not found in devx") with open(path, encoding="utf-8") as f: return yaml.safe_load(f) def _read_skill(skill_name: str) -> str: """Read a skill file from devx's .devin/skills/ directory.""" skill_path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md" if not skill_path.exists(): pytest.fail(f"SKILL.md not found for {skill_name} in devx") return skill_path.read_text(encoding="utf-8") def _get_step_names(job: dict) -> list[str]: """Extract step names from a job dict.""" names = [] for step in job.get("steps", []): if "name" in step: names.append(step["name"]) return names def _find_step(job: dict, name_part: str) -> dict | None: """Find a step by partial name match.""" for step in job.get("steps", []): if "name" in step and name_part.lower() in step["name"].lower(): return step return None def _get_run_commands(step: dict) -> str: """Get the run command from a step.""" return step.get("run", "") # ============================================================================ # devx ci.yml — spec validation + PR size # ============================================================================ class TestDevxCiWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow("ci.yml") def test_validate_job_exists(self, workflow: dict) -> None: assert "validate" in workflow["jobs"] def test_has_spec_validation_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Validate spec file" in s for s in steps), "validate job must have 'Validate spec file' step" def test_has_pr_size_check_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Check PR size" in s for s in steps), "validate job must have 'Check PR size' step" def test_spec_validation_uses_correct_module(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.validate_spec" in cmd assert "--github-output" in cmd def test_pr_size_uses_correct_module(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Check PR size") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.check_pr_size" in cmd assert "--github-output" in cmd def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None env = step.get("env", {}) assert env.get("DEVX_TASK_PREFIX") == "DEVX" def test_has_auto_merge_job(self, workflow: dict) -> None: assert "auto-merge" in workflow["jobs"], "ci.yml must have 'auto-merge' job" # ============================================================================ # devx post-merge.yml — release + publish # ============================================================================ class TestDevxPostMergeWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow("post-merge.yml") def test_post_merge_workflow_exists(self, workflow: dict) -> None: assert workflow is not None def test_has_release_and_maintain_job(self, workflow: dict) -> None: assert "release-and-maintain" in workflow["jobs"], "post-merge must have 'release-and-maintain' job" def test_has_publish_step(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) steps = _get_step_names(job) assert any("publish" in s.lower() for s in steps), "post-merge must have a publish step" # ============================================================================ # Skill files — spec-driven-development SKILL.md in devx # ============================================================================ class TestSpecDrivenDevelopmentSkill: REQUIRED_SECTIONS = [ "## Overview", "## Workflow", "## Spec Template", "## CI Validation", "## Acceptance Criteria", ] def test_skill_exists_in_repo(self) -> None: skill_path = _DEVX / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" assert skill_path.exists(), "SKILL.md not found in devx" def test_skill_has_required_sections(self) -> None: content = _read_skill("spec-driven-development") for section in self.REQUIRED_SECTIONS: assert section in content, f"SKILL.md missing section: {section}" def test_skill_mentions_req_ids(self) -> None: content = _read_skill("spec-driven-development") assert "REQ-" in content, "SKILL.md must mention REQ-ID format" def test_skill_mentions_pr_size_limit(self) -> None: content = _read_skill("spec-driven-development") assert "500" in content, "SKILL.md must mention 500 line PR size limit" def test_skill_mentions_nightly_gate(self) -> None: content = _read_skill("spec-driven-development") assert "nightly" in content.lower(), "SKILL.md must mention nightly gate" # ============================================================================ # devx-workflow skill — exists in devx, mentions spec gates # ============================================================================ class TestDevxWorkflowSkill: def test_skill_exists(self) -> None: path = _DEVX / ".devin" / "skills" / "devx-workflow" / "SKILL.md" assert path.exists(), "devx-workflow SKILL.md not found in devx" def test_mentions_spec_validation(self) -> None: content = _read_skill("devx-workflow") assert "validate_spec" in content, "devx-workflow skill must mention validate_spec" def test_mentions_pr_size_check(self) -> None: content = _read_skill("devx-workflow") assert "check_pr_size" in content, "devx-workflow skill must mention check_pr_size" def test_mentions_pr_workflow_commands(self) -> None: content = _read_skill("devx-workflow") assert "make create-pr" in content or "make push-with-pr" in content, ( "devx-workflow skill must mention PR creation commands" ) def test_mentions_auto_merge(self) -> None: content = _read_skill("devx-workflow") assert "auto-merge" in content.lower() or "ready-to-merge" in content, ( "devx-workflow skill must mention auto-merge" ) def test_has_correct_task_prefix(self) -> None: content = _read_skill("devx-workflow") assert "DEVX" in content, "devx-workflow skill must mention task prefix DEVX" # ============================================================================ # testing-and-debugging skill — exists in devx, mentions spec workflow # ============================================================================ class TestTestingAndDebuggingSkill: def test_skill_exists(self) -> None: path = _DEVX / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" assert path.exists(), "testing-and-debugging SKILL.md not found in devx" def test_has_required_sections(self) -> None: content = _read_skill("testing-and-debugging") assert "CI Failure Investigation" in content or "CI failure" in content, ( "testing-and-debugging skill must have CI failure section" ) def test_mentions_spec_driven_workflow(self) -> None: content = _read_skill("testing-and-debugging") assert "spec" in content.lower(), "testing-and-debugging skill must mention spec-driven workflow" # ============================================================================ # pr-review skill — deep review with auto-fix, exists in devx # ============================================================================ class TestPrReviewSkill: def test_skill_exists(self) -> None: path = _DEVX / ".devin" / "skills" / "pr-review" / "SKILL.md" assert path.exists(), "pr-review SKILL.md not found in devx" def test_mentions_all_review_categories(self) -> None: content = _read_skill("pr-review") required_categories = [ "Functional Correctness", "Completeness", "Architecture", "Reliability", "Robustness", "Security", "Technical Excellence", "Test Quality", ] for cat in required_categories: assert cat in content, f"pr-review skill missing category: {cat}" def test_mentions_auto_fix(self) -> None: content = _read_skill("pr-review") assert "auto-fix" in content.lower() or "auto fix" in content.lower(), "pr-review skill must mention auto-fix" def test_mentions_gitea_mcp(self) -> None: content = _read_skill("pr-review") assert "mcp" in content.lower(), "pr-review skill must mention Gitea MCP" def test_mentions_inline_comments(self) -> None: content = _read_skill("pr-review") assert "inline" in content.lower(), "pr-review skill must mention inline comments" def test_mentions_ready_to_merge(self) -> None: content = _read_skill("pr-review") assert "ready-to-merge" in content, "pr-review skill must mention ready-to-merge label" def test_mentions_resolve_discussion(self) -> None: content = _read_skill("pr-review") assert "resolve" in content.lower(), "pr-review skill must mention resolving discussions" def test_mentions_summary(self) -> None: content = _read_skill("pr-review") assert "summary" in content.lower(), "pr-review skill must mention posting a summary" def test_no_pr_review_module_remains(self) -> None: """The old devx.ci.pr_review module should be deleted.""" path = _DEVX / "src" / "devx" / "ci" / "pr_review.py" assert not path.exists(), "devx.ci.pr_review module should be deleted (replaced by pr-review skill)" def test_no_pr_review_test_remains(self) -> None: """The old test_pr_review.py should be deleted.""" path = _DEVX / "tests" / "unit" / "test_pr_review.py" assert not path.exists(), "tests/unit/test_pr_review.py should be deleted" def test_no_pr_review_in_workflows(self) -> None: """No devx CI workflow should reference devx.ci.pr_review.""" wf_dir = _DEVX / ".gitea" / "workflows" if not wf_dir.exists(): pytest.skip("No workflows directory") for wf_file in wf_dir.glob("*.yml"): content = wf_file.read_text(encoding="utf-8") assert "devx.ci.pr_review" not in content, f"{wf_file.name} still references devx.ci.pr_review" # ============================================================================ # Skill consistency — all devx skills have proper structure # ============================================================================ class TestSkillConsistency: DEVX_SKILLS = [ "devx-workflow", "testing-and-debugging", "spec-driven-development", "pr-review", ] @pytest.mark.parametrize("skill_name", DEVX_SKILLS) def test_skill_has_title(self, skill_name: str) -> None: path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}" content = path.read_text(encoding="utf-8") first_line = content.strip().split("\n")[0] assert first_line.startswith("# "), f"{skill_name}: SKILL.md must start with a # title" @pytest.mark.parametrize("skill_name", DEVX_SKILLS) def test_skill_not_empty(self, skill_name: str) -> None: path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}" content = path.read_text(encoding="utf-8").strip() assert len(content) > 100, f"{skill_name}: SKILL.md is too short ({len(content)} chars)" @pytest.mark.parametrize("skill_name", DEVX_SKILLS) def test_skill_has_sections(self, skill_name: str) -> None: path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}" content = path.read_text(encoding="utf-8") section_count = content.count("\n## ") assert section_count >= 2, f"{skill_name}: SKILL.md must have at least 2 sections (found {section_count})" # ============================================================================ # AGENTS.md — spec-driven development section in devx # ============================================================================ class TestAgentsMdSpecSection: def test_agents_md_has_spec_driven_section(self) -> None: path = _DEVX / "AGENTS.md" if not path.exists(): pytest.skip("AGENTS.md not found in devx") content = path.read_text(encoding="utf-8") assert "## Spec-Driven Development" in content, "AGENTS.md must have '## Spec-Driven Development' section" def test_agents_md_mentions_validate_spec(self) -> None: path = _DEVX / "AGENTS.md" if not path.exists(): pytest.skip("AGENTS.md not found in devx") content = path.read_text(encoding="utf-8") assert "validate_spec" in content or "devx.ci.validate_spec" in content, ( "AGENTS.md must mention devx.ci.validate_spec" ) def test_agents_md_pr_workflow_section_intact(self) -> None: """Ensure the PR Workflow section wasn't accidentally deleted.""" path = _DEVX / "AGENTS.md" if not path.exists(): pytest.skip("AGENTS.md not found in devx") content = path.read_text(encoding="utf-8") assert "## PR Workflow" in content, "AGENTS.md must still have '## PR Workflow' section"