"""Structural tests for spec-driven development workflows and skills. These tests parse the actual workflow YAML files in each repo and assert that the new spec-driven development steps, jobs, and env vars are present and correctly wired. They also validate that the spec-driven-development skill exists in each repo's .devin/skills/ directory with required sections. This is a "contract test" — it verifies that the workflows we wrote match the intended structure, catching regressions if someone edits a workflow and accidentally removes a step or breaks a job dependency. """ from __future__ import annotations from pathlib import Path import pytest import yaml # Repo root paths # __file__ = .../devx/tests/unit/test_spec_driven_workflows.py # parents[3] = .../oblachno (the monorepo root containing all repos) _OBLACHNO_ROOT = Path(__file__).resolve().parents[3] _INFRA = _OBLACHNO_ROOT / "infra" _GRM = _OBLACHNO_ROOT / "grm" _SSO_BRIDGE = _OBLACHNO_ROOT / "sso-bridge" _DEVX = _OBLACHNO_ROOT / "devx" def _load_workflow(repo_path: Path, filename: str) -> dict: """Load a workflow YAML file and return parsed dict.""" path = repo_path / ".gitea" / "workflows" / filename if not path.exists(): pytest.skip(f"Workflow {filename} not found in {repo_path.name}") with open(path, encoding="utf-8") as f: return yaml.safe_load(f) def _skip_if_repo_missing(repo_name: str) -> None: """Skip test if the sibling repo directory doesn't exist (CI only checks out one repo).""" repo_path = _OBLACHNO_ROOT / repo_name if not repo_path.is_dir(): pytest.skip(f"Repo {repo_name} not found at {repo_path} (CI only checks out devx)") def _read_skill(repo_name: str, skill_name: str) -> str: """Read a skill file from a repo, skipping if the repo or file doesn't exist.""" _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" if not skill_path.exists(): pytest.skip(f"SKILL.md not found in {repo_name}/{skill_name}") return skill_path.read_text(encoding="utf-8") def _get_step_names(job: dict) -> list[str]: """Extract step names from a job dict.""" names = [] for step in job.get("steps", []): if "name" in step: names.append(step["name"]) return names def _find_step(job: dict, name_part: str) -> dict | None: """Find a step by partial name match.""" for step in job.get("steps", []): if "name" in step and name_part.lower() in step["name"].lower(): return step return None def _get_run_commands(step: dict) -> str: """Get the run command from a step.""" return step.get("run", "") # ============================================================================ # Infra ci.yml — spec validation + PR size + fast molecule # ============================================================================ class TestInfraCiWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_INFRA, "ci.yml") def test_validate_job_exists(self, workflow: dict) -> None: assert "validate" in workflow["jobs"] def test_has_spec_validation_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Validate spec file" in s for s in steps), "validate job must have 'Validate spec file' step" def test_has_pr_size_check_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Check PR size" in s for s in steps), "validate job must have 'Check PR size' step" def test_spec_validation_uses_correct_module(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.validate_spec" in cmd assert "--github-output" in cmd def test_pr_size_uses_correct_module(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Check PR size") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.check_pr_size" in cmd assert "--github-output" in cmd def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None env = step.get("env", {}) assert env.get("DEVX_TASK_PREFIX") == "OBL-INFRA" def test_has_fast_molecule_job(self, workflow: dict) -> None: assert "fast-molecule" in workflow["jobs"], "ci.yml must have 'fast-molecule' job (replaced molecule-tests)" def test_no_full_molecule_tests_job(self, workflow: dict) -> None: assert "molecule-tests" not in workflow["jobs"], "ci.yml must NOT have 'molecule-tests' job (moved to nightly)" def test_no_staging_deploy_in_ci(self, workflow: dict) -> None: # The staging deploy was moved to post-merge (auto-deploy-staging) job_names = list(workflow["jobs"].keys()) assert "staging-health-gate" not in job_names, "staging-health-gate removed from ci.yml (moved to nightly)" assert "pre-deploy-checks" not in job_names, "pre-deploy-checks removed from ci.yml (moved to nightly)" assert "deploy" not in job_names, "deploy job removed from ci.yml (moved to post-merge)" def test_fast_molecule_uses_devx_module(self, workflow: dict) -> None: job = workflow["jobs"]["fast-molecule"] step = _find_step(job, "Detect changed roles") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.fast_molecule" in cmd assert "--github-output" in cmd def test_fast_molecule_timeout_is_short(self, workflow: dict) -> None: job = workflow["jobs"]["fast-molecule"] assert job.get("timeout-minutes", 999) <= 30, ( "fast-molecule timeout should be <= 30 min (was 120 for full suite)" ) def test_fast_molecule_no_matrix(self, workflow: dict) -> None: job = workflow["jobs"]["fast-molecule"] assert "strategy" not in job or "matrix" not in job.get("strategy", {}), ( "fast-molecule should not use matrix (single runner)" ) def test_auto_merge_depends_on_fast_molecule(self, workflow: dict) -> None: job = workflow["jobs"].get("auto-merge", {}) needs = job.get("needs", []) assert "fast-molecule" in needs, "auto-merge must depend on fast-molecule (not deploy)" def test_auto_merge_does_not_depend_on_deploy(self, workflow: dict) -> None: job = workflow["jobs"].get("auto-merge", {}) needs = job.get("needs", []) assert "deploy" not in needs, "auto-merge must NOT depend on deploy (removed from PR pipeline)" # ============================================================================ # Infra nightly.yml — full molecule + staging deploy + gate # ============================================================================ class TestInfraNightlyWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_INFRA, "nightly.yml") def test_nightly_workflow_exists(self, workflow: dict) -> None: assert workflow is not None def test_has_full_molecule_job(self, workflow: dict) -> None: assert "full-molecule" in workflow["jobs"] def test_has_set_gate_status_job(self, workflow: dict) -> None: assert "set-gate-status" in workflow["jobs"] def test_has_staging_deploy_job(self, workflow: dict) -> None: assert "staging-deploy" in workflow["jobs"] def test_full_molecule_uses_matrix(self, workflow: dict) -> None: job = workflow["jobs"]["full-molecule"] strategy = job.get("strategy", {}) assert "matrix" in strategy, "full-molecule must use matrix (6 runners)" assert "runner-index" in strategy["matrix"] def test_full_molecule_timeout_is_long(self, workflow: dict) -> None: job = workflow["jobs"]["full-molecule"] assert job.get("timeout-minutes", 0) >= 90, "full-molecule timeout should be >= 90 min (full suite)" def test_set_gate_status_depends_on_full_molecule(self, workflow: dict) -> None: job = workflow["jobs"]["set-gate-status"] needs = job.get("needs", []) assert "full-molecule" in needs def test_set_gate_status_uses_nightly_gate_module(self, workflow: dict) -> None: job = workflow["jobs"]["set-gate-status"] step = _find_step(job, "Set nightly gate") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.nightly_gate" in cmd assert "set-passed" in cmd or "set-failed" in cmd def test_staging_deploy_depends_on_gate(self, workflow: dict) -> None: job = workflow["jobs"]["staging-deploy"] needs = job.get("needs", []) assert "set-gate-status" in needs assert "full-molecule" in needs def test_staging_deploy_only_on_success(self, workflow: dict) -> None: job = workflow["jobs"]["staging-deploy"] if_cond = job.get("if", "") assert "success" in if_cond, "staging-deploy must only run when full-molecule succeeds" def test_nightly_runs_on_schedule(self, workflow: dict) -> None: on = workflow.get("on", workflow.get(True, {})) # YAML may parse 'on' as True (boolean) if isinstance(on, dict): assert "schedule" in on, "nightly must have schedule trigger" else: pytest.fail("Could not parse 'on' trigger from nightly.yml") # ============================================================================ # Infra post-merge.yml — auto-deploy staging with nightly gate # ============================================================================ class TestInfraPostMergeWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_INFRA, "post-merge.yml") def test_has_auto_deploy_staging_job(self, workflow: dict) -> None: assert "auto-deploy-staging" in workflow["jobs"], "post-merge must have 'auto-deploy-staging' job" def test_has_staging_deploy_job(self, workflow: dict) -> None: assert "staging-deploy" in workflow["jobs"], "post-merge must have 'staging-deploy' reusable workflow job" def test_auto_deploy_staging_checks_nightly_gate(self, workflow: dict) -> None: job = workflow["jobs"]["auto-deploy-staging"] step = _find_step(job, "Check nightly gate") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.nightly_gate" in cmd assert "--action check" in cmd def test_staging_deploy_depends_on_auto_deploy_staging(self, workflow: dict) -> None: job = workflow["jobs"]["staging-deploy"] needs = job.get("needs", []) assert "auto-deploy-staging" in needs def test_staging_deploy_gated_on_gate_passed(self, workflow: dict) -> None: job = workflow["jobs"]["staging-deploy"] if_cond = job.get("if", "") assert "gate-passed" in if_cond, "staging-deploy must check gate-passed output" def test_auto_deploy_production_waits_for_staging(self, workflow: dict) -> None: job = workflow["jobs"].get("auto-deploy-production", {}) needs = job.get("needs", []) assert "staging-deploy" in needs, "auto-deploy-production must wait for staging-deploy" # ============================================================================ # GRM ci.yml — spec validation + PR size # ============================================================================ class TestGrmCiWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_GRM, "ci.yml") def test_has_spec_validation_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Validate spec file" in s for s in steps) def test_has_pr_size_check_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Check PR size" in s for s in steps) def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None env = step.get("env", {}) assert env.get("DEVX_TASK_PREFIX") == "GRM" # ============================================================================ # GRM post-merge.yml — auto-create infra dependency PR # ============================================================================ class TestGrmPostMergeWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_GRM, "post-merge.yml") def test_has_create_dependency_pr_step(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None, "grm post-merge must have 'Create infra dependency PR' step" def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.create_dependency_pr" in cmd assert "--package grm" in cmd assert "--repo oblachno/infra" in cmd def test_dependency_pr_is_best_effort(self, workflow: dict) -> None: import re job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None cmd = _get_run_commands(step) # Must not fail the workflow if PR creation fails. # The || echo may be split across lines with backslash continuation in YAML. # Normalize: remove backslashes and collapse whitespace. cmd_normalized = " ".join(cmd.replace("\\", " ").split()) assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step, ( "dependency PR step must be best-effort (|| echo or continue-on-error)" ) # ============================================================================ # sso-bridge ci.yml — spec validation + PR size # ============================================================================ class TestSsoBridgeCiWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_SSO_BRIDGE, "ci.yml") def test_has_spec_validation_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Validate spec file" in s for s in steps) def test_has_pr_size_check_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Check PR size" in s for s in steps) def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None env = step.get("env", {}) assert env.get("DEVX_TASK_PREFIX") == "SSO" # ============================================================================ # sso-bridge post-merge.yml — auto-publish + auto-create dependency PR # ============================================================================ class TestSsoBridgePostMergeWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_SSO_BRIDGE, "post-merge.yml") def test_post_merge_workflow_exists(self, workflow: dict) -> None: assert workflow is not None def test_has_release_and_maintain_job(self, workflow: dict) -> None: assert "release-and-maintain" in workflow["jobs"] def test_has_create_dependency_pr_step(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None cmd = _get_run_commands(step) assert "devx.ci.create_dependency_pr" in cmd assert "--package sso_bridge" in cmd assert "--repo oblachno/infra" in cmd def test_dependency_pr_is_best_effort(self, workflow: dict) -> None: import re job = workflow["jobs"].get("release-and-maintain", {}) step = _find_step(job, "Create infra dependency PR") assert step is not None cmd = _get_run_commands(step) # The || echo may be split across lines with backslash continuation in YAML. cmd_normalized = " ".join(cmd.replace("\\", " ").split()) assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step def test_has_publish_step(self, workflow: dict) -> None: job = workflow["jobs"].get("release-and-maintain", {}) steps = _get_step_names(job) assert any("publish" in s.lower() for s in steps), "sso-bridge post-merge must have a publish step" # ============================================================================ # devx ci.yml — spec validation + PR size # ============================================================================ class TestDevxCiWorkflow: @pytest.fixture def workflow(self) -> dict: return _load_workflow(_DEVX, "ci.yml") def test_has_spec_validation_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Validate spec file" in s for s in steps) def test_has_pr_size_check_step(self, workflow: dict) -> None: steps = _get_step_names(workflow["jobs"]["validate"]) assert any("Check PR size" in s for s in steps) def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None: step = _find_step(workflow["jobs"]["validate"], "Validate spec file") assert step is not None env = step.get("env", {}) assert env.get("DEVX_TASK_PREFIX") == "DEVX" # ============================================================================ # Skill files — spec-driven-development SKILL.md in all repos # ============================================================================ class TestSpecDrivenDevelopmentSkill: REQUIRED_SECTIONS = [ "## Overview", "## Workflow", "## Spec Template", "## CI Validation", "## Acceptance Criteria", ] @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_exists_in_repo(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" assert skill_path.exists(), f"SKILL.md not found in {repo_name}" @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_has_required_sections(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" content = skill_path.read_text(encoding="utf-8") for section in self.REQUIRED_SECTIONS: assert section in content, f"SKILL.md in {repo_name} missing section: {section}" @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_req_ids(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" content = skill_path.read_text(encoding="utf-8") assert "REQ-" in content, f"SKILL.md in {repo_name} must mention REQ-ID format" @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_pr_size_limit(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" content = skill_path.read_text(encoding="utf-8") assert "500" in content, f"SKILL.md in {repo_name} must mention 500 line PR size limit" @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_nightly_gate(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" content = skill_path.read_text(encoding="utf-8") assert "nightly" in content.lower(), f"SKILL.md in {repo_name} must mention nightly gate" def test_skill_exists_in_shared_dir(self) -> None: skill_path = _OBLACHNO_ROOT / ".devin" / "skills" / "spec-driven-development" / "SKILL.md" if not skill_path.exists(): pytest.skip("Shared .devin/skills/ not found (CI only checks out devx repo)") assert skill_path.exists(), "SKILL.md not found in shared .devin/skills/" # ============================================================================ # devx-workflow skill — exists in repos with PR workflow, mentions spec gates # ============================================================================ class TestDevxWorkflowSkill: # Repos that have a PR workflow and need the devx-workflow skill REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"] @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_skill_exists(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" assert path.exists(), f"devx-workflow SKILL.md not found in {repo_name}" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_spec_validation(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "validate_spec" in content, f"devx-workflow skill in {repo_name} must mention validate_spec" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_pr_size_check(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "check_pr_size" in content, f"devx-workflow skill in {repo_name} must mention check_pr_size" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_pr_workflow_commands(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "make create-pr" in content or "make push-with-pr" in content, ( f"devx-workflow skill in {repo_name} must mention PR creation commands" ) @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_auto_merge(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "auto-merge" in content.lower() or "ready-to-merge" in content, ( f"devx-workflow skill in {repo_name} must mention auto-merge" ) @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_has_correct_task_prefix(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) """Each repo's devx-workflow skill must mention its correct task prefix.""" path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") expected_prefixes = { "infra": "OBL-INFRA", "grm": "GRM", "sso-bridge": "SSO", "devx": "DEVX", } prefix = expected_prefixes[repo_name] assert prefix in content, f"devx-workflow skill in {repo_name} must mention task prefix {prefix}" def test_not_in_mattermost_oidc(self) -> None: """mattermost-oidc has no PR workflow — should NOT have devx-workflow skill.""" path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" assert not path.exists(), "mattermost-oidc should NOT have devx-workflow skill (no PR workflow)" # Repo-specific content checks def test_infra_mentions_nightly_gate(self) -> None: _skip_if_repo_missing("infra") path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "nightly" in content.lower(), "infra devx-workflow skill must mention nightly gate" assert "nightly_gate" in content, "infra devx-workflow skill must mention devx.ci.nightly_gate module" def test_infra_mentions_fast_molecule(self) -> None: _skip_if_repo_missing("infra") path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "fast_molecule" in content, "infra devx-workflow skill must mention devx.ci.fast_molecule" def test_infra_mentions_auto_deploy_staging(self) -> None: _skip_if_repo_missing("infra") path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "staging" in content.lower(), "infra devx-workflow skill must mention staging auto-deploy" def test_grm_mentions_dependency_pr(self) -> None: _skip_if_repo_missing("grm") path = _OBLACHNO_ROOT / "grm" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "create_dependency_pr" in content, "grm devx-workflow skill must mention create_dependency_pr" def test_sso_bridge_mentions_dependency_pr(self) -> None: _skip_if_repo_missing("sso-bridge") path = _OBLACHNO_ROOT / "sso-bridge" / ".devin" / "skills" / "devx-workflow" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "create_dependency_pr" in content, "sso-bridge devx-workflow skill must mention create_dependency_pr" # ============================================================================ # testing-and-debugging skill — exists in all repos, mentions spec workflow # ============================================================================ class TestTestingAndDebuggingSkill: # All repos have a testing-and-debugging skill ALL_REPOS = ["infra", "grm", "sso-bridge", "devx", "mattermost-oidc"] @pytest.mark.parametrize("repo_name", ALL_REPOS) def test_skill_exists(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" assert path.exists(), f"testing-and-debugging SKILL.md not found in {repo_name}" @pytest.mark.parametrize("repo_name", ALL_REPOS) def test_has_required_sections(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" content = path.read_text(encoding="utf-8") # All testing-and-debugging skills should have a CI failure investigation section assert "CI Failure Investigation" in content or "CI failure" in content, ( f"testing-and-debugging skill in {repo_name} must have CI failure section" ) # Repos with PR workflow should mention spec-driven workflow @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_mentions_spec_driven_workflow(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "spec" in content.lower(), ( f"testing-and-debugging skill in {repo_name} must mention spec-driven workflow" ) def test_infra_mentions_nightly(self) -> None: _skip_if_repo_missing("infra") path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "nightly" in content.lower(), "infra testing-and-debugging skill must mention nightly tests" def test_infra_mentions_fast_molecule(self) -> None: _skip_if_repo_missing("infra") path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "fast" in content.lower() and "molecule" in content.lower(), ( "infra testing-and-debugging skill must mention fast molecule" ) def test_mattermost_oidc_no_spec_mention(self) -> None: """mattermost-oidc has no spec-driven workflow — skill should NOT mention it.""" _skip_if_repo_missing("mattermost-oidc") path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md" content = path.read_text(encoding="utf-8") # mattermost-oidc has no PR workflow, no spec validation assert "validate_spec" not in content, ( "mattermost-oidc testing-and-debugging skill should NOT mention validate_spec" ) # ============================================================================ # pr-review skill — deep review with auto-fix, exists in repos with PR workflow # ============================================================================ class TestPrReviewSkill: REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"] @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_skill_exists(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" assert path.exists(), f"pr-review SKILL.md not found in {repo_name}" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_all_review_categories(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") required_categories = [ "Functional Correctness", "Completeness", "Architecture", "Reliability", "Robustness", "Security", "Technical Excellence", "Test Quality", ] for cat in required_categories: assert cat in content, f"pr-review skill in {repo_name} missing category: {cat}" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_auto_fix(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "auto-fix" in content.lower() or "auto fix" in content.lower(), ( f"pr-review skill in {repo_name} must mention auto-fix" ) @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_gitea_mcp(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "mcp" in content.lower(), f"pr-review skill in {repo_name} must mention Gitea MCP" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_inline_comments(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "inline" in content.lower(), f"pr-review skill in {repo_name} must mention inline comments" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_ready_to_merge(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "ready-to-merge" in content, f"pr-review skill in {repo_name} must mention ready-to-merge label" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_resolve_discussion(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "resolve" in content.lower(), f"pr-review skill in {repo_name} must mention resolving discussions" @pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_summary(self, repo_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md" content = path.read_text(encoding="utf-8") assert "summary" in content.lower(), f"pr-review skill in {repo_name} must mention posting a summary" def test_not_in_mattermost_oidc(self) -> None: """mattermost-oidc has no PR workflow — should NOT have pr-review skill.""" path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "pr-review" / "SKILL.md" assert not path.exists(), "mattermost-oidc should NOT have pr-review skill (no PR workflow)" def test_no_pr_review_module_remains(self) -> None: """The old devx.ci.pr_review module should be deleted.""" path = _DEVX / "src" / "devx" / "ci" / "pr_review.py" assert not path.exists(), "devx.ci.pr_review module should be deleted (replaced by pr-review skill)" def test_no_pr_review_test_remains(self) -> None: """The old test_pr_review.py should be deleted.""" path = _DEVX / "tests" / "unit" / "test_pr_review.py" assert not path.exists(), "tests/unit/test_pr_review.py should be deleted" def test_no_pr_review_in_workflows(self) -> None: """No CI workflow should reference devx.ci.pr_review.""" for repo_name in ["infra", "grm", "sso-bridge", "devx"]: wf_dir = _OBLACHNO_ROOT / repo_name / ".gitea" / "workflows" if not wf_dir.exists(): continue for wf_file in wf_dir.glob("*.yml"): content = wf_file.read_text(encoding="utf-8") assert "devx.ci.pr_review" not in content, ( f"{repo_name}/{wf_file.name} still references devx.ci.pr_review" ) # ============================================================================ # Skill consistency — all skills have proper structure # ============================================================================ class TestSkillConsistency: ALL_SKILLS = [ ("infra", "devx-workflow"), ("infra", "testing-and-debugging"), ("infra", "spec-driven-development"), ("infra", "pr-review"), ("grm", "devx-workflow"), ("grm", "testing-and-debugging"), ("grm", "spec-driven-development"), ("grm", "pr-review"), ("sso-bridge", "devx-workflow"), ("sso-bridge", "testing-and-debugging"), ("sso-bridge", "spec-driven-development"), ("sso-bridge", "pr-review"), ("devx", "devx-workflow"), ("devx", "testing-and-debugging"), ("devx", "spec-driven-development"), ("devx", "pr-review"), ("mattermost-oidc", "testing-and-debugging"), ] @pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) def test_skill_has_title(self, repo_name: str, skill_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" content = path.read_text(encoding="utf-8") first_line = content.strip().split("\n")[0] assert first_line.startswith("# "), f"{repo_name}/{skill_name}: SKILL.md must start with a # title" @pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) def test_skill_not_empty(self, repo_name: str, skill_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" content = path.read_text(encoding="utf-8").strip() assert len(content) > 100, f"{repo_name}/{skill_name}: SKILL.md is too short ({len(content)} chars)" @pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) def test_skill_has_sections(self, repo_name: str, skill_name: str) -> None: _skip_if_repo_missing(repo_name) path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" content = path.read_text(encoding="utf-8") # Must have at least 2 ## sections section_count = content.count("\n## ") assert section_count >= 2, ( f"{repo_name}/{skill_name}: SKILL.md must have at least 2 sections (found {section_count})" ) # ============================================================================ # AGENTS.md — spec-driven development section in all repos # ============================================================================ class TestAgentsMdSpecSection: @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_has_spec_driven_section(self, repo_name: str) -> None: path = _OBLACHNO_ROOT / repo_name / "AGENTS.md" if not path.exists(): pytest.skip(f"AGENTS.md not found in {repo_name}") content = path.read_text(encoding="utf-8") assert "## Spec-Driven Development" in content, ( f"AGENTS.md in {repo_name} must have '## Spec-Driven Development' section" ) @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_mentions_validate_spec(self, repo_name: str) -> None: path = _OBLACHNO_ROOT / repo_name / "AGENTS.md" if not path.exists(): pytest.skip(f"AGENTS.md not found in {repo_name}") content = path.read_text(encoding="utf-8") assert "validate_spec" in content or "devx.ci.validate_spec" in content, ( f"AGENTS.md in {repo_name} must mention devx.ci.validate_spec" ) @pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_pr_workflow_section_intact(self, repo_name: str) -> None: """Ensure the PR Workflow section wasn't accidentally deleted.""" path = _OBLACHNO_ROOT / repo_name / "AGENTS.md" if not path.exists(): pytest.skip(f"AGENTS.md not found in {repo_name}") content = path.read_text(encoding="utf-8") assert "## PR Workflow" in content, f"AGENTS.md in {repo_name} must still have '## PR Workflow' section"