Compare commits

...
1 Commits
Author SHA1 Message Date
emilandDevin <158243242+devin-ai-integration[bot]@users.noreply.github.com> c9bf8dbcc6 refactor: remove cross-repo contract tests from devx
CI / validate (pull_request) Successful in 1m2s
CI / auto-merge (pull_request) Successful in 16s
devx tests were validating infra, grm, sso-bridge, and Mattermost OIDC
workflow/skill files. This is an architecture violation — devx must not
be aware of other repos. Those repos consume devx, not the other way
round.

Tests now only validate devx's own workflows and skills.

Closes DEVX-160

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-08-26 08:24:45 +00:00
2 changed files with 171 additions and 618 deletions
+31
View File
@@ -0,0 +1,31 @@
# DEVX-160: Remove cross-repo contract tests from devx
## Problem
devx unit tests (`test_spec_driven_workflows.py`) were validating workflow
YAML and skill files in infra, grm, sso-bridge, and Mattermost OIDC repos.
This is an architecture violation — devx must not be aware of other repos.
Those repos consume devx; devx does not test them.
## Approach
Rewrite `test_spec_driven_workflows.py` to only test devx's own workflows
and skills. Remove all references to `_OBLACHNO_ROOT`, `_INFRA`, `_GRM`,
`_SSO_BRIDGE`, and parametrized repo lists.
REQ-1: No references to other repos in devx tests
REQ-2: All devx workflow/skill tests still pass
REQ-3: 100% coverage maintained
## Test Plan
- Run `pytest tests/unit/test_spec_driven_workflows.py` — all pass
- Run full test suite with coverage — 100%
## Deploy Plan
- Merge to master
## Rollback Plan
- Revert the merge commit
## Acceptance Criteria
- [x] REQ-1: No references to other repos in devx tests
- [x] REQ-2: All devx workflow/skill tests still pass
- [x] REQ-3: 100% coverage maintained
+140 -618
View File
@@ -1,13 +1,12 @@
"""Structural tests for spec-driven development workflows and skills. """Structural tests for spec-driven development workflows and skills in devx.
These tests parse the actual workflow YAML files in each repo and assert These tests parse devx's own workflow YAML files and assert that the
that the new spec-driven development steps, jobs, and env vars are present spec-driven development steps, jobs, and env vars are present and
and correctly wired. They also validate that the spec-driven-development correctly wired. They also validate that the skills exist in devx's
skill exists in each repo's .devin/skills/ directory with required sections. own .devin/skills/ directory with required sections.
This is a "contract test" — it verifies that the workflows we wrote match devx must NOT be aware of other repos (infra, grm, sso-bridge). Those
the intended structure, catching regressions if someone edits a workflow repos consume devx; devx does not test them.
and accidentally removes a step or breaks a job dependency.
""" """
from __future__ import annotations from __future__ import annotations
@@ -17,38 +16,26 @@ from pathlib import Path
import pytest import pytest
import yaml import yaml
# Repo root paths # devx repo root
# __file__ = .../devx/tests/unit/test_spec_driven_workflows.py # __file__ = .../devx/tests/unit/test_spec_driven_workflows.py
# parents[3] = .../oblachno (the monorepo root containing all repos) # parents[2] = .../devx
_OBLACHNO_ROOT = Path(__file__).resolve().parents[3] _DEVX = Path(__file__).resolve().parents[2]
_INFRA = _OBLACHNO_ROOT / "infra"
_GRM = _OBLACHNO_ROOT / "grm"
_SSO_BRIDGE = _OBLACHNO_ROOT / "sso-bridge"
_DEVX = _OBLACHNO_ROOT / "devx"
def _load_workflow(repo_path: Path, filename: str) -> dict: def _load_workflow(filename: str) -> dict:
"""Load a workflow YAML file and return parsed dict.""" """Load a devx workflow YAML file and return parsed dict."""
path = repo_path / ".gitea" / "workflows" / filename path = _DEVX / ".gitea" / "workflows" / filename
if not path.exists(): if not path.exists():
pytest.skip(f"Workflow {filename} not found in {repo_path.name}") pytest.skip(f"Workflow {filename} not found in devx")
with open(path, encoding="utf-8") as f: with open(path, encoding="utf-8") as f:
return yaml.safe_load(f) return yaml.safe_load(f)
def _skip_if_repo_missing(repo_name: str) -> None: def _read_skill(skill_name: str) -> str:
"""Skip test if the sibling repo directory doesn't exist (CI only checks out one repo).""" """Read a skill file from devx's .devin/skills/ directory."""
repo_path = _OBLACHNO_ROOT / repo_name skill_path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
if not repo_path.is_dir():
pytest.skip(f"Repo {repo_name} not found at {repo_path} (CI only checks out devx)")
def _read_skill(repo_name: str, skill_name: str) -> str:
"""Read a skill file from a repo, skipping if the repo or file doesn't exist."""
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md"
if not skill_path.exists(): if not skill_path.exists():
pytest.skip(f"SKILL.md not found in {repo_name}/{skill_name}") pytest.fail(f"SKILL.md not found for {skill_name} in devx")
return skill_path.read_text(encoding="utf-8") return skill_path.read_text(encoding="utf-8")
@@ -75,14 +62,14 @@ def _get_run_commands(step: dict) -> str:
# ============================================================================ # ============================================================================
# Infra ci.yml — spec validation + PR size + fast molecule # devx ci.yml — spec validation + PR size
# ============================================================================ # ============================================================================
class TestInfraCiWorkflow: class TestDevxCiWorkflow:
@pytest.fixture @pytest.fixture
def workflow(self) -> dict: def workflow(self) -> dict:
return _load_workflow(_INFRA, "ci.yml") return _load_workflow("ci.yml")
def test_validate_job_exists(self, workflow: dict) -> None: def test_validate_job_exists(self, workflow: dict) -> None:
assert "validate" in workflow["jobs"] assert "validate" in workflow["jobs"]
@@ -113,321 +100,36 @@ class TestInfraCiWorkflow:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file") step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None assert step is not None
env = step.get("env", {}) env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "OBL-INFRA" assert env.get("DEVX_TASK_PREFIX") == "DEVX"
def test_has_fast_molecule_job(self, workflow: dict) -> None: def test_has_auto_merge_job(self, workflow: dict) -> None:
assert "fast-molecule" in workflow["jobs"], "ci.yml must have 'fast-molecule' job (replaced molecule-tests)" assert "auto-merge" in workflow["jobs"], "ci.yml must have 'auto-merge' job"
def test_no_full_molecule_tests_job(self, workflow: dict) -> None:
assert "molecule-tests" not in workflow["jobs"], "ci.yml must NOT have 'molecule-tests' job (moved to nightly)"
def test_no_staging_deploy_in_ci(self, workflow: dict) -> None:
# The staging deploy was moved to post-merge (auto-deploy-staging)
job_names = list(workflow["jobs"].keys())
assert "staging-health-gate" not in job_names, "staging-health-gate removed from ci.yml (moved to nightly)"
assert "pre-deploy-checks" not in job_names, "pre-deploy-checks removed from ci.yml (moved to nightly)"
assert "deploy" not in job_names, "deploy job removed from ci.yml (moved to post-merge)"
def test_fast_molecule_uses_devx_module(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
step = _find_step(job, "Detect changed roles")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.fast_molecule" in cmd
assert "--github-output" in cmd
def test_fast_molecule_timeout_is_short(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
assert job.get("timeout-minutes", 999) <= 30, (
"fast-molecule timeout should be <= 30 min (was 120 for full suite)"
)
def test_fast_molecule_no_matrix(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
assert "strategy" not in job or "matrix" not in job.get("strategy", {}), (
"fast-molecule should not use matrix (single runner)"
)
def test_auto_merge_depends_on_fast_molecule(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-merge", {})
needs = job.get("needs", [])
assert "fast-molecule" in needs, "auto-merge must depend on fast-molecule (not deploy)"
def test_auto_merge_does_not_depend_on_deploy(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-merge", {})
needs = job.get("needs", [])
assert "deploy" not in needs, "auto-merge must NOT depend on deploy (removed from PR pipeline)"
# ============================================================================ # ============================================================================
# Infra nightly.yml — full molecule + staging deploy + gate # devx post-merge.yml — release + publish
# ============================================================================ # ============================================================================
class TestInfraNightlyWorkflow: class TestDevxPostMergeWorkflow:
@pytest.fixture @pytest.fixture
def workflow(self) -> dict: def workflow(self) -> dict:
return _load_workflow(_INFRA, "nightly.yml") return _load_workflow("post-merge.yml")
def test_nightly_workflow_exists(self, workflow: dict) -> None:
assert workflow is not None
def test_has_full_molecule_job(self, workflow: dict) -> None:
assert "full-molecule" in workflow["jobs"]
def test_has_set_gate_status_job(self, workflow: dict) -> None:
assert "set-gate-status" in workflow["jobs"]
def test_has_staging_deploy_job(self, workflow: dict) -> None:
assert "staging-deploy" in workflow["jobs"]
def test_full_molecule_uses_matrix(self, workflow: dict) -> None:
job = workflow["jobs"]["full-molecule"]
strategy = job.get("strategy", {})
assert "matrix" in strategy, "full-molecule must use matrix (6 runners)"
assert "runner-index" in strategy["matrix"]
def test_full_molecule_timeout_is_long(self, workflow: dict) -> None:
job = workflow["jobs"]["full-molecule"]
assert job.get("timeout-minutes", 0) >= 90, "full-molecule timeout should be >= 90 min (full suite)"
def test_set_gate_status_depends_on_full_molecule(self, workflow: dict) -> None:
job = workflow["jobs"]["set-gate-status"]
needs = job.get("needs", [])
assert "full-molecule" in needs
def test_set_gate_status_uses_nightly_gate_module(self, workflow: dict) -> None:
job = workflow["jobs"]["set-gate-status"]
step = _find_step(job, "Set nightly gate")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.nightly_gate" in cmd
assert "set-passed" in cmd or "set-failed" in cmd
def test_staging_deploy_depends_on_gate(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
needs = job.get("needs", [])
assert "set-gate-status" in needs
assert "full-molecule" in needs
def test_staging_deploy_only_on_success(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
if_cond = job.get("if", "")
assert "success" in if_cond, "staging-deploy must only run when full-molecule succeeds"
def test_nightly_runs_on_schedule(self, workflow: dict) -> None:
on = workflow.get("on", workflow.get(True, {}))
# YAML may parse 'on' as True (boolean)
if isinstance(on, dict):
assert "schedule" in on, "nightly must have schedule trigger"
else:
pytest.fail("Could not parse 'on' trigger from nightly.yml")
# ============================================================================
# Infra post-merge.yml — auto-deploy staging with nightly gate
# ============================================================================
class TestInfraPostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_INFRA, "post-merge.yml")
def test_has_auto_deploy_staging_job(self, workflow: dict) -> None:
assert "auto-deploy-staging" in workflow["jobs"], "post-merge must have 'auto-deploy-staging' job"
def test_has_staging_deploy_job(self, workflow: dict) -> None:
assert "staging-deploy" in workflow["jobs"], "post-merge must have 'staging-deploy' reusable workflow job"
def test_auto_deploy_staging_checks_nightly_gate(self, workflow: dict) -> None:
job = workflow["jobs"]["auto-deploy-staging"]
step = _find_step(job, "Check nightly gate")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.nightly_gate" in cmd
assert "--action check" in cmd
def test_staging_deploy_depends_on_auto_deploy_staging(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
needs = job.get("needs", [])
assert "auto-deploy-staging" in needs
def test_staging_deploy_gated_on_gate_passed(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
if_cond = job.get("if", "")
assert "gate-passed" in if_cond, "staging-deploy must check gate-passed output"
def test_auto_deploy_production_waits_for_staging(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-deploy-production", {})
needs = job.get("needs", [])
assert "staging-deploy" in needs, "auto-deploy-production must wait for staging-deploy"
# ============================================================================
# GRM ci.yml — spec validation + PR size
# ============================================================================
class TestGrmCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_GRM, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "GRM"
# ============================================================================
# GRM post-merge.yml — auto-create infra dependency PR
# ============================================================================
class TestGrmPostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_GRM, "post-merge.yml")
def test_has_create_dependency_pr_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None, "grm post-merge must have 'Create infra dependency PR' step"
def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.create_dependency_pr" in cmd
assert "--package grm" in cmd
assert "--repo oblachno/infra" in cmd
def test_dependency_pr_is_best_effort(self, workflow: dict) -> None:
import re
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
# Must not fail the workflow if PR creation fails.
# The || echo may be split across lines with backslash continuation in YAML.
# Normalize: remove backslashes and collapse whitespace.
cmd_normalized = " ".join(cmd.replace("\\", " ").split())
assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step, (
"dependency PR step must be best-effort (|| echo or continue-on-error)"
)
# ============================================================================
# sso-bridge ci.yml — spec validation + PR size
# ============================================================================
class TestSsoBridgeCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_SSO_BRIDGE, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "SSO"
# ============================================================================
# sso-bridge post-merge.yml — auto-publish + auto-create dependency PR
# ============================================================================
class TestSsoBridgePostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_SSO_BRIDGE, "post-merge.yml")
def test_post_merge_workflow_exists(self, workflow: dict) -> None: def test_post_merge_workflow_exists(self, workflow: dict) -> None:
assert workflow is not None assert workflow is not None
def test_has_release_and_maintain_job(self, workflow: dict) -> None: def test_has_release_and_maintain_job(self, workflow: dict) -> None:
assert "release-and-maintain" in workflow["jobs"] assert "release-and-maintain" in workflow["jobs"], "post-merge must have 'release-and-maintain' job"
def test_has_create_dependency_pr_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.create_dependency_pr" in cmd
assert "--package sso_bridge" in cmd
assert "--repo oblachno/infra" in cmd
def test_dependency_pr_is_best_effort(self, workflow: dict) -> None:
import re
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
# The || echo may be split across lines with backslash continuation in YAML.
cmd_normalized = " ".join(cmd.replace("\\", " ").split())
assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step
def test_has_publish_step(self, workflow: dict) -> None: def test_has_publish_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {}) job = workflow["jobs"].get("release-and-maintain", {})
steps = _get_step_names(job) steps = _get_step_names(job)
assert any("publish" in s.lower() for s in steps), "sso-bridge post-merge must have a publish step" assert any("publish" in s.lower() for s in steps), "post-merge must have a publish step"
# ============================================================================ # ============================================================================
# devx ci.yml — spec validation + PR size # Skill files — spec-driven-development SKILL.md in devx
# ============================================================================
class TestDevxCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_DEVX, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "DEVX"
# ============================================================================
# Skill files — spec-driven-development SKILL.md in all repos
# ============================================================================ # ============================================================================
@@ -440,227 +142,96 @@ class TestSpecDrivenDevelopmentSkill:
"## Acceptance Criteria", "## Acceptance Criteria",
] ]
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_exists_in_repo(self) -> None:
def test_skill_exists_in_repo(self, repo_name: str) -> None: skill_path = _DEVX / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
_skip_if_repo_missing(repo_name) assert skill_path.exists(), "SKILL.md not found in devx"
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
assert skill_path.exists(), f"SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_has_required_sections(self) -> None:
def test_skill_has_required_sections(self, repo_name: str) -> None: content = _read_skill("spec-driven-development")
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
for section in self.REQUIRED_SECTIONS: for section in self.REQUIRED_SECTIONS:
assert section in content, f"SKILL.md in {repo_name} missing section: {section}" assert section in content, f"SKILL.md missing section: {section}"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_req_ids(self) -> None:
def test_skill_mentions_req_ids(self, repo_name: str) -> None: content = _read_skill("spec-driven-development")
_skip_if_repo_missing(repo_name) assert "REQ-" in content, "SKILL.md must mention REQ-ID format"
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "REQ-" in content, f"SKILL.md in {repo_name} must mention REQ-ID format"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_pr_size_limit(self) -> None:
def test_skill_mentions_pr_size_limit(self, repo_name: str) -> None: content = _read_skill("spec-driven-development")
_skip_if_repo_missing(repo_name) assert "500" in content, "SKILL.md must mention 500 line PR size limit"
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "500" in content, f"SKILL.md in {repo_name} must mention 500 line PR size limit"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_skill_mentions_nightly_gate(self) -> None:
def test_skill_mentions_nightly_gate(self, repo_name: str) -> None: content = _read_skill("spec-driven-development")
_skip_if_repo_missing(repo_name) assert "nightly" in content.lower(), "SKILL.md must mention nightly gate"
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), f"SKILL.md in {repo_name} must mention nightly gate"
def test_skill_exists_in_shared_dir(self) -> None:
skill_path = _OBLACHNO_ROOT / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
if not skill_path.exists():
pytest.skip("Shared .devin/skills/ not found (CI only checks out devx repo)")
assert skill_path.exists(), "SKILL.md not found in shared .devin/skills/"
# ============================================================================ # ============================================================================
# devx-workflow skill — exists in repos with PR workflow, mentions spec gates # devx-workflow skill — exists in devx, mentions spec gates
# ============================================================================ # ============================================================================
class TestDevxWorkflowSkill: class TestDevxWorkflowSkill:
# Repos that have a PR workflow and need the devx-workflow skill def test_skill_exists(self) -> None:
REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"] path = _DEVX / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
assert path.exists(), "devx-workflow SKILL.md not found in devx"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_spec_validation(self) -> None:
def test_skill_exists(self, repo_name: str) -> None: content = _read_skill("devx-workflow")
_skip_if_repo_missing(repo_name) assert "validate_spec" in content, "devx-workflow skill must mention validate_spec"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
assert path.exists(), f"devx-workflow SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_pr_size_check(self) -> None:
def test_mentions_spec_validation(self, repo_name: str) -> None: content = _read_skill("devx-workflow")
_skip_if_repo_missing(repo_name) assert "check_pr_size" in content, "devx-workflow skill must mention check_pr_size"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "validate_spec" in content, f"devx-workflow skill in {repo_name} must mention validate_spec"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_pr_workflow_commands(self) -> None:
def test_mentions_pr_size_check(self, repo_name: str) -> None: content = _read_skill("devx-workflow")
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "check_pr_size" in content, f"devx-workflow skill in {repo_name} must mention check_pr_size"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_pr_workflow_commands(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "make create-pr" in content or "make push-with-pr" in content, ( assert "make create-pr" in content or "make push-with-pr" in content, (
f"devx-workflow skill in {repo_name} must mention PR creation commands" "devx-workflow skill must mention PR creation commands"
) )
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_auto_merge(self) -> None:
def test_mentions_auto_merge(self, repo_name: str) -> None: content = _read_skill("devx-workflow")
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "auto-merge" in content.lower() or "ready-to-merge" in content, ( assert "auto-merge" in content.lower() or "ready-to-merge" in content, (
f"devx-workflow skill in {repo_name} must mention auto-merge" "devx-workflow skill must mention auto-merge"
) )
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_has_correct_task_prefix(self) -> None:
def test_has_correct_task_prefix(self, repo_name: str) -> None: content = _read_skill("devx-workflow")
_skip_if_repo_missing(repo_name) assert "DEVX" in content, "devx-workflow skill must mention task prefix DEVX"
"""Each repo's devx-workflow skill must mention its correct task prefix."""
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
expected_prefixes = {
"infra": "OBL-INFRA",
"grm": "GRM",
"sso-bridge": "SSO",
"devx": "DEVX",
}
prefix = expected_prefixes[repo_name]
assert prefix in content, f"devx-workflow skill in {repo_name} must mention task prefix {prefix}"
def test_not_in_mattermost_oidc(self) -> None:
"""mattermost-oidc has no PR workflow — should NOT have devx-workflow skill."""
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
assert not path.exists(), "mattermost-oidc should NOT have devx-workflow skill (no PR workflow)"
# Repo-specific content checks
def test_infra_mentions_nightly_gate(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), "infra devx-workflow skill must mention nightly gate"
assert "nightly_gate" in content, "infra devx-workflow skill must mention devx.ci.nightly_gate module"
def test_infra_mentions_fast_molecule(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "fast_molecule" in content, "infra devx-workflow skill must mention devx.ci.fast_molecule"
def test_infra_mentions_auto_deploy_staging(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "staging" in content.lower(), "infra devx-workflow skill must mention staging auto-deploy"
def test_grm_mentions_dependency_pr(self) -> None:
_skip_if_repo_missing("grm")
path = _OBLACHNO_ROOT / "grm" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "create_dependency_pr" in content, "grm devx-workflow skill must mention create_dependency_pr"
def test_sso_bridge_mentions_dependency_pr(self) -> None:
_skip_if_repo_missing("sso-bridge")
path = _OBLACHNO_ROOT / "sso-bridge" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "create_dependency_pr" in content, "sso-bridge devx-workflow skill must mention create_dependency_pr"
# ============================================================================ # ============================================================================
# testing-and-debugging skill — exists in all repos, mentions spec workflow # testing-and-debugging skill — exists in devx, mentions spec workflow
# ============================================================================ # ============================================================================
class TestTestingAndDebuggingSkill: class TestTestingAndDebuggingSkill:
# All repos have a testing-and-debugging skill def test_skill_exists(self) -> None:
ALL_REPOS = ["infra", "grm", "sso-bridge", "devx", "mattermost-oidc"] path = _DEVX / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
assert path.exists(), "testing-and-debugging SKILL.md not found in devx"
@pytest.mark.parametrize("repo_name", ALL_REPOS) def test_has_required_sections(self) -> None:
def test_skill_exists(self, repo_name: str) -> None: content = _read_skill("testing-and-debugging")
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
assert path.exists(), f"testing-and-debugging SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", ALL_REPOS)
def test_has_required_sections(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
# All testing-and-debugging skills should have a CI failure investigation section
assert "CI Failure Investigation" in content or "CI failure" in content, ( assert "CI Failure Investigation" in content or "CI failure" in content, (
f"testing-and-debugging skill in {repo_name} must have CI failure section" "testing-and-debugging skill must have CI failure section"
) )
# Repos with PR workflow should mention spec-driven workflow def test_mentions_spec_driven_workflow(self) -> None:
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) content = _read_skill("testing-and-debugging")
def test_mentions_spec_driven_workflow(self, repo_name: str) -> None: assert "spec" in content.lower(), "testing-and-debugging skill must mention spec-driven workflow"
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "spec" in content.lower(), (
f"testing-and-debugging skill in {repo_name} must mention spec-driven workflow"
)
def test_infra_mentions_nightly(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), "infra testing-and-debugging skill must mention nightly tests"
def test_infra_mentions_fast_molecule(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "fast" in content.lower() and "molecule" in content.lower(), (
"infra testing-and-debugging skill must mention fast molecule"
)
def test_mattermost_oidc_no_spec_mention(self) -> None:
"""mattermost-oidc has no spec-driven workflow — skill should NOT mention it."""
_skip_if_repo_missing("mattermost-oidc")
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
# mattermost-oidc has no PR workflow, no spec validation
assert "validate_spec" not in content, (
"mattermost-oidc testing-and-debugging skill should NOT mention validate_spec"
)
# ============================================================================ # ============================================================================
# pr-review skill — deep review with auto-fix, exists in repos with PR workflow # pr-review skill — deep review with auto-fix, exists in devx
# ============================================================================ # ============================================================================
class TestPrReviewSkill: class TestPrReviewSkill:
REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"] def test_skill_exists(self) -> None:
path = _DEVX / ".devin" / "skills" / "pr-review" / "SKILL.md"
assert path.exists(), "pr-review SKILL.md not found in devx"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_all_review_categories(self) -> None:
def test_skill_exists(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
assert path.exists(), f"pr-review SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_all_review_categories(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
required_categories = [ required_categories = [
"Functional Correctness", "Functional Correctness",
"Completeness", "Completeness",
@@ -672,56 +243,31 @@ class TestPrReviewSkill:
"Test Quality", "Test Quality",
] ]
for cat in required_categories: for cat in required_categories:
assert cat in content, f"pr-review skill in {repo_name} missing category: {cat}" assert cat in content, f"pr-review skill missing category: {cat}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_auto_fix(self) -> None:
def test_mentions_auto_fix(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "auto-fix" in content.lower() or "auto fix" in content.lower(), "pr-review skill must mention auto-fix"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "auto-fix" in content.lower() or "auto fix" in content.lower(), (
f"pr-review skill in {repo_name} must mention auto-fix"
)
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_gitea_mcp(self) -> None:
def test_mentions_gitea_mcp(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "mcp" in content.lower(), "pr-review skill must mention Gitea MCP"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "mcp" in content.lower(), f"pr-review skill in {repo_name} must mention Gitea MCP"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_inline_comments(self) -> None:
def test_mentions_inline_comments(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "inline" in content.lower(), "pr-review skill must mention inline comments"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "inline" in content.lower(), f"pr-review skill in {repo_name} must mention inline comments"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_ready_to_merge(self) -> None:
def test_mentions_ready_to_merge(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "ready-to-merge" in content, "pr-review skill must mention ready-to-merge label"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "ready-to-merge" in content, f"pr-review skill in {repo_name} must mention ready-to-merge label"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_resolve_discussion(self) -> None:
def test_mentions_resolve_discussion(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "resolve" in content.lower(), "pr-review skill must mention resolving discussions"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "resolve" in content.lower(), f"pr-review skill in {repo_name} must mention resolving discussions"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW) def test_mentions_summary(self) -> None:
def test_mentions_summary(self, repo_name: str) -> None: content = _read_skill("pr-review")
_skip_if_repo_missing(repo_name) assert "summary" in content.lower(), "pr-review skill must mention posting a summary"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "summary" in content.lower(), f"pr-review skill in {repo_name} must mention posting a summary"
def test_not_in_mattermost_oidc(self) -> None:
"""mattermost-oidc has no PR workflow — should NOT have pr-review skill."""
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "pr-review" / "SKILL.md"
assert not path.exists(), "mattermost-oidc should NOT have pr-review skill (no PR workflow)"
def test_no_pr_review_module_remains(self) -> None: def test_no_pr_review_module_remains(self) -> None:
"""The old devx.ci.pr_review module should be deleted.""" """The old devx.ci.pr_review module should be deleted."""
@@ -734,102 +280,78 @@ class TestPrReviewSkill:
assert not path.exists(), "tests/unit/test_pr_review.py should be deleted" assert not path.exists(), "tests/unit/test_pr_review.py should be deleted"
def test_no_pr_review_in_workflows(self) -> None: def test_no_pr_review_in_workflows(self) -> None:
"""No CI workflow should reference devx.ci.pr_review.""" """No devx CI workflow should reference devx.ci.pr_review."""
for repo_name in ["infra", "grm", "sso-bridge", "devx"]: wf_dir = _DEVX / ".gitea" / "workflows"
wf_dir = _OBLACHNO_ROOT / repo_name / ".gitea" / "workflows" if not wf_dir.exists():
if not wf_dir.exists(): pytest.skip("No workflows directory")
continue for wf_file in wf_dir.glob("*.yml"):
for wf_file in wf_dir.glob("*.yml"): content = wf_file.read_text(encoding="utf-8")
content = wf_file.read_text(encoding="utf-8") assert "devx.ci.pr_review" not in content, f"{wf_file.name} still references devx.ci.pr_review"
assert "devx.ci.pr_review" not in content, (
f"{repo_name}/{wf_file.name} still references devx.ci.pr_review"
)
# ============================================================================ # ============================================================================
# Skill consistency — all skills have proper structure # Skill consistency — all devx skills have proper structure
# ============================================================================ # ============================================================================
class TestSkillConsistency: class TestSkillConsistency:
ALL_SKILLS = [ DEVX_SKILLS = [
("infra", "devx-workflow"), "devx-workflow",
("infra", "testing-and-debugging"), "testing-and-debugging",
("infra", "spec-driven-development"), "spec-driven-development",
("infra", "pr-review"), "pr-review",
("grm", "devx-workflow"),
("grm", "testing-and-debugging"),
("grm", "spec-driven-development"),
("grm", "pr-review"),
("sso-bridge", "devx-workflow"),
("sso-bridge", "testing-and-debugging"),
("sso-bridge", "spec-driven-development"),
("sso-bridge", "pr-review"),
("devx", "devx-workflow"),
("devx", "testing-and-debugging"),
("devx", "spec-driven-development"),
("devx", "pr-review"),
("mattermost-oidc", "testing-and-debugging"),
] ]
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) @pytest.mark.parametrize("skill_name", DEVX_SKILLS)
def test_skill_has_title(self, repo_name: str, skill_name: str) -> None: def test_skill_has_title(self, skill_name: str) -> None:
_skip_if_repo_missing(repo_name) path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}"
content = path.read_text(encoding="utf-8") content = path.read_text(encoding="utf-8")
first_line = content.strip().split("\n")[0] first_line = content.strip().split("\n")[0]
assert first_line.startswith("# "), f"{repo_name}/{skill_name}: SKILL.md must start with a # title" assert first_line.startswith("# "), f"{skill_name}: SKILL.md must start with a # title"
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) @pytest.mark.parametrize("skill_name", DEVX_SKILLS)
def test_skill_not_empty(self, repo_name: str, skill_name: str) -> None: def test_skill_not_empty(self, skill_name: str) -> None:
_skip_if_repo_missing(repo_name) path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}"
content = path.read_text(encoding="utf-8").strip() content = path.read_text(encoding="utf-8").strip()
assert len(content) > 100, f"{repo_name}/{skill_name}: SKILL.md is too short ({len(content)} chars)" assert len(content) > 100, f"{skill_name}: SKILL.md is too short ({len(content)} chars)"
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS) @pytest.mark.parametrize("skill_name", DEVX_SKILLS)
def test_skill_has_sections(self, repo_name: str, skill_name: str) -> None: def test_skill_has_sections(self, skill_name: str) -> None:
_skip_if_repo_missing(repo_name) path = _DEVX / ".devin" / "skills" / skill_name / "SKILL.md"
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md" assert path.exists(), f"SKILL.md not found for {skill_name}"
content = path.read_text(encoding="utf-8") content = path.read_text(encoding="utf-8")
# Must have at least 2 ## sections
section_count = content.count("\n## ") section_count = content.count("\n## ")
assert section_count >= 2, ( assert section_count >= 2, f"{skill_name}: SKILL.md must have at least 2 sections (found {section_count})"
f"{repo_name}/{skill_name}: SKILL.md must have at least 2 sections (found {section_count})"
)
# ============================================================================ # ============================================================================
# AGENTS.md — spec-driven development section in all repos # AGENTS.md — spec-driven development section in devx
# ============================================================================ # ============================================================================
class TestAgentsMdSpecSection: class TestAgentsMdSpecSection:
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_has_spec_driven_section(self) -> None:
def test_agents_md_has_spec_driven_section(self, repo_name: str) -> None: path = _DEVX / "AGENTS.md"
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md"
if not path.exists(): if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}") pytest.skip("AGENTS.md not found in devx")
content = path.read_text(encoding="utf-8") content = path.read_text(encoding="utf-8")
assert "## Spec-Driven Development" in content, ( assert "## Spec-Driven Development" in content, "AGENTS.md must have '## Spec-Driven Development' section"
f"AGENTS.md in {repo_name} must have '## Spec-Driven Development' section"
)
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_mentions_validate_spec(self) -> None:
def test_agents_md_mentions_validate_spec(self, repo_name: str) -> None: path = _DEVX / "AGENTS.md"
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md"
if not path.exists(): if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}") pytest.skip("AGENTS.md not found in devx")
content = path.read_text(encoding="utf-8") content = path.read_text(encoding="utf-8")
assert "validate_spec" in content or "devx.ci.validate_spec" in content, ( assert "validate_spec" in content or "devx.ci.validate_spec" in content, (
f"AGENTS.md in {repo_name} must mention devx.ci.validate_spec" "AGENTS.md must mention devx.ci.validate_spec"
) )
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"]) def test_agents_md_pr_workflow_section_intact(self) -> None:
def test_agents_md_pr_workflow_section_intact(self, repo_name: str) -> None:
"""Ensure the PR Workflow section wasn't accidentally deleted.""" """Ensure the PR Workflow section wasn't accidentally deleted."""
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md" path = _DEVX / "AGENTS.md"
if not path.exists(): if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}") pytest.skip("AGENTS.md not found in devx")
content = path.read_text(encoding="utf-8") content = path.read_text(encoding="utf-8")
assert "## PR Workflow" in content, f"AGENTS.md in {repo_name} must still have '## PR Workflow' section" assert "## PR Workflow" in content, "AGENTS.md must still have '## PR Workflow' section"