DEVX-155: Replace pr_review with spec-driven CI gates and pr-review skill
Post-merge / detect-and-configure (push) Failing after 18s
Post-merge / release-and-maintain (push) Skipped

This commit was merged in pull request #274.
This commit is contained in:
2026-08-24 20:39:09 +00:00
parent a06caa0e88
commit 11c4a1fc9e
26 changed files with 7823 additions and 5308 deletions
+169
View File
@@ -0,0 +1,169 @@
"""Unit tests for devx.ci.check_pr_size."""
from unittest.mock import MagicMock, patch
from click.testing import CliRunner
from devx.ci.check_pr_size import (
check_size,
cli,
get_diff_stats,
has_refactoring_label,
is_excluded,
)
class TestIsExcluded:
def test_excludes_changelog(self) -> None:
assert is_excluded("CHANGELOG.md", ["CHANGELOG.md"])
def test_excludes_svg_glob(self) -> None:
assert is_excluded("docs/badges/coverage.svg", ["*.svg"])
def test_does_not_exclude_source(self) -> None:
assert not is_excluded("src/devx/ci/check_pr_size.py", ["CHANGELOG.md", "*.svg"])
def test_excludes_readme(self) -> None:
assert is_excluded("README.md", ["README.md"])
class TestCheckSize:
def test_under_limits_passes(self) -> None:
stats = [("src/main.py", 100, 50), ("tests/test_main.py", 80, 20)]
ok, detail = check_size(stats, max_lines=500, max_files=10, excluded_patterns=[])
assert ok is True
assert "250" in detail # 100+50+80+20
def test_over_lines_fails(self) -> None:
stats = [("src/main.py", 300, 300)]
ok, detail = check_size(stats, max_lines=500, max_files=10, excluded_patterns=[])
assert ok is False
assert "600" in detail
def test_over_files_fails(self) -> None:
stats = [(f"src/file{i}.py", 10, 5) for i in range(15)]
ok, detail = check_size(stats, max_lines=500, max_files=10, excluded_patterns=[])
assert ok is False
assert "15" in detail
def test_excluded_files_not_counted(self) -> None:
stats = [("CHANGELOG.md", 500, 500), ("src/main.py", 10, 5)]
ok, detail = check_size(stats, max_lines=500, max_files=10, excluded_patterns=["CHANGELOG.md"])
assert ok is True
assert "15" in detail # only 10+5
def test_empty_stats_passes(self) -> None:
ok, detail = check_size([], max_lines=500, max_files=10, excluded_patterns=[])
assert ok is True
class TestGetDiffStats:
@patch("devx.ci.check_pr_size.subprocess.run")
def test_parses_numstat_output(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="10\t5\tsrc/main.py\n20\t10\ttests/test_main.py\n",
stderr="",
)
stats = get_diff_stats("origin/master", "HEAD")
assert len(stats) == 2
assert stats[0] == ("src/main.py", 10, 5)
assert stats[1] == ("tests/test_main.py", 20, 10)
@patch("devx.ci.check_pr_size.subprocess.run")
def test_handles_binary_files(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="-\t-\timage.png\n",
stderr="",
)
stats = get_diff_stats("origin/master", "HEAD")
assert len(stats) == 1
assert stats[0] == ("image.png", 0, 0)
@patch("devx.ci.check_pr_size.subprocess.run")
def test_empty_output(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
stats = get_diff_stats("origin/master", "HEAD")
assert stats == []
@patch("devx.ci.check_pr_size.subprocess.run")
def test_git_diff_failure_raises(self, mock_run: MagicMock) -> None:
import pytest
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="fatal: bad ref")
with pytest.raises(Exception, match="git diff|bad ref"):
get_diff_stats("origin/master", "HEAD")
@patch("devx.ci.check_pr_size.subprocess.run")
def test_malformed_line_skipped(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="not_a_valid_line\n10\t5\tsrc/main.py\n",
stderr="",
)
stats = get_diff_stats("origin/master", "HEAD")
assert len(stats) == 1
assert stats[0] == ("src/main.py", 10, 5)
class TestCli:
@patch("devx.ci.check_pr_size.subprocess.run")
def test_passes_when_small(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="10\t5\tsrc/main.py\n",
stderr="",
)
runner = CliRunner()
result = runner.invoke(cli, ["--base", "origin/master", "--head", "HEAD"])
assert result.exit_code == 0
assert "OK" in result.output
@patch("devx.ci.check_pr_size.subprocess.run")
def test_fails_when_too_large(self, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="300\t300\tsrc/main.py\n",
stderr="",
)
runner = CliRunner()
result = runner.invoke(cli, ["--base", "origin/master", "--head", "HEAD", "--max-lines", "500"])
assert result.exit_code != 0
assert "600" in result.output
@patch("devx.ci.check_pr_size.subprocess.run")
@patch("devx.ci.check_pr_size.has_refactoring_label", return_value=True)
def test_bypasses_with_refactoring_label(self, mock_label: MagicMock, mock_run: MagicMock) -> None:
mock_run.return_value = MagicMock(
returncode=0,
stdout="300\t300\tsrc/main.py\n",
stderr="",
)
runner = CliRunner()
result = runner.invoke(
cli,
["--base", "origin/master", "--head", "HEAD", "--repo", "owner/repo", "--pr-number", "42"],
)
assert result.exit_code == 0
assert "bypassed" in result.output.lower()
class TestHasRefactoringLabel:
@patch("devx.ci.check_pr_size.GiteaClient")
@patch("devx.ci.check_pr_size.get_ci_token", return_value="fake-token")
def test_returns_true_when_label_present(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.get_pr.return_value = {"labels": [{"name": "refactoring"}, {"name": "bug"}]}
assert has_refactoring_label("owner/repo", 42) is True
@patch("devx.ci.check_pr_size.GiteaClient")
@patch("devx.ci.check_pr_size.get_ci_token", return_value="fake-token")
def test_returns_false_when_label_absent(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.get_pr.return_value = {"labels": [{"name": "bug"}]}
assert has_refactoring_label("owner/repo", 42) is False
@patch("devx.ci.check_pr_size.get_ci_token", side_effect=Exception("no token"))
def test_returns_false_on_error(self, mock_token: MagicMock) -> None:
assert has_refactoring_label("owner/repo", 42) is False
-7
View File
@@ -107,13 +107,6 @@ class TestCiCommands:
assert result.exit_code == 0
mock_run.assert_called_once_with("devx.ci.post_merge", ["DEVX-1"])
@patch("devx.cli._run_module")
def test_ci_pr_review(self, mock_run: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(cli, ["ci", "pr-review", "42"])
assert result.exit_code == 0
mock_run.assert_called_once_with("devx.ci.pr_review", ["42"])
@patch("devx.cli._run_module")
def test_ci_publish(self, mock_run: MagicMock) -> None:
runner = CliRunner()
+182
View File
@@ -0,0 +1,182 @@
"""Unit tests for devx.ci.create_dependency_pr."""
from pathlib import Path
from unittest.mock import MagicMock, patch
import click
from click.testing import CliRunner
from devx.ci.create_dependency_pr import (
cli,
create_vikunja_task,
find_existing_pr,
find_pinned_version,
update_pinned_version,
)
class TestFindPinnedVersion:
def test_finds_pip_git_pin(self, tmp_path: Path) -> None:
content = "grm @ git+https://git.example.com/repo.git@v0.5.1"
path = tmp_path / "pyproject.toml"
path.write_text(content)
version = find_pinned_version("grm", str(path))
assert version == "0.5.1"
def test_finds_pyproject_pin(self, tmp_path: Path) -> None:
content = 'grm = "0.5.1"'
path = tmp_path / "pyproject.toml"
path.write_text(content)
version = find_pinned_version("grm", str(path))
assert version == "0.5.1"
def test_finds_ansible_var_pin(self, tmp_path: Path) -> None:
content = 'grm_version: "0.5.1"'
path = tmp_path / "images.yml"
path.write_text(content)
version = find_pinned_version("grm", str(path))
assert version == "0.5.1"
def test_finds_image_version_pin(self, tmp_path: Path) -> None:
content = 'sso_bridge_image_version: "1.2.3"'
path = tmp_path / "images.yml"
path.write_text(content)
version = find_pinned_version("sso_bridge", str(path))
assert version == "1.2.3"
def test_returns_none_when_not_found(self, tmp_path: Path) -> None:
path = tmp_path / "pyproject.toml"
path.write_text('other = "1.0.0"')
assert find_pinned_version("grm", str(path)) is None
def test_returns_none_when_file_missing(self, tmp_path: Path) -> None:
assert find_pinned_version("grm", str(tmp_path / "nonexistent.toml")) is None
class TestUpdatePinnedVersion:
def test_updates_pip_git_pin(self, tmp_path: Path) -> None:
content = "grm @ git+https://git.example.com/repo.git@v0.5.1"
path = tmp_path / "pyproject.toml"
path.write_text(content)
changed = update_pinned_version(str(path), "grm", "0.5.1", "0.5.2")
assert changed is True
assert "0.5.2" in path.read_text()
assert "0.5.1" not in path.read_text()
def test_updates_pyproject_pin(self, tmp_path: Path) -> None:
content = 'grm = "0.5.1"'
path = tmp_path / "pyproject.toml"
path.write_text(content)
changed = update_pinned_version(str(path), "grm", "0.5.1", "0.5.2")
assert changed is True
assert 'grm = "0.5.2"' in path.read_text()
def test_no_change_when_version_not_found(self, tmp_path: Path) -> None:
content = 'other = "1.0.0"'
path = tmp_path / "pyproject.toml"
path.write_text(content)
changed = update_pinned_version(str(path), "grm", "0.5.1", "0.5.2")
assert changed is False
def test_no_change_when_file_missing(self, tmp_path: Path) -> None:
changed = update_pinned_version(str(tmp_path / "nonexistent"), "grm", "0.5.1", "0.5.2")
assert changed is False
class TestFindExistingPr:
@patch("devx.tools.create_pr.GiteaClient")
def test_returns_pr_when_found(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.list_prs.return_value = [
{"head": {"ref": "deps/grm-0.5.2"}, "number": 42},
{"head": {"ref": "other-branch"}, "number": 43},
]
result = find_existing_pr(mock_client, "deps/grm-0.5.2")
assert result is not None
assert result["number"] == 42
@patch("devx.tools.create_pr.GiteaClient")
def test_returns_none_when_not_found(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.list_prs.return_value = []
result = find_existing_pr(mock_client, "deps/grm-0.5.2")
assert result is None
class TestCli:
@patch("devx.ci.create_dependency_pr.find_pinned_version")
@patch("devx.ci.create_dependency_pr.get_ci_token")
def test_same_version_no_pr(self, mock_token: MagicMock, mock_find: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_find.return_value = "0.5.2"
runner = CliRunner()
result = runner.invoke(
cli,
[
"--package",
"grm",
"--new-version",
"0.5.2",
"--source-repo",
"oblachno/grm",
],
)
assert result.exit_code == 0
assert "no pr needed" in result.output.lower()
@patch("devx.ci.create_dependency_pr.find_pinned_version")
@patch("devx.ci.create_dependency_pr.get_ci_token")
def test_dry_run(self, mock_token: MagicMock, mock_find: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_find.return_value = "0.5.1"
runner = CliRunner()
result = runner.invoke(
cli,
[
"--package",
"grm",
"--new-version",
"0.5.2",
"--source-repo",
"oblachno/grm",
"--dry-run",
],
)
assert result.exit_code == 0
assert "DRY RUN" in result.output
@patch("devx.ci.create_dependency_pr.find_pinned_version")
@patch("devx.ci.create_dependency_pr.get_ci_token")
def test_version_not_found_fails(self, mock_token: MagicMock, mock_find: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_find.return_value = None
runner = CliRunner()
result = runner.invoke(
cli,
[
"--package",
"nonexistent",
"--new-version",
"1.0.0",
"--source-repo",
"oblachno/test",
],
)
assert result.exit_code != 0
class TestCreateVikunjaTask:
def test_returns_none_when_no_token(self) -> None:
with patch("devx.ci.create_dependency_pr.get_vikunja_token", side_effect=click.ClickException("no token")):
result = create_vikunja_task("Test", "desc")
assert result is None
def test_returns_identifier_on_success(self) -> None:
with (
patch("devx.ci.create_dependency_pr.get_vikunja_token", return_value="fake-token"),
patch("devx.api_clients.VikunjaClient") as mock_client_cls,
):
mock_client = mock_client_cls.return_value
mock_client.create_task.return_value = {"identifier": "OBL-INFRA-999"}
result = create_vikunja_task("Test", "desc")
assert result == "OBL-INFRA-999"
+82
View File
@@ -0,0 +1,82 @@
"""Unit tests for devx.ci.fast_molecule."""
from pathlib import Path
from unittest.mock import MagicMock, patch
from click.testing import CliRunner
from devx.ci.fast_molecule import (
build_molecule_commands,
cli,
get_molecule_scenarios,
)
class TestGetMoleculeScenarios:
def test_finds_scenarios(self, tmp_path: Path) -> None:
roles_dir = tmp_path / "ansible" / "roles" / "myrole" / "molecule"
roles_dir.mkdir(parents=True)
(roles_dir / "default").mkdir()
(roles_dir / "default" / "molecule.yml").write_text("name: default")
(roles_dir / "full").mkdir()
(roles_dir / "full" / "molecule.yml").write_text("name: full")
(roles_dir / "no_scenario").mkdir() # No molecule.yml
scenarios = get_molecule_scenarios("myrole", str(tmp_path / "ansible" / "roles"))
assert sorted(scenarios) == ["default", "full"]
def test_returns_empty_when_no_molecule_dir(self, tmp_path: Path) -> None:
scenarios = get_molecule_scenarios("nonexistent", str(tmp_path / "ansible" / "roles"))
assert scenarios == []
class TestBuildMoleculeCommands:
def test_builds_commands_for_roles(self, tmp_path: Path) -> None:
roles_dir = tmp_path / "ansible" / "roles"
for role in ["role_a", "role_b"]:
mol_dir = roles_dir / role / "molecule" / "default"
mol_dir.mkdir(parents=True)
(mol_dir / "molecule.yml").write_text("name: default")
commands = build_molecule_commands({"role_a", "role_b"}, str(roles_dir))
assert len(commands) == 2
assert all("molecule test -s default" in c for c in commands)
assert all("--destroy=never" in c for c in commands)
assert all("ubuntu-2604" in c for c in commands)
def test_empty_when_no_scenarios(self, tmp_path: Path) -> None:
commands = build_molecule_commands({"nonexistent"}, str(tmp_path / "ansible" / "roles"))
assert commands == []
def test_empty_when_no_roles(self) -> None:
assert build_molecule_commands(set()) == []
class TestCli:
@patch("devx.ci.fast_molecule.get_changed_files")
def test_no_changes(self, mock_get: MagicMock) -> None:
mock_get.return_value = []
runner = CliRunner()
result = runner.invoke(cli, ["--base", "origin/master", "--head", "HEAD"])
assert result.exit_code == 0
assert "No files changed" in result.output
@patch("devx.ci.fast_molecule.detect_changed_roles")
@patch("devx.ci.fast_molecule.get_changed_files")
def test_no_ansible_changes(self, mock_get: MagicMock, mock_detect: MagicMock) -> None:
mock_get.return_value = ["src/main.py", "README.md"]
mock_detect.return_value = set()
runner = CliRunner()
result = runner.invoke(cli, ["--base", "origin/master", "--head", "HEAD"])
assert result.exit_code == 0
assert "No Ansible roles changed" in result.output
@patch("devx.ci.fast_molecule.detect_changed_roles")
@patch("devx.ci.fast_molecule.get_changed_files")
def test_detects_changed_roles(self, mock_get: MagicMock, mock_detect: MagicMock) -> None:
mock_get.return_value = ["ansible/roles/sso_bridge/tasks/main.yml"]
mock_detect.return_value = {"sso_bridge"}
runner = CliRunner()
result = runner.invoke(cli, ["--base", "origin/master", "--head", "HEAD"])
assert result.exit_code == 0
assert "sso_bridge" in result.output
+106
View File
@@ -0,0 +1,106 @@
"""Unit tests for devx.ci.nightly_gate."""
from unittest.mock import MagicMock, patch
from click.testing import CliRunner
from devx.ci.nightly_gate import cli, get_nightly_status, set_nightly_status
class TestGetNightlyStatus:
@patch("devx.ci.nightly_gate.GiteaClient")
def test_returns_value_when_set(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.get_repo_variable.return_value = "passed:12345"
result = get_nightly_status(mock_client)
assert result == "passed:12345"
@patch("devx.ci.nightly_gate.GiteaClient")
def test_returns_empty_when_not_set(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
mock_client.get_repo_variable.return_value = None
result = get_nightly_status(mock_client)
assert result == ""
class TestSetNightlyStatus:
@patch("devx.ci.nightly_gate.GiteaClient")
def test_sets_passed(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
set_nightly_status(mock_client, "passed:12345")
mock_client.set_repo_variable.assert_called_once_with("NIGHTLY_STATUS", "passed:12345")
@patch("devx.ci.nightly_gate.GiteaClient")
def test_sets_failed(self, mock_client_cls: MagicMock) -> None:
mock_client = mock_client_cls.return_value
set_nightly_status(mock_client, "failed:99999")
mock_client.set_repo_variable.assert_called_once_with("NIGHTLY_STATUS", "failed:99999")
class TestCli:
@patch("devx.ci.nightly_gate.GiteaClient")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_check_bootstrap_allows_deploy(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_client = mock_client_cls.return_value
mock_client.get_repo_variable.return_value = None
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "check"])
assert result.exit_code == 0
assert "bootstrap" in result.output.lower()
@patch("devx.ci.nightly_gate.GiteaClient")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_check_passed_allows_deploy(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_client = mock_client_cls.return_value
mock_client.get_repo_variable.return_value = "passed:12345"
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "check"])
assert result.exit_code == 0
assert "passed" in result.output.lower()
@patch("devx.ci.nightly_gate.GiteaClient")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_check_failed_blocks_deploy(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_client = mock_client_cls.return_value
mock_client.get_repo_variable.return_value = "failed:99999"
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "check"])
assert result.exit_code != 0
assert "blocked" in result.output.lower()
@patch("devx.ci.nightly_gate.GiteaClient")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_set_passed(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_client = mock_client_cls.return_value
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "set-passed", "--run-id", "12345"])
assert result.exit_code == 0
mock_client.set_repo_variable.assert_called_once_with("NIGHTLY_STATUS", "passed:12345")
@patch("devx.ci.nightly_gate.GiteaClient")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_set_failed(self, mock_token: MagicMock, mock_client_cls: MagicMock) -> None:
mock_token.return_value = "fake-token"
mock_client = mock_client_cls.return_value
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "set-failed", "--run-id", "99999"])
assert result.exit_code == 0
mock_client.set_repo_variable.assert_called_once_with("NIGHTLY_STATUS", "failed:99999")
@patch("devx.ci.nightly_gate.get_ci_token")
def test_fails_without_token(self, mock_token: MagicMock) -> None:
import click as click_mod
mock_token.side_effect = click_mod.ClickException("No token")
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "oblachno/infra", "--action", "check"])
assert result.exit_code != 0
def test_fails_with_invalid_repo(self) -> None:
runner = CliRunner()
result = runner.invoke(cli, ["--repo", "invalid", "--action", "check"])
assert result.exit_code != 0
-1022
View File
@@ -1,1022 +0,0 @@
"""Unit tests for scripts/ci/pr_review.py."""
from unittest.mock import MagicMock, patch
import pytest
from click.testing import CliRunner
from devx.ci.pr_review import (
ReviewResult,
build_review_body,
check_architecture_compliance,
check_best_practices,
check_commit_conventions,
check_documentation,
check_function_length,
check_i18n,
check_resource_management,
check_security,
check_test_coverage,
is_python_file,
is_workflow_only,
main,
post_review,
run_review,
)
from devx.exceptions import APIError
class TestIsPythonFile:
def test_python_file_in_src(self) -> None:
assert is_python_file("src/devx/cli.py") is True
def test_python_file_in_scripts(self) -> None:
assert is_python_file("scripts/ci/release.py") is True
def test_test_file_excluded(self) -> None:
assert is_python_file("tests/unit/test_cli.py") is False
def test_non_python_file(self) -> None:
assert is_python_file("README.md") is False
def test_yaml_file(self) -> None:
assert is_python_file(".gitea/workflows/ci.yml") is False
class TestIsWorkflowOnly:
def test_yaml_is_workflow(self) -> None:
assert is_workflow_only(".gitea/workflows/ci.yml") is True
def test_md_is_workflow(self) -> None:
assert is_workflow_only("README.md") is True
def test_python_is_not_workflow(self) -> None:
assert is_workflow_only("src/devx/cli.py") is False
def test_ansible_is_workflow(self) -> None:
assert is_workflow_only("ansible/tasks/main.yml") is True
class TestReviewResult:
def test_empty_result_has_no_issues(self) -> None:
result = ReviewResult()
assert result.has_issues is False
def test_add_issue_makes_has_issues_true(self) -> None:
result = ReviewResult()
result.add_issue("src/foo.py", 10, "bad code")
assert result.has_issues is True
assert len(result.issues) == 1
assert result.issues[0]["path"] == "src/foo.py"
assert result.issues[0]["new_position"] == 10
def test_add_summary(self) -> None:
result = ReviewResult()
result.add_summary("all good")
assert "all good" in result.summary
class TestCheckArchitectureCompliance:
def test_subprocess_in_cli_triggers_issue(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ subprocess.run(['ls'])\n",
}
]
check_architecture_compliance(files, result)
assert result.has_issues
assert "subprocess" in result.issues[0]["body"].lower()
def test_subprocess_in_other_file_ok(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/executor.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ subprocess.run(['ls'])\n",
}
]
check_architecture_compliance(files, result)
assert not result.has_issues
def test_no_changes_adds_ok_summary(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_architecture_compliance(files, result)
assert any("Architecture compliance: OK" in s for s in result.summary)
def test_non_python_file_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "README.md", "patch": "@@ -1,1 +1,2 @@\n+subprocess.run(['ls'])\n"}]
check_architecture_compliance(files, result)
assert not result.has_issues
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_architecture_compliance(files, result)
assert not result.has_issues
def test_os_system_in_cli_triggers_issue(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ os.system('ls')\n",
}
]
check_architecture_compliance(files, result)
assert result.has_issues
assert "os.system" in result.issues[0]["body"]
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -1,2 @@\n+ subprocess.run(['ls'])\n",
}
]
check_architecture_compliance(files, result)
assert result.has_issues
class TestCheckBestPractices:
def test_print_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ print('hello')\n",
}
]
check_best_practices(files, result)
assert result.has_issues
assert "print()" in result.issues[0]["body"]
def test_bare_except_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/runner_manager.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ except:\n pass\n",
}
]
check_best_practices(files, result)
assert result.has_issues
assert "bare except" in result.issues[0]["body"]
def test_todo_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ # TODO: fix this\n",
}
]
check_best_practices(files, result)
assert result.has_issues
assert "TODO" in result.issues[0]["body"]
def test_clean_code_no_issues(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ click.echo('hello')\n",
}
]
check_best_practices(files, result)
assert not result.has_issues
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_best_practices(files, result)
assert not result.has_issues
def test_non_python_file_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "README.md", "patch": "@@ -1,1 +1,2 @@\n+print('hello')\n"}]
check_best_practices(files, result)
assert not result.has_issues
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -1,2 @@\n+ print('hello')\n",
}
]
check_best_practices(files, result)
assert result.has_issues
assert "print()" in result.issues[0]["body"]
class TestCheckSecurity:
def test_hardcoded_secret_triggers_error(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/config.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ token = 'abc123secrettoken456'\n",
}
]
check_security(files, result)
assert result.has_issues
assert "secret" in result.issues[0]["body"].lower()
def test_example_token_not_flagged(self) -> None:
result = ReviewResult()
files = [
{
"filename": ".env.example",
"patch": "@@ -1,1 +1,2 @@\n+token = your-example-token\n",
}
]
check_security(files, result)
assert not result.has_issues
def test_shell_true_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/executor.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ subprocess.run('ls', shell=True)\n",
}
]
check_best_practices(files, result)
assert result.has_issues
assert "shell=True" in result.issues[0]["body"]
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/config.py", "patch": ""}]
check_security(files, result)
assert not result.has_issues
def test_non_python_file_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "docs/config.md", "patch": "@@ -1,1 +1,2 @@\n+token = 'abc123secrettoken456'\n"}]
check_security(files, result)
assert not result.has_issues
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [
{
"filename": "src/devx/config.py",
"patch": "@@ -1,2 @@\n+ token = 'abc123secrettoken456'\n",
}
]
check_security(files, result)
assert result.has_issues
assert "secret" in result.issues[0]["body"].lower()
class TestCheckI18n:
def test_raw_string_in_echo_triggers_warning(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+click.echo("Hello world")\n'}]
check_i18n(files, result)
assert result.has_issues
assert any("i18n" in i["body"] for i in result.issues)
def test_translated_string_no_warning(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+click.echo(_("Hello world"))\n'}]
check_i18n(files, result)
assert not result.has_issues
def test_fstring_in_echo_triggers_warning(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+click.echo(f"Hello {name}")\n'}]
check_i18n(files, result)
assert result.has_issues
def test_raw_exception_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": '@@ -1,1 +1,1 @@\n+raise click.ClickException("Something went wrong")\n',
}
]
check_i18n(files, result)
assert result.has_issues
def test_non_src_file_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "scripts/ci/test.py", "patch": '@@ -1,1 +1,1 @@\n+click.echo("Hello world")\n'}]
check_i18n(files, result)
assert not result.has_issues
def test_comment_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+# click.echo("Hello world")\n'}]
check_i18n(files, result)
assert not result.has_issues
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_i18n(files, result)
assert not result.has_issues
def test_clean_code_adds_ok_summary(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+click.echo(_("Hello world"))\n'}]
check_i18n(files, result)
assert any("i18n: OK" in s for s in result.summary)
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,2 @@\n+click.echo("Hello world")\n'}]
check_i18n(files, result)
assert result.has_issues
assert any("i18n" in i["body"] for i in result.issues)
class TestCheckResourceManagement:
def test_open_without_with_triggers_warning(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+f = open("file.txt")\n'}]
check_resource_management(files, result)
assert result.has_issues
assert any("resource" in i["body"].lower() for i in result.issues)
def test_open_with_with_no_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": '@@ -1,1 +1,1 @@\n+with open("file.txt") as f:\n+ pass\n',
}
]
check_resource_management(files, result)
assert not result.has_issues
def test_popen_without_cleanup_triggers_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/executor.py",
"patch": '@@ -1,1 +1,1 @@\n+proc = subprocess.Popen(["cmd"])\n',
}
]
check_resource_management(files, result)
assert result.has_issues
def test_popen_with_communicate_no_warning(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/executor.py",
"patch": '@@ -1,1 +1,1 @@\n+out, err = subprocess.Popen(["cmd"], stdout=PIPE).communicate()\n',
}
]
check_resource_management(files, result)
assert not result.has_issues
def test_comment_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,1 +1,1 @@\n+# f = open("file.txt")\n'}]
check_resource_management(files, result)
assert not result.has_issues
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_resource_management(files, result)
assert not result.has_issues
def test_non_python_file_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "docs/config.md", "patch": '@@ -1,1 +1,1 @@\n+f = open("file.txt")\n'}]
check_resource_management(files, result)
assert not result.has_issues
def test_clean_code_adds_ok_summary(self) -> None:
result = ReviewResult()
files = [
{
"filename": "src/devx/cli.py",
"patch": '@@ -1,1 +1,1 @@\n+with open("file.txt") as f:\n+ data = f.read()\n',
}
]
check_resource_management(files, result)
assert any("Resource management: OK" in s for s in result.summary)
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": '@@ -1,2 @@\n+f = open("file.txt")\n'}]
check_resource_management(files, result)
assert result.has_issues
assert any("resource" in i["body"].lower() for i in result.issues)
class TestCheckFunctionLength:
def test_long_function_triggers_warning(self) -> None:
result = ReviewResult()
# Create a patch with a function that adds > 50 lines
added_lines = "\n".join(f"+ x = {i}" for i in range(55))
patch = f"@@ -10,3 +10,59 @@\n+def foo():\n+ pass\n{added_lines}\n"
files = [{"filename": "src/devx/cli.py", "patch": patch}]
check_function_length(files, result)
assert result.has_issues
assert "foo" in result.issues[0]["body"]
def test_short_function_no_warning(self) -> None:
result = ReviewResult()
patch = "@@ -10,3 +10,8 @@\n def foo():\n pass\n+ x = 1\n+ y = 2\n+ z = 3\n"
files = [{"filename": "src/devx/cli.py", "patch": patch}]
check_function_length(files, result)
assert not result.has_issues
def test_empty_patch_skipped(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": ""}]
check_function_length(files, result)
assert not result.has_issues
def test_non_python_file_skipped(self) -> None:
result = ReviewResult()
added_lines = "\n".join(f"+ x = {i}" for i in range(55))
patch = f"@@ -10,3 +10,59 @@\n+def foo():\n+ pass\n{added_lines}\n"
files = [{"filename": "README.md", "patch": patch}]
check_function_length(files, result)
assert not result.has_issues
def test_multiple_functions_resets_count(self) -> None:
"""Two short functions back-to-back should not trigger the length warning."""
result = ReviewResult()
patch = "@@ -10,3 +10,15 @@\n+def foo():\n+ x = 1\n+def bar():\n+ y = 2\n"
files = [{"filename": "src/devx/cli.py", "patch": patch}]
check_function_length(files, result)
assert not result.has_issues
def test_long_function_followed_by_new_hunk(self) -> None:
"""Long function followed by @@ header triggers the warning at hunk boundary."""
result = ReviewResult()
added_lines = "\n".join(f"+ x = {i}" for i in range(55))
patch = (
f"@@ -10,3 +10,59 @@\n+def foo():\n+ pass\n{added_lines}\n@@ -100,3 +100,5 @@\n+def bar():\n+ pass\n"
)
files = [{"filename": "src/devx/cli.py", "patch": patch}]
check_function_length(files, result)
assert result.has_issues
assert "foo" in result.issues[0]["body"]
def test_long_function_followed_by_new_def(self) -> None:
"""Long function followed by another def triggers the warning at def boundary."""
result = ReviewResult()
added_lines = "\n".join(f"+ x = {i}" for i in range(55))
patch = f"@@ -10,3 +10,60 @@\n+def foo():\n+ pass\n{added_lines}\n+def bar():\n+ pass\n"
files = [{"filename": "src/devx/cli.py", "patch": patch}]
check_function_length(files, result)
assert result.has_issues
assert "foo" in result.issues[0]["body"]
def test_malformed_hunk_header_no_line_number(self) -> None:
"""A @@ header without a +N line number is handled gracefully."""
result = ReviewResult()
files = [{"filename": "src/devx/cli.py", "patch": "@@ -1,2 @@\n+def foo():\n+ pass\n"}]
check_function_length(files, result)
assert not result.has_issues
class TestCheckDocumentation:
def test_src_changes_without_docs_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py"}]
check_documentation(files, result)
assert any("WARNING" in s for s in result.summary)
def test_src_changes_with_docs_ok(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py"}, {"filename": "docs/user/cli-commands.md"}]
check_documentation(files, result)
assert any("Documentation: OK" in s for s in result.summary)
def test_ansible_changes_without_docs_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "ansible/roles/gitea-runner/tasks/main.yml"}]
check_documentation(files, result)
assert any("WARNING" in s for s in result.summary)
def test_only_doc_changes_ok(self) -> None:
result = ReviewResult()
files = [{"filename": "README.md"}]
check_documentation(files, result)
assert any("Documentation: OK" in s for s in result.summary)
def test_tofu_changes_without_docs_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "tofu/modules/hetzner-vm/main.tf"}]
check_documentation(files, result)
assert any("WARNING" in s for s in result.summary)
def test_workflow_changes_info(self) -> None:
result = ReviewResult()
files = [{"filename": ".gitea/workflows/ci.yml"}]
check_documentation(files, result)
assert any("INFO" in s for s in result.summary)
def test_todo_in_doc_patch_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "docs/guide.md", "patch": "+TODO: fix this later\n+Some content\n"}]
check_documentation(files, result)
assert any("TODO" in s for s in result.summary)
def test_todo_in_readme_patch_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "README.md", "patch": "+FIXME: broken\n"}]
check_documentation(files, result)
assert any("FIXME" in s for s in result.summary)
def test_no_todo_in_doc_patch_ok(self) -> None:
result = ReviewResult()
files = [{"filename": "docs/guide.md", "patch": "+Some content\n"}]
check_documentation(files, result)
assert not any("TODO" in s for s in result.summary)
class TestCheckTestCoverage:
def test_src_changes_without_tests_warns(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py"}]
check_test_coverage(files, result)
assert any("WARNING" in s for s in result.summary)
def test_src_changes_with_tests_ok(self) -> None:
result = ReviewResult()
files = [{"filename": "src/devx/cli.py"}, {"filename": "tests/unit/test_cli.py"}]
check_test_coverage(files, result)
assert any("Tests: OK" in s for s in result.summary)
def test_only_test_changes_ok(self) -> None:
result = ReviewResult()
files = [{"filename": "tests/unit/test_cli.py"}]
check_test_coverage(files, result)
assert any("Tests: OK" in s for s in result.summary)
class TestBuildReviewBody:
def test_body_contains_summary(self) -> None:
result = ReviewResult()
result.add_summary("- Architecture compliance: OK")
body = build_review_body(result)
assert "Architecture compliance: OK" in body
assert "Automated PR Review" in body
def test_body_contains_issues(self) -> None:
result = ReviewResult()
result.add_issue("src/foo.py", 10, "bad code")
body = build_review_body(result)
assert "1 issue(s) found" in body
assert "src/foo.py:10" in body
assert "bad code" in body
def test_body_contains_no_issues_message(self) -> None:
result = ReviewResult()
body = build_review_body(result)
assert "No issues found" in body
def test_body_contains_auto_merge_note(self) -> None:
"""Review body must mention auto-merge."""
result = ReviewResult()
body = build_review_body(result)
assert "Auto-merge" in body
class TestRunReview:
@patch("devx.ci.pr_review.GiteaClient")
def test_run_review_with_no_files(self, mock_client_class: MagicMock) -> None:
mock_client = mock_client_class.return_value
mock_client.get_pr_files.return_value = []
result = run_review(mock_client, "42")
assert "No files changed" in result.summary[0]
@patch("devx.ci.pr_review.GiteaClient")
def test_run_review_finds_issues(self, mock_client_class: MagicMock) -> None:
mock_client = mock_client_class.return_value
mock_client.get_pr_files.return_value = [
{
"filename": "src/devx/cli.py",
"patch": "@@ -10,3 +10,4 @@\n def foo():\n pass\n+ print('hello')\n",
}
]
mock_client.get_pr_commits.return_value = [{"commit": {"message": "fix: resolve print issue"}}]
result = run_review(mock_client, "42")
assert result.has_issues
def test_run_review_handles_api_error(self) -> None:
client = MagicMock()
client.get_pr_files.side_effect = APIError(404, "Not found")
result = run_review(client, "42")
assert any("ERROR" in s for s in result.summary)
class TestCheckCommitConventions:
def test_conventional_commit_found(self) -> None:
"""Should report OK when at least one commit is conventional."""
client = MagicMock()
client.get_pr_commits.return_value = [
{"commit": {"message": "fix: resolve bug\n\nDetails"}},
{"commit": {"message": "wip: testing"}},
]
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("OK" in s for s in result.summary)
def test_no_conventional_commit(self) -> None:
"""Should warn when no commits are conventional."""
client = MagicMock()
client.get_pr_commits.return_value = [
{"commit": {"message": "updated stuff"}},
{"commit": {"message": "wip: testing"}},
]
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("WARNING" in s for s in result.summary)
def test_merge_commits_excluded(self) -> None:
"""Merge commits should be excluded from the check."""
client = MagicMock()
client.get_pr_commits.return_value = [
{"commit": {"message": "Merge branch 'feature' into master"}},
{"commit": {"message": "fix: resolve bug"}},
]
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("OK" in s for s in result.summary)
def test_all_merges_and_reverts(self) -> None:
"""Should report OK when all commits are merges/reverts."""
client = MagicMock()
client.get_pr_commits.return_value = [
{"commit": {"message": "Merge branch 'feature' into master"}},
{"commit": {"message": "Revert: bad commit"}},
]
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("merges/reverts" in s for s in result.summary)
def test_no_commits(self) -> None:
"""Should report OK when there are no commits."""
client = MagicMock()
client.get_pr_commits.return_value = []
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("no commits" in s for s in result.summary)
def test_api_error(self) -> None:
"""Should report ERROR when API call fails."""
client = MagicMock()
client.get_pr_commits.side_effect = APIError(500, "server error")
result = ReviewResult()
check_commit_conventions(client, "42", result)
assert any("ERROR" in s for s in result.summary)
class TestPostReview:
def test_post_review_with_issues(self) -> None:
client = MagicMock()
result = ReviewResult()
result.add_issue("src/foo.py", 10, "bad code")
post_review(client, "42", result)
client.create_review.assert_called_once()
call_args = client.create_review.call_args
assert call_args[1]["event"] == "REQUEST_CHANGES"
assert call_args[1]["comments"] == result.issues
def test_post_review_without_issues_uses_comment_not_approve(self) -> None:
"""Automated review posts COMMENT, not APPROVE (self-approval not allowed)."""
client = MagicMock()
result = ReviewResult()
post_review(client, "42", result)
client.create_review.assert_called_once()
call_args = client.create_review.call_args
assert call_args[1]["event"] == "COMMENT"
assert call_args[1]["comments"] == []
class TestMain:
@patch("devx.ci.pr_review.run_review")
@patch("devx.ci.pr_review.GiteaClient")
def test_dry_run_does_not_post(self, mock_client_class: MagicMock, mock_run: MagicMock) -> None:
mock_run.return_value = ReviewResult()
runner = CliRunner()
result = runner.invoke(main, ["42", "my-org/my-repo", "--dry-run"], env={"CI_GITEA_TOKEN": "fake"})
assert result.exit_code == 0
assert "[dry-run]" in result.output
mock_client_class.return_value.create_review.assert_not_called()
@patch("devx.ci.pr_review.run_review")
@patch("devx.ci.pr_review.GiteaClient")
def test_post_review_on_success(self, mock_client_class: MagicMock, mock_run: MagicMock) -> None:
mock_run.return_value = ReviewResult()
mock_client_class.return_value.create_review.return_value = {"id": 123}
runner = CliRunner()
result = runner.invoke(main, ["42", "my-org/my-repo"], env={"CI_GITEA_TOKEN": "fake"})
assert result.exit_code == 0
assert "Review #123" in result.output
mock_client_class.return_value.create_review.assert_called_once()
@patch("devx.ci.pr_review.run_review")
@patch("devx.ci.pr_review.GiteaClient")
def test_self_approval_falls_back_to_comment(self, mock_client_class: MagicMock, mock_run: MagicMock) -> None:
"""If REQUEST_CHANGES fails with 422 (self-approval), fall back to COMMENT."""
mock_run.return_value = ReviewResult()
client = mock_client_class.return_value
client.create_review.side_effect = [
APIError(422, "approve your own pull is not allowed"),
{"id": 124},
]
runner = CliRunner()
result = runner.invoke(main, ["42", "my-org/my-repo"], env={"CI_GITEA_TOKEN": "fake"})
assert result.exit_code == 0
assert "Review #124" in result.output
assert client.create_review.call_count == 2
@patch("devx.ci.pr_review.run_review")
@patch("devx.ci.pr_review.GiteaClient")
def test_other_api_error_re_raises(self, mock_client_class: MagicMock, mock_run: MagicMock) -> None:
"""Non-approval API errors should re-raise, not fall back."""
mock_run.return_value = ReviewResult()
client = mock_client_class.return_value
client.create_review.side_effect = APIError(500, "Internal server error")
runner = CliRunner()
result = runner.invoke(main, ["42", "my-org/my-repo"], env={"CI_GITEA_TOKEN": "fake"})
assert result.exit_code != 0
@patch.dict("os.environ", {"CI_GITEA_API_TOKEN": "", "CI_GITEA_TOKEN": ""})
def test_no_token_raises(self) -> None:
runner = CliRunner()
result = runner.invoke(main, ["42", "my-org/my-repo"], env={"CI_GITEA_API_TOKEN": "", "CI_GITEA_TOKEN": ""})
assert result.exit_code != 0
assert "CI_GITEA_TOKEN" in result.output
class TestManualReview:
@patch("devx.ci.pr_review.GiteaClient")
def test_approve_success(self, mock_client_class: MagicMock) -> None:
mock_client_class.return_value.create_review.return_value = {"id": 200}
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"All 13 REVIEW_CHECKLIST.md categories verified. Architecture: clean. Security: no issues.",
"--checklist-confirmed",
"--checklist-categories",
"1,2,3,4,5,6,7,8,9,10,11,12,13",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code == 0
assert "Review #200" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_approve_without_checklist_confirmed_fails(self, mock_client_class: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-categories",
"1,2,3,4,5,6,7,8",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code != 0
assert "checklist-confirmed" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_approve_with_too_few_categories_fails(self, mock_client_class: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-confirmed",
"--checklist-categories",
"1,2,3",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code != 0
assert "at least 8" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_approve_with_short_body_fails(self, mock_client_class: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"LGTM",
"--checklist-confirmed",
"--checklist-categories",
"1,2,3,4,5,6,7,8",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code != 0
assert "50 characters" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_approve_with_invalid_category_fails(self, mock_client_class: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-confirmed",
"--checklist-categories",
"1,2,abc,4",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code != 0
assert "Invalid" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_request_changes_success(self, mock_client_class: MagicMock) -> None:
mock_client_class.return_value.create_review.return_value = {"id": 201}
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"REQUEST_CHANGES",
"--body",
"Please fix the architecture issues in the CLI module before merging.",
],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code == 0
assert "Review #201" in result.output
@patch("devx.ci.pr_review.GiteaClient")
def test_manual_review_dry_run(self, mock_client_class: MagicMock) -> None:
runner = CliRunner()
result = runner.invoke(
main,
["42", "oblachno-oss/devx", "--event", "COMMENT", "--body", "x" * 60, "--dry-run"],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code == 0
assert "[dry-run]" in result.output
mock_client_class.return_value.create_review.assert_not_called()
@patch("devx.ci.pr_review.GiteaClient")
def test_manual_review_self_approval_fallback_to_comment(
self, mock_client_class: MagicMock, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Self-approval with no CI token available → fall back to COMMENT."""
monkeypatch.delenv("CI_GITEA_API_TOKEN", raising=False)
monkeypatch.delenv("CI_GITEA_TOKEN", raising=False)
client = mock_client_class.return_value
client.create_review.side_effect = [
APIError(422, "approve your own pull is not allowed"),
{"id": 202},
]
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-confirmed",
"--checklist-categories",
"1,2,3,4,5,6,7,8",
],
env={"REVIEWER_GITEA_API_TOKEN": "fake-reviewer"},
)
assert result.exit_code == 0
assert "Review #202" in result.output
# Without CI_GITEA_API_TOKEN, the fallback is COMMENT
assert "Self-approval not allowed. Posting COMMENT instead." in result.output
assert client.create_review.call_count == 2
assert client.create_review.call_args_list[1].kwargs.get("event") == "COMMENT"
@patch("devx.ci.pr_review.GiteaClient")
def test_manual_review_self_approval_falls_back_to_ci_token(self, mock_client_class: MagicMock) -> None:
"""Self-approval with CI token available → retry APPROVE with CI token (different user)."""
client = mock_client_class.return_value
client.create_review.side_effect = [
APIError(422, "approve your own pull is not allowed"),
{"id": 303},
]
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-confirmed",
"--checklist-categories",
"1,2,3,4,5,6,7,8",
],
env={"REVIEWER_GITEA_API_TOKEN": "fake-reviewer", "CI_GITEA_API_TOKEN": "fake-ci"},
)
assert result.exit_code == 0
assert "Review #303" in result.output
assert "Retrying with CI token" in result.output
# Second call should still be APPROVE (CI token retry)
assert client.create_review.call_count == 2
assert client.create_review.call_args_list[1].kwargs.get("event") == "APPROVE"
@patch("devx.ci.pr_review.GiteaClient")
def test_manual_review_ci_token_also_fails_falls_back_to_comment(self, mock_client_class: MagicMock) -> None:
"""Self-approval + CI token retry also fails → fall back to COMMENT."""
client = mock_client_class.return_value
client.create_review.side_effect = [
APIError(422, "approve your own pull is not allowed"),
APIError(422, "approve your own pull is not allowed"),
{"id": 404},
]
runner = CliRunner()
result = runner.invoke(
main,
[
"42",
"oblachno-oss/devx",
"--event",
"APPROVE",
"--body",
"x" * 60,
"--checklist-confirmed",
"--checklist-categories",
"1,2,3,4,5,6,7,8",
],
env={"REVIEWER_GITEA_API_TOKEN": "fake-reviewer", "CI_GITEA_API_TOKEN": "fake-ci"},
)
assert result.exit_code == 0
assert "Review #404" in result.output
assert "CI token also cannot approve" in result.output
# Third call should be COMMENT (final fallback)
assert client.create_review.call_count == 3
assert client.create_review.call_args_list[2].kwargs.get("event") == "COMMENT"
@patch("devx.ci.pr_review.GiteaClient")
def test_manual_review_other_error_re_raises(self, mock_client_class: MagicMock) -> None:
client = mock_client_class.return_value
client.create_review.side_effect = APIError(500, "Internal server error")
runner = CliRunner()
result = runner.invoke(
main,
["42", "oblachno-oss/devx", "--event", "COMMENT", "--body", "x" * 60],
env={"CI_GITEA_TOKEN": "fake"},
)
assert result.exit_code != 0
def test_main_module_block() -> None:
import devx.ci.pr_review as pr
with patch.object(pr, "main") as mock_main:
with patch.object(pr, "__name__", "__main__"):
pr.main([])
mock_main.assert_called_once_with([])
+835
View File
@@ -0,0 +1,835 @@
"""Structural tests for spec-driven development workflows and skills.
These tests parse the actual workflow YAML files in each repo and assert
that the new spec-driven development steps, jobs, and env vars are present
and correctly wired. They also validate that the spec-driven-development
skill exists in each repo's .devin/skills/ directory with required sections.
This is a "contract test" — it verifies that the workflows we wrote match
the intended structure, catching regressions if someone edits a workflow
and accidentally removes a step or breaks a job dependency.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
# Repo root paths
# __file__ = .../devx/tests/unit/test_spec_driven_workflows.py
# parents[3] = .../oblachno (the monorepo root containing all repos)
_OBLACHNO_ROOT = Path(__file__).resolve().parents[3]
_INFRA = _OBLACHNO_ROOT / "infra"
_GRM = _OBLACHNO_ROOT / "grm"
_SSO_BRIDGE = _OBLACHNO_ROOT / "sso-bridge"
_DEVX = _OBLACHNO_ROOT / "devx"
def _load_workflow(repo_path: Path, filename: str) -> dict:
"""Load a workflow YAML file and return parsed dict."""
path = repo_path / ".gitea" / "workflows" / filename
if not path.exists():
pytest.skip(f"Workflow {filename} not found in {repo_path.name}")
with open(path, encoding="utf-8") as f:
return yaml.safe_load(f)
def _skip_if_repo_missing(repo_name: str) -> None:
"""Skip test if the sibling repo directory doesn't exist (CI only checks out one repo)."""
repo_path = _OBLACHNO_ROOT / repo_name
if not repo_path.is_dir():
pytest.skip(f"Repo {repo_name} not found at {repo_path} (CI only checks out devx)")
def _read_skill(repo_name: str, skill_name: str) -> str:
"""Read a skill file from a repo, skipping if the repo or file doesn't exist."""
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md"
if not skill_path.exists():
pytest.skip(f"SKILL.md not found in {repo_name}/{skill_name}")
return skill_path.read_text(encoding="utf-8")
def _get_step_names(job: dict) -> list[str]:
"""Extract step names from a job dict."""
names = []
for step in job.get("steps", []):
if "name" in step:
names.append(step["name"])
return names
def _find_step(job: dict, name_part: str) -> dict | None:
"""Find a step by partial name match."""
for step in job.get("steps", []):
if "name" in step and name_part.lower() in step["name"].lower():
return step
return None
def _get_run_commands(step: dict) -> str:
"""Get the run command from a step."""
return step.get("run", "")
# ============================================================================
# Infra ci.yml — spec validation + PR size + fast molecule
# ============================================================================
class TestInfraCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_INFRA, "ci.yml")
def test_validate_job_exists(self, workflow: dict) -> None:
assert "validate" in workflow["jobs"]
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps), "validate job must have 'Validate spec file' step"
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps), "validate job must have 'Check PR size' step"
def test_spec_validation_uses_correct_module(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.validate_spec" in cmd
assert "--github-output" in cmd
def test_pr_size_uses_correct_module(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Check PR size")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.check_pr_size" in cmd
assert "--github-output" in cmd
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "OBL-INFRA"
def test_has_fast_molecule_job(self, workflow: dict) -> None:
assert "fast-molecule" in workflow["jobs"], "ci.yml must have 'fast-molecule' job (replaced molecule-tests)"
def test_no_full_molecule_tests_job(self, workflow: dict) -> None:
assert "molecule-tests" not in workflow["jobs"], "ci.yml must NOT have 'molecule-tests' job (moved to nightly)"
def test_no_staging_deploy_in_ci(self, workflow: dict) -> None:
# The staging deploy was moved to post-merge (auto-deploy-staging)
job_names = list(workflow["jobs"].keys())
assert "staging-health-gate" not in job_names, "staging-health-gate removed from ci.yml (moved to nightly)"
assert "pre-deploy-checks" not in job_names, "pre-deploy-checks removed from ci.yml (moved to nightly)"
assert "deploy" not in job_names, "deploy job removed from ci.yml (moved to post-merge)"
def test_fast_molecule_uses_devx_module(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
step = _find_step(job, "Detect changed roles")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.fast_molecule" in cmd
assert "--github-output" in cmd
def test_fast_molecule_timeout_is_short(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
assert job.get("timeout-minutes", 999) <= 30, (
"fast-molecule timeout should be <= 30 min (was 120 for full suite)"
)
def test_fast_molecule_no_matrix(self, workflow: dict) -> None:
job = workflow["jobs"]["fast-molecule"]
assert "strategy" not in job or "matrix" not in job.get("strategy", {}), (
"fast-molecule should not use matrix (single runner)"
)
def test_auto_merge_depends_on_fast_molecule(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-merge", {})
needs = job.get("needs", [])
assert "fast-molecule" in needs, "auto-merge must depend on fast-molecule (not deploy)"
def test_auto_merge_does_not_depend_on_deploy(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-merge", {})
needs = job.get("needs", [])
assert "deploy" not in needs, "auto-merge must NOT depend on deploy (removed from PR pipeline)"
# ============================================================================
# Infra nightly.yml — full molecule + staging deploy + gate
# ============================================================================
class TestInfraNightlyWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_INFRA, "nightly.yml")
def test_nightly_workflow_exists(self, workflow: dict) -> None:
assert workflow is not None
def test_has_full_molecule_job(self, workflow: dict) -> None:
assert "full-molecule" in workflow["jobs"]
def test_has_set_gate_status_job(self, workflow: dict) -> None:
assert "set-gate-status" in workflow["jobs"]
def test_has_staging_deploy_job(self, workflow: dict) -> None:
assert "staging-deploy" in workflow["jobs"]
def test_full_molecule_uses_matrix(self, workflow: dict) -> None:
job = workflow["jobs"]["full-molecule"]
strategy = job.get("strategy", {})
assert "matrix" in strategy, "full-molecule must use matrix (6 runners)"
assert "runner-index" in strategy["matrix"]
def test_full_molecule_timeout_is_long(self, workflow: dict) -> None:
job = workflow["jobs"]["full-molecule"]
assert job.get("timeout-minutes", 0) >= 90, "full-molecule timeout should be >= 90 min (full suite)"
def test_set_gate_status_depends_on_full_molecule(self, workflow: dict) -> None:
job = workflow["jobs"]["set-gate-status"]
needs = job.get("needs", [])
assert "full-molecule" in needs
def test_set_gate_status_uses_nightly_gate_module(self, workflow: dict) -> None:
job = workflow["jobs"]["set-gate-status"]
step = _find_step(job, "Set nightly gate")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.nightly_gate" in cmd
assert "set-passed" in cmd or "set-failed" in cmd
def test_staging_deploy_depends_on_gate(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
needs = job.get("needs", [])
assert "set-gate-status" in needs
assert "full-molecule" in needs
def test_staging_deploy_only_on_success(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
if_cond = job.get("if", "")
assert "success" in if_cond, "staging-deploy must only run when full-molecule succeeds"
def test_nightly_runs_on_schedule(self, workflow: dict) -> None:
on = workflow.get("on", workflow.get(True, {}))
# YAML may parse 'on' as True (boolean)
if isinstance(on, dict):
assert "schedule" in on, "nightly must have schedule trigger"
else:
pytest.fail("Could not parse 'on' trigger from nightly.yml")
# ============================================================================
# Infra post-merge.yml — auto-deploy staging with nightly gate
# ============================================================================
class TestInfraPostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_INFRA, "post-merge.yml")
def test_has_auto_deploy_staging_job(self, workflow: dict) -> None:
assert "auto-deploy-staging" in workflow["jobs"], "post-merge must have 'auto-deploy-staging' job"
def test_has_staging_deploy_job(self, workflow: dict) -> None:
assert "staging-deploy" in workflow["jobs"], "post-merge must have 'staging-deploy' reusable workflow job"
def test_auto_deploy_staging_checks_nightly_gate(self, workflow: dict) -> None:
job = workflow["jobs"]["auto-deploy-staging"]
step = _find_step(job, "Check nightly gate")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.nightly_gate" in cmd
assert "--action check" in cmd
def test_staging_deploy_depends_on_auto_deploy_staging(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
needs = job.get("needs", [])
assert "auto-deploy-staging" in needs
def test_staging_deploy_gated_on_gate_passed(self, workflow: dict) -> None:
job = workflow["jobs"]["staging-deploy"]
if_cond = job.get("if", "")
assert "gate-passed" in if_cond, "staging-deploy must check gate-passed output"
def test_auto_deploy_production_waits_for_staging(self, workflow: dict) -> None:
job = workflow["jobs"].get("auto-deploy-production", {})
needs = job.get("needs", [])
assert "staging-deploy" in needs, "auto-deploy-production must wait for staging-deploy"
# ============================================================================
# GRM ci.yml — spec validation + PR size
# ============================================================================
class TestGrmCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_GRM, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "GRM"
# ============================================================================
# GRM post-merge.yml — auto-create infra dependency PR
# ============================================================================
class TestGrmPostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_GRM, "post-merge.yml")
def test_has_create_dependency_pr_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None, "grm post-merge must have 'Create infra dependency PR' step"
def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.create_dependency_pr" in cmd
assert "--package grm" in cmd
assert "--repo oblachno/infra" in cmd
def test_dependency_pr_is_best_effort(self, workflow: dict) -> None:
import re
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
# Must not fail the workflow if PR creation fails.
# The || echo may be split across lines with backslash continuation in YAML.
# Normalize: remove backslashes and collapse whitespace.
cmd_normalized = " ".join(cmd.replace("\\", " ").split())
assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step, (
"dependency PR step must be best-effort (|| echo or continue-on-error)"
)
# ============================================================================
# sso-bridge ci.yml — spec validation + PR size
# ============================================================================
class TestSsoBridgeCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_SSO_BRIDGE, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "SSO"
# ============================================================================
# sso-bridge post-merge.yml — auto-publish + auto-create dependency PR
# ============================================================================
class TestSsoBridgePostMergeWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_SSO_BRIDGE, "post-merge.yml")
def test_post_merge_workflow_exists(self, workflow: dict) -> None:
assert workflow is not None
def test_has_release_and_maintain_job(self, workflow: dict) -> None:
assert "release-and-maintain" in workflow["jobs"]
def test_has_create_dependency_pr_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
def test_dependency_pr_uses_correct_module(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
assert "devx.ci.create_dependency_pr" in cmd
assert "--package sso_bridge" in cmd
assert "--repo oblachno/infra" in cmd
def test_dependency_pr_is_best_effort(self, workflow: dict) -> None:
import re
job = workflow["jobs"].get("release-and-maintain", {})
step = _find_step(job, "Create infra dependency PR")
assert step is not None
cmd = _get_run_commands(step)
# The || echo may be split across lines with backslash continuation in YAML.
cmd_normalized = " ".join(cmd.replace("\\", " ").split())
assert bool(re.search(r"\|\|\s*echo", cmd_normalized)) or "continue-on-error" in step
def test_has_publish_step(self, workflow: dict) -> None:
job = workflow["jobs"].get("release-and-maintain", {})
steps = _get_step_names(job)
assert any("publish" in s.lower() for s in steps), "sso-bridge post-merge must have a publish step"
# ============================================================================
# devx ci.yml — spec validation + PR size
# ============================================================================
class TestDevxCiWorkflow:
@pytest.fixture
def workflow(self) -> dict:
return _load_workflow(_DEVX, "ci.yml")
def test_has_spec_validation_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Validate spec file" in s for s in steps)
def test_has_pr_size_check_step(self, workflow: dict) -> None:
steps = _get_step_names(workflow["jobs"]["validate"])
assert any("Check PR size" in s for s in steps)
def test_spec_validation_sets_task_prefix(self, workflow: dict) -> None:
step = _find_step(workflow["jobs"]["validate"], "Validate spec file")
assert step is not None
env = step.get("env", {})
assert env.get("DEVX_TASK_PREFIX") == "DEVX"
# ============================================================================
# Skill files — spec-driven-development SKILL.md in all repos
# ============================================================================
class TestSpecDrivenDevelopmentSkill:
REQUIRED_SECTIONS = [
"## Overview",
"## Workflow",
"## Spec Template",
"## CI Validation",
"## Acceptance Criteria",
]
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_skill_exists_in_repo(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
assert skill_path.exists(), f"SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_skill_has_required_sections(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
for section in self.REQUIRED_SECTIONS:
assert section in content, f"SKILL.md in {repo_name} missing section: {section}"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_skill_mentions_req_ids(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "REQ-" in content, f"SKILL.md in {repo_name} must mention REQ-ID format"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_skill_mentions_pr_size_limit(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "500" in content, f"SKILL.md in {repo_name} must mention 500 line PR size limit"
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_skill_mentions_nightly_gate(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
skill_path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
content = skill_path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), f"SKILL.md in {repo_name} must mention nightly gate"
def test_skill_exists_in_shared_dir(self) -> None:
skill_path = _OBLACHNO_ROOT / ".devin" / "skills" / "spec-driven-development" / "SKILL.md"
if not skill_path.exists():
pytest.skip("Shared .devin/skills/ not found (CI only checks out devx repo)")
assert skill_path.exists(), "SKILL.md not found in shared .devin/skills/"
# ============================================================================
# devx-workflow skill — exists in repos with PR workflow, mentions spec gates
# ============================================================================
class TestDevxWorkflowSkill:
# Repos that have a PR workflow and need the devx-workflow skill
REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"]
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_skill_exists(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
assert path.exists(), f"devx-workflow SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_spec_validation(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "validate_spec" in content, f"devx-workflow skill in {repo_name} must mention validate_spec"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_pr_size_check(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "check_pr_size" in content, f"devx-workflow skill in {repo_name} must mention check_pr_size"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_pr_workflow_commands(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "make create-pr" in content or "make push-with-pr" in content, (
f"devx-workflow skill in {repo_name} must mention PR creation commands"
)
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_auto_merge(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "auto-merge" in content.lower() or "ready-to-merge" in content, (
f"devx-workflow skill in {repo_name} must mention auto-merge"
)
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_has_correct_task_prefix(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
"""Each repo's devx-workflow skill must mention its correct task prefix."""
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
expected_prefixes = {
"infra": "OBL-INFRA",
"grm": "GRM",
"sso-bridge": "SSO",
"devx": "DEVX",
}
prefix = expected_prefixes[repo_name]
assert prefix in content, f"devx-workflow skill in {repo_name} must mention task prefix {prefix}"
def test_not_in_mattermost_oidc(self) -> None:
"""mattermost-oidc has no PR workflow — should NOT have devx-workflow skill."""
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
assert not path.exists(), "mattermost-oidc should NOT have devx-workflow skill (no PR workflow)"
# Repo-specific content checks
def test_infra_mentions_nightly_gate(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), "infra devx-workflow skill must mention nightly gate"
assert "nightly_gate" in content, "infra devx-workflow skill must mention devx.ci.nightly_gate module"
def test_infra_mentions_fast_molecule(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "fast_molecule" in content, "infra devx-workflow skill must mention devx.ci.fast_molecule"
def test_infra_mentions_auto_deploy_staging(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "staging" in content.lower(), "infra devx-workflow skill must mention staging auto-deploy"
def test_grm_mentions_dependency_pr(self) -> None:
_skip_if_repo_missing("grm")
path = _OBLACHNO_ROOT / "grm" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "create_dependency_pr" in content, "grm devx-workflow skill must mention create_dependency_pr"
def test_sso_bridge_mentions_dependency_pr(self) -> None:
_skip_if_repo_missing("sso-bridge")
path = _OBLACHNO_ROOT / "sso-bridge" / ".devin" / "skills" / "devx-workflow" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "create_dependency_pr" in content, "sso-bridge devx-workflow skill must mention create_dependency_pr"
# ============================================================================
# testing-and-debugging skill — exists in all repos, mentions spec workflow
# ============================================================================
class TestTestingAndDebuggingSkill:
# All repos have a testing-and-debugging skill
ALL_REPOS = ["infra", "grm", "sso-bridge", "devx", "mattermost-oidc"]
@pytest.mark.parametrize("repo_name", ALL_REPOS)
def test_skill_exists(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
assert path.exists(), f"testing-and-debugging SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", ALL_REPOS)
def test_has_required_sections(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
# All testing-and-debugging skills should have a CI failure investigation section
assert "CI Failure Investigation" in content or "CI failure" in content, (
f"testing-and-debugging skill in {repo_name} must have CI failure section"
)
# Repos with PR workflow should mention spec-driven workflow
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_mentions_spec_driven_workflow(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "spec" in content.lower(), (
f"testing-and-debugging skill in {repo_name} must mention spec-driven workflow"
)
def test_infra_mentions_nightly(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "nightly" in content.lower(), "infra testing-and-debugging skill must mention nightly tests"
def test_infra_mentions_fast_molecule(self) -> None:
_skip_if_repo_missing("infra")
path = _OBLACHNO_ROOT / "infra" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "fast" in content.lower() and "molecule" in content.lower(), (
"infra testing-and-debugging skill must mention fast molecule"
)
def test_mattermost_oidc_no_spec_mention(self) -> None:
"""mattermost-oidc has no spec-driven workflow — skill should NOT mention it."""
_skip_if_repo_missing("mattermost-oidc")
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "testing-and-debugging" / "SKILL.md"
content = path.read_text(encoding="utf-8")
# mattermost-oidc has no PR workflow, no spec validation
assert "validate_spec" not in content, (
"mattermost-oidc testing-and-debugging skill should NOT mention validate_spec"
)
# ============================================================================
# pr-review skill — deep review with auto-fix, exists in repos with PR workflow
# ============================================================================
class TestPrReviewSkill:
REPOS_WITH_PR_WORKFLOW = ["infra", "grm", "sso-bridge", "devx"]
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_skill_exists(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
assert path.exists(), f"pr-review SKILL.md not found in {repo_name}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_all_review_categories(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
required_categories = [
"Functional Correctness",
"Completeness",
"Architecture",
"Reliability",
"Robustness",
"Security",
"Technical Excellence",
"Test Quality",
]
for cat in required_categories:
assert cat in content, f"pr-review skill in {repo_name} missing category: {cat}"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_auto_fix(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "auto-fix" in content.lower() or "auto fix" in content.lower(), (
f"pr-review skill in {repo_name} must mention auto-fix"
)
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_gitea_mcp(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "mcp" in content.lower(), f"pr-review skill in {repo_name} must mention Gitea MCP"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_inline_comments(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "inline" in content.lower(), f"pr-review skill in {repo_name} must mention inline comments"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_ready_to_merge(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "ready-to-merge" in content, f"pr-review skill in {repo_name} must mention ready-to-merge label"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_resolve_discussion(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "resolve" in content.lower(), f"pr-review skill in {repo_name} must mention resolving discussions"
@pytest.mark.parametrize("repo_name", REPOS_WITH_PR_WORKFLOW)
def test_mentions_summary(self, repo_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / "pr-review" / "SKILL.md"
content = path.read_text(encoding="utf-8")
assert "summary" in content.lower(), f"pr-review skill in {repo_name} must mention posting a summary"
def test_not_in_mattermost_oidc(self) -> None:
"""mattermost-oidc has no PR workflow — should NOT have pr-review skill."""
path = _OBLACHNO_ROOT / "mattermost-oidc" / ".devin" / "skills" / "pr-review" / "SKILL.md"
assert not path.exists(), "mattermost-oidc should NOT have pr-review skill (no PR workflow)"
def test_no_pr_review_module_remains(self) -> None:
"""The old devx.ci.pr_review module should be deleted."""
path = _DEVX / "src" / "devx" / "ci" / "pr_review.py"
assert not path.exists(), "devx.ci.pr_review module should be deleted (replaced by pr-review skill)"
def test_no_pr_review_test_remains(self) -> None:
"""The old test_pr_review.py should be deleted."""
path = _DEVX / "tests" / "unit" / "test_pr_review.py"
assert not path.exists(), "tests/unit/test_pr_review.py should be deleted"
def test_no_pr_review_in_workflows(self) -> None:
"""No CI workflow should reference devx.ci.pr_review."""
for repo_name in ["infra", "grm", "sso-bridge", "devx"]:
wf_dir = _OBLACHNO_ROOT / repo_name / ".gitea" / "workflows"
if not wf_dir.exists():
continue
for wf_file in wf_dir.glob("*.yml"):
content = wf_file.read_text(encoding="utf-8")
assert "devx.ci.pr_review" not in content, (
f"{repo_name}/{wf_file.name} still references devx.ci.pr_review"
)
# ============================================================================
# Skill consistency — all skills have proper structure
# ============================================================================
class TestSkillConsistency:
ALL_SKILLS = [
("infra", "devx-workflow"),
("infra", "testing-and-debugging"),
("infra", "spec-driven-development"),
("infra", "pr-review"),
("grm", "devx-workflow"),
("grm", "testing-and-debugging"),
("grm", "spec-driven-development"),
("grm", "pr-review"),
("sso-bridge", "devx-workflow"),
("sso-bridge", "testing-and-debugging"),
("sso-bridge", "spec-driven-development"),
("sso-bridge", "pr-review"),
("devx", "devx-workflow"),
("devx", "testing-and-debugging"),
("devx", "spec-driven-development"),
("devx", "pr-review"),
("mattermost-oidc", "testing-and-debugging"),
]
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS)
def test_skill_has_title(self, repo_name: str, skill_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md"
content = path.read_text(encoding="utf-8")
first_line = content.strip().split("\n")[0]
assert first_line.startswith("# "), f"{repo_name}/{skill_name}: SKILL.md must start with a # title"
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS)
def test_skill_not_empty(self, repo_name: str, skill_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md"
content = path.read_text(encoding="utf-8").strip()
assert len(content) > 100, f"{repo_name}/{skill_name}: SKILL.md is too short ({len(content)} chars)"
@pytest.mark.parametrize("repo_name, skill_name", ALL_SKILLS)
def test_skill_has_sections(self, repo_name: str, skill_name: str) -> None:
_skip_if_repo_missing(repo_name)
path = _OBLACHNO_ROOT / repo_name / ".devin" / "skills" / skill_name / "SKILL.md"
content = path.read_text(encoding="utf-8")
# Must have at least 2 ## sections
section_count = content.count("\n## ")
assert section_count >= 2, (
f"{repo_name}/{skill_name}: SKILL.md must have at least 2 sections (found {section_count})"
)
# ============================================================================
# AGENTS.md — spec-driven development section in all repos
# ============================================================================
class TestAgentsMdSpecSection:
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_agents_md_has_spec_driven_section(self, repo_name: str) -> None:
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md"
if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}")
content = path.read_text(encoding="utf-8")
assert "## Spec-Driven Development" in content, (
f"AGENTS.md in {repo_name} must have '## Spec-Driven Development' section"
)
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_agents_md_mentions_validate_spec(self, repo_name: str) -> None:
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md"
if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}")
content = path.read_text(encoding="utf-8")
assert "validate_spec" in content or "devx.ci.validate_spec" in content, (
f"AGENTS.md in {repo_name} must mention devx.ci.validate_spec"
)
@pytest.mark.parametrize("repo_name", ["infra", "grm", "sso-bridge", "devx"])
def test_agents_md_pr_workflow_section_intact(self, repo_name: str) -> None:
"""Ensure the PR Workflow section wasn't accidentally deleted."""
path = _OBLACHNO_ROOT / repo_name / "AGENTS.md"
if not path.exists():
pytest.skip(f"AGENTS.md not found in {repo_name}")
content = path.read_text(encoding="utf-8")
assert "## PR Workflow" in content, f"AGENTS.md in {repo_name} must still have '## PR Workflow' section"
+223
View File
@@ -0,0 +1,223 @@
"""Unit tests for devx.ci.validate_spec."""
from pathlib import Path
from unittest.mock import patch
from click.testing import CliRunner
from devx.ci.validate_spec import (
AC_CHECKED_RE,
AC_UNCHECKED_RE,
REQ_ID_RE,
cli,
find_spec_file,
validate_spec_content,
)
VALID_SPEC = """\
# OBL-INFRA-531: Fix sso-bridge role for pip install
## Problem
The sso-bridge role uses scripts.sso_bridge.listener but the pip
package uses sso_bridge.listener.
## Approach
REQ-1: Update molecule verify.yml to use sso_bridge.listener
REQ-2: Add infra repo clone task to sso_bridge role
## Test Plan
- Run molecule test for sso_bridge role
- Verify pip package is installed correctly
## Deploy Plan
- Merge PR
- Auto-deploy to staging
## Rollback Plan
- Revert PR
- Re-deploy previous version
## Acceptance Criteria
- [x] Molecule test passes with sso_bridge.listener
- [x] Infra repo is cloned by sso_bridge role
"""
SPEC_MISSING_SECTION = """\
# OBL-INFRA-531: Fix sso-bridge
## Problem
Something is broken.
## Approach
REQ-1: Fix it
## Test Plan
Run tests
"""
SPEC_UNCHECKED_AC = """\
# OBL-INFRA-531: Fix sso-bridge
## Problem
Broken.
## Approach
REQ-1: Fix it
## Test Plan
Run tests
## Deploy Plan
Deploy
## Rollback Plan
Revert
## Acceptance Criteria
- [x] Fixed
- [ ] Verified in staging
"""
SPEC_NO_REQ_IDS = """\
# OBL-INFRA-531: Fix sso-bridge
## Problem
Broken.
## Approach
Fix it.
## Test Plan
Run tests
## Deploy Plan
Deploy
## Rollback Plan
Revert
## Acceptance Criteria
- [x] Fixed
"""
class TestFindSpecFile:
def test_finds_exact_match(self, tmp_path: Path) -> None:
specs_dir = tmp_path / "specs"
specs_dir.mkdir()
(specs_dir / "OBL-INFRA-531.md").write_text("content")
result = find_spec_file("OBL-INFRA-531", str(specs_dir))
assert result is not None
assert result.name == "OBL-INFRA-531.md"
def test_finds_case_insensitive(self, tmp_path: Path) -> None:
specs_dir = tmp_path / "specs"
specs_dir.mkdir()
(specs_dir / "obl-infra-531.md").write_text("content")
result = find_spec_file("OBL-INFRA-531", str(specs_dir))
assert result is not None
def test_returns_none_when_not_found(self, tmp_path: Path) -> None:
specs_dir = tmp_path / "specs"
specs_dir.mkdir()
result = find_spec_file("OBL-INFRA-999", str(specs_dir))
assert result is None
def test_returns_none_when_dir_missing(self, tmp_path: Path) -> None:
result = find_spec_file("OBL-INFRA-531", str(tmp_path / "nonexistent"))
assert result is None
class TestValidateSpecContent:
def test_valid_spec_passes(self) -> None:
errors = validate_spec_content(VALID_SPEC)
assert errors == []
def test_missing_sections(self) -> None:
errors = validate_spec_content(SPEC_MISSING_SECTION)
assert len(errors) >= 3 # Missing Deploy Plan, Rollback Plan, Acceptance Criteria
assert any("Deploy Plan" in e for e in errors)
assert any("Rollback Plan" in e for e in errors)
assert any("Acceptance Criteria" in e for e in errors)
def test_unchecked_ac_fails(self) -> None:
errors = validate_spec_content(SPEC_UNCHECKED_AC)
assert len(errors) == 1
assert "unchecked" in errors[0].lower()
def test_no_req_ids_fails(self) -> None:
errors = validate_spec_content(SPEC_NO_REQ_IDS)
assert any("REQ-ID" in e for e in errors)
def test_empty_content_fails(self) -> None:
errors = validate_spec_content("")
assert len(errors) >= 2 # Missing sections + no REQ-IDs
class TestRegexPatterns:
def test_req_id_re_matches(self) -> None:
assert REQ_ID_RE.search("REQ-1: Do something")
assert REQ_ID_RE.search("REQ-42: Another thing")
assert not REQ_ID_RE.search("REQ: no number")
def test_ac_checked_re_matches(self) -> None:
assert AC_CHECKED_RE.search("- [x] Done")
assert AC_CHECKED_RE.search(" - [x] Indented")
assert not AC_CHECKED_RE.search("- [ ] Not done")
def test_ac_unchecked_re_matches(self) -> None:
assert AC_UNCHECKED_RE.search("- [ ] Not done")
assert AC_UNCHECKED_RE.search(" - [ ] Indented")
assert not AC_UNCHECKED_RE.search("- [x] Done")
class TestCli:
def test_fails_without_task_id(self) -> None:
runner = CliRunner()
result = runner.invoke(cli, ["--branch", "no-task-id"])
assert result.exit_code != 0
def test_allow_missing_succeeds_without_task_id(self) -> None:
runner = CliRunner()
result = runner.invoke(cli, ["--branch", "no-task-id", "--allow-missing"])
assert result.exit_code == 0
assert "WARNING" in result.output
def test_fails_when_spec_not_found(self, tmp_path: Path) -> None:
runner = CliRunner()
with patch("devx.ci.validate_spec.extract_task_id", return_value="OBL-INFRA-999"):
result = runner.invoke(
cli,
["--branch", "OBL-INFRA-999-test", "--specs-dir", str(tmp_path / "specs")],
)
assert result.exit_code != 0
assert "No spec file found" in result.output
def test_passes_with_valid_spec(self, tmp_path: Path) -> None:
specs_dir = tmp_path / "specs"
specs_dir.mkdir()
(specs_dir / "OBL-INFRA-531.md").write_text(VALID_SPEC)
runner = CliRunner()
with patch("devx.ci.validate_spec.extract_task_id", return_value="OBL-INFRA-531"):
result = runner.invoke(
cli,
["--branch", "OBL-INFRA-531-fix-foo", "--specs-dir", str(specs_dir)],
)
assert result.exit_code == 0
assert "Spec validated" in result.output
def test_fails_with_unchecked_ac(self, tmp_path: Path) -> None:
specs_dir = tmp_path / "specs"
specs_dir.mkdir()
(specs_dir / "OBL-INFRA-531.md").write_text(SPEC_UNCHECKED_AC)
runner = CliRunner()
with patch("devx.ci.validate_spec.extract_task_id", return_value="OBL-INFRA-531"):
result = runner.invoke(
cli,
["--branch", "OBL-INFRA-531-fix-foo", "--specs-dir", str(specs_dir)],
)
assert result.exit_code != 0
assert "unchecked" in result.output.lower()