feat: cross-runner molecule cancellation via Gitea API polling
Gitea Actions does not implement fail-fast/max-parallel for matrix jobs, so a failing runner does not stop the others. Added molecule_ci_guard.py which polls the Gitea API in a background thread. If any other molecule runner reports failure, the current runner kills its molecule subprocess and exits early. CI returns to a 3-runner matrix; each runner executes its assigned pairs in parallel via run_molecule_parallel.py, guarded by molecule_ci_guard.py. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Emil Simeonov
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
78c9b635e9
commit
92ca7ef7bb
@@ -0,0 +1,325 @@
|
||||
"""Unit tests for scripts/molecule_ci_guard.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess # nosec B404
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from scripts.molecule_ci_guard import (
|
||||
any_other_runner_failed,
|
||||
cli,
|
||||
get_running_jobs,
|
||||
poll_for_other_failures,
|
||||
)
|
||||
|
||||
|
||||
class TestGetRunningJobs:
|
||||
def test_returns_jobs(self) -> None:
|
||||
with patch("scripts.molecule_ci_guard.requests.get") as mock_get:
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"jobs": [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
}
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
jobs = get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
assert len(jobs) == 2
|
||||
mock_get.assert_called_once()
|
||||
|
||||
def test_raises_on_request_error(self) -> None:
|
||||
with patch("scripts.molecule_ci_guard.requests.get") as mock_get:
|
||||
mock_get.side_effect = requests.RequestException("boom")
|
||||
with pytest.raises(requests.RequestException):
|
||||
get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
|
||||
|
||||
class TestAnyOtherRunnerFailed:
|
||||
def test_detects_other_failure(self) -> None:
|
||||
jobs = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (2)", "conclusion": "running"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is True
|
||||
|
||||
def test_ignores_current_runner(self) -> None:
|
||||
jobs = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
|
||||
|
||||
def test_ignores_non_molecule_jobs(self) -> None:
|
||||
jobs = [
|
||||
{"name": "quality", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
|
||||
|
||||
|
||||
class TestPollForOtherFailures:
|
||||
def test_sets_failed_event_when_other_runner_fails(self) -> None:
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
def side_effect(*args, **kwargs):
|
||||
if stop_event.wait.call_count < 1:
|
||||
return [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
return []
|
||||
|
||||
with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = side_effect
|
||||
stop_event.is_set.side_effect = [False, False]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"molecule-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_called_once()
|
||||
|
||||
def test_poll_warns_on_api_error(self) -> None:
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = requests.RequestException("boom")
|
||||
stop_event.is_set.side_effect = [False, True]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"molecule-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_not_called()
|
||||
|
||||
|
||||
class TestCli:
|
||||
def test_runs_without_api_env(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
env = {"PATH": os.environ.get("PATH", "")}
|
||||
with (
|
||||
patch.dict(os.environ, env, clear=True),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
):
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_exits_when_other_runner_fails(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"REPO_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "oblachno-oss/grm",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
mock_get_jobs.return_value = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
mock_killpg.assert_called()
|
||||
|
||||
def test_default_owner_repo_fallback(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"REPO_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "invalid",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
):
|
||||
mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_sleeps_while_waiting_for_process(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"REPO_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "oblachno-oss/grm",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("time.sleep") as mock_sleep,
|
||||
):
|
||||
mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
|
||||
proc = MagicMock()
|
||||
proc.poll.side_effect = [None, 0]
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
mock_sleep.assert_called()
|
||||
|
||||
def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"REPO_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "oblachno-oss/grm",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
mock_killpg.side_effect = ProcessLookupError("no such process")
|
||||
mock_get_jobs.return_value = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_exits_when_other_runner_fails_with_timeout(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"REPO_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "oblachno-oss/grm",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
|
||||
mock_get_jobs.return_value = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
|
||||
|
||||
def test_main_module_block() -> None:
|
||||
import scripts.molecule_ci_guard as mg
|
||||
|
||||
with open(mg.__file__) as f:
|
||||
source = f.read()
|
||||
source = source.replace('if __name__ == "__main__":\n cli()\n', "")
|
||||
namespace = dict(mg.__dict__)
|
||||
exec(compile(source, mg.__file__, "exec"), namespace)
|
||||
assert callable(namespace["cli"])
|
||||
Reference in New Issue
Block a user