diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index e6497a5..4e57acd 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -29,16 +29,28 @@ jobs: molecule-tests: needs: quality runs-on: docker + strategy: + matrix: + runner-index: [0, 1, 2] steps: - uses: actions/checkout@v4 - name: Set up environment run: make setup - - name: Run all molecule tests in parallel + - name: Discover assigned test pairs + run: | + . .venv/bin/activate + PAIRS=$(python3 scripts/distribute_molecule.py --runner-index ${{ matrix.runner-index }} --max-runners 3) + echo "Assigned pairs: $PAIRS" + echo "TEST_PAIRS=$PAIRS" >> $GITHUB_ENV + - name: Run molecule tests run: | set -euo pipefail . .venv/bin/activate - export DOCKER_HOST="unix:///run/user/$(id -u)/docker.sock" - export ANSIBLE_INJECT_INVOCATION=1 - PAIRS=$(python3 scripts/distribute_molecule.py --runner-index 0 --max-runners 1) - echo "Running all pairs in parallel: $PAIRS" - python3 scripts/run_molecule_parallel.py $PAIRS + python3 scripts/molecule_ci_guard.py $TEST_PAIRS + env: + GITEA_URL: ${{ github.server_url }} + REPO_TOKEN: ${{ secrets.REPO_TOKEN }} + RUN_ID: ${{ github.run_id }} + JOB_NAME: ${{ github.job }} + MATRIX_INDEX: ${{ matrix.runner-index }} + GITEA_REPOSITORY: ${{ github.repository }} diff --git a/scripts/molecule_ci_guard.py b/scripts/molecule_ci_guard.py new file mode 100644 index 0000000..f5c8c0a --- /dev/null +++ b/scripts/molecule_ci_guard.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Run molecule tests while polling Gitea for other runner failures. + +Usage: + python3 scripts/molecule_ci_guard.py ... + +Environment variables: + GITEA_URL Base URL of the Gitea instance. + REPO_TOKEN API token with repo access. + RUN_ID Workflow run ID (GITHUB_RUN_ID). + JOB_NAME Base job name (GITHUB_JOB), e.g. "molecule-tests". + MATRIX_INDEX Current matrix index (runner-index). + OWNER, REPO Repository owner/name. +""" + +from __future__ import annotations + +import os +import signal +import subprocess # nosec B404 +import sys +import threading +import time +from pathlib import Path + +import click +import requests + +from gitea_runner_manager.i18n import _ + +POLL_INTERVAL = 10 + + +def get_running_jobs( + gitea_url: str, owner: str, repo: str, token: str, run_id: int +) -> list[dict]: + """Return jobs for the given workflow run.""" + url = f"{gitea_url}/api/v1/repos/{owner}/{repo}/actions/runs/{run_id}/jobs" + headers = {"Authorization": f"token {token}"} + response = requests.get(url, headers=headers, timeout=30) + response.raise_for_status() + data = response.json() + return data.get("jobs", []) + + +def any_other_runner_failed( + jobs: list[dict], current_job_name: str, current_index: int +) -> bool: + """Return True if any other molecule matrix job has failed.""" + for job in jobs: + name = job.get("name", "") + if not name.startswith(current_job_name): + continue + # Skip the current runner instance + if name == f"{current_job_name} ({current_index})" or name == current_job_name: + continue + if job.get("conclusion") == "failure": + return True + return False + + +def poll_for_other_failures( + gitea_url: str, + owner: str, + repo: str, + token: str, + run_id: int, + job_name: str, + current_index: int, + stop_event: threading.Event, + failed_event: threading.Event, +) -> None: + """Background thread: poll API and signal if another runner fails.""" + while not stop_event.is_set(): + try: + jobs = get_running_jobs(gitea_url, owner, repo, token, run_id) + if any_other_runner_failed(jobs, job_name, current_index): + click.echo( + _( + "Another molecule runner failed. Stopping this runner early." + ) + ) + failed_event.set() + return + except requests.RequestException as exc: + click.echo(_("API poll warning: {exc}", exc=exc)) + stop_event.wait(POLL_INTERVAL) + + +@click.command() +@click.argument("pairs", nargs=-1, required=True) +def cli(pairs: tuple[str, ...]) -> None: + """Run molecule pairs and stop if another CI runner fails.""" + gitea_url = os.environ.get("GITEA_URL", "") + token = os.environ.get("REPO_TOKEN", "") + run_id = int(os.environ.get("RUN_ID", "0")) + job_name = os.environ.get("JOB_NAME", "molecule-tests") + current_index = int(os.environ.get("MATRIX_INDEX", "0")) + repository = os.environ.get("GITEA_REPOSITORY", "oblachno-oss/grm") + owner, sep, repo = repository.partition("/") + if not owner or not repo: + owner, repo = "oblachno-oss", "grm" + + if not all([gitea_url, token, run_id]): + click.echo( + _( + "GITEA_URL/REPO_TOKEN/RUN_ID not set; running without cross-runner cancellation." + ) + ) + + repo_root = Path(__file__).resolve().parent.parent + script_path = repo_root / "scripts" / "run_molecule_parallel.py" + + env = os.environ.copy() + env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock") + env.setdefault("ANSIBLE_INJECT_INVOCATION", "1") + + click.echo(_("Starting molecule runner with {count} pairs.", count=len(pairs))) + + process = subprocess.Popen( # nosec B603 + [sys.executable, str(script_path), *pairs], + env=env, + stdout=None, + stderr=None, + preexec_fn=os.setsid, + ) + + stop_event = threading.Event() + failed_event = threading.Event() + + if gitea_url and token and run_id: + poller = threading.Thread( + target=poll_for_other_failures, + args=( + gitea_url, + owner, + repo, + token, + run_id, + job_name, + current_index, + stop_event, + failed_event, + ), + daemon=True, + ) + poller.start() + + try: + while process.poll() is None: + if failed_event.is_set(): + try: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + sys.exit(1) + time.sleep(1) + finally: + stop_event.set() + + sys.exit(process.returncode) + + +if __name__ == "__main__": # pragma: no cover + cli() diff --git a/tests/unit/test_molecule_ci_guard.py b/tests/unit/test_molecule_ci_guard.py new file mode 100644 index 0000000..23174d2 --- /dev/null +++ b/tests/unit/test_molecule_ci_guard.py @@ -0,0 +1,325 @@ +"""Unit tests for scripts/molecule_ci_guard.py.""" + +from __future__ import annotations + +import os +import subprocess # nosec B404 +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from scripts.molecule_ci_guard import ( + any_other_runner_failed, + cli, + get_running_jobs, + poll_for_other_failures, +) + + +class TestGetRunningJobs: + def test_returns_jobs(self) -> None: + with patch("scripts.molecule_ci_guard.requests.get") as mock_get: + mock_response = MagicMock() + mock_response.json.return_value = { + "jobs": [ + {"name": "molecule-tests (0)", "conclusion": "success"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + } + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + jobs = get_running_jobs("https://gitea.example", "owner", "repo", "token", 123) + assert len(jobs) == 2 + mock_get.assert_called_once() + + def test_raises_on_request_error(self) -> None: + with patch("scripts.molecule_ci_guard.requests.get") as mock_get: + mock_get.side_effect = requests.RequestException("boom") + with pytest.raises(requests.RequestException): + get_running_jobs("https://gitea.example", "owner", "repo", "token", 123) + + +class TestAnyOtherRunnerFailed: + def test_detects_other_failure(self) -> None: + jobs = [ + {"name": "molecule-tests (0)", "conclusion": "success"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + {"name": "molecule-tests (2)", "conclusion": "running"}, + ] + assert any_other_runner_failed(jobs, "molecule-tests", 0) is True + + def test_ignores_current_runner(self) -> None: + jobs = [ + {"name": "molecule-tests (0)", "conclusion": "failure"}, + {"name": "molecule-tests (1)", "conclusion": "success"}, + ] + assert any_other_runner_failed(jobs, "molecule-tests", 0) is False + + def test_ignores_non_molecule_jobs(self) -> None: + jobs = [ + {"name": "quality", "conclusion": "failure"}, + {"name": "molecule-tests (1)", "conclusion": "success"}, + ] + assert any_other_runner_failed(jobs, "molecule-tests", 0) is False + + +class TestPollForOtherFailures: + def test_sets_failed_event_when_other_runner_fails(self) -> None: + stop_event = MagicMock() + failed_event = MagicMock() + + def side_effect(*args, **kwargs): + if stop_event.wait.call_count < 1: + return [ + {"name": "molecule-tests (0)", "conclusion": "success"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + return [] + + with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs: + mock_get_jobs.side_effect = side_effect + stop_event.is_set.side_effect = [False, False] + stop_event.wait.return_value = True + + poll_for_other_failures( + "https://gitea.example", + "owner", + "repo", + "token", + 123, + "molecule-tests", + 0, + stop_event, + failed_event, + ) + + failed_event.set.assert_called_once() + + def test_poll_warns_on_api_error(self) -> None: + stop_event = MagicMock() + failed_event = MagicMock() + + with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs: + mock_get_jobs.side_effect = requests.RequestException("boom") + stop_event.is_set.side_effect = [False, True] + stop_event.wait.return_value = True + + poll_for_other_failures( + "https://gitea.example", + "owner", + "repo", + "token", + 123, + "molecule-tests", + 0, + stop_event, + failed_event, + ) + + failed_event.set.assert_not_called() + + +class TestCli: + def test_runs_without_api_env(self) -> None: + from click.testing import CliRunner + + env = {"PATH": os.environ.get("PATH", "")} + with ( + patch.dict(os.environ, env, clear=True), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + ): + proc = MagicMock() + proc.poll.return_value = 0 + proc.returncode = 0 + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 0 + + def test_exits_when_other_runner_fails(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + mock_get_jobs.return_value = [ + {"name": "molecule-tests (0)", "conclusion": "running"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + + proc = MagicMock() + proc.poll.return_value = None + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + mock_killpg.assert_called() + + def test_default_owner_repo_fallback(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "invalid", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + ): + mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}] + proc = MagicMock() + proc.poll.return_value = 0 + proc.returncode = 0 + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 0 + + def test_sleeps_while_waiting_for_process(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("time.sleep") as mock_sleep, + ): + mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}] + proc = MagicMock() + proc.poll.side_effect = [None, 0] + proc.returncode = 0 + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 0 + mock_sleep.assert_called() + + def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + mock_killpg.side_effect = ProcessLookupError("no such process") + mock_get_jobs.return_value = [ + {"name": "molecule-tests (0)", "conclusion": "running"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + + proc = MagicMock() + proc.poll.return_value = None + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + + def test_exits_when_other_runner_fails_with_timeout(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + mock_killpg.side_effect = [None, ProcessLookupError("no such process")] + mock_get_jobs.return_value = [ + {"name": "molecule-tests (0)", "conclusion": "running"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + + proc = MagicMock() + proc.poll.return_value = None + proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)] + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + + +def test_main_module_block() -> None: + import scripts.molecule_ci_guard as mg + + with open(mg.__file__) as f: + source = f.read() + source = source.replace('if __name__ == "__main__":\n cli()\n', "") + namespace = dict(mg.__dict__) + exec(compile(source, mg.__file__, "exec"), namespace) + assert callable(namespace["cli"])