feat: cross-runner molecule cancellation via Gitea API polling

Gitea Actions does not implement fail-fast/max-parallel for matrix jobs,
so a failing runner does not stop the others. Added molecule_ci_guard.py
which polls the Gitea API in a background thread. If any other molecule
runner reports failure, the current runner kills its molecule subprocess
and exits early.

CI returns to a 3-runner matrix; each runner executes its assigned pairs
in parallel via run_molecule_parallel.py, guarded by molecule_ci_guard.py.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.gitea/workflows/ci.yml

@@ -29,16 +29,28 @@ jobs:
   molecule-tests:
     needs: quality
     runs-on: docker
+    strategy:
+      matrix:
+        runner-index: [0, 1, 2]
     steps:
       - uses: actions/checkout@v4
       - name: Set up environment
         run: make setup
-      - name: Run all molecule tests in parallel
+      - name: Discover assigned test pairs
+        run: |
+          . .venv/bin/activate
+          PAIRS=$(python3 scripts/distribute_molecule.py --runner-index ${{ matrix.runner-index }} --max-runners 3)
+          echo "Assigned pairs: $PAIRS"
+          echo "TEST_PAIRS=$PAIRS" >> $GITHUB_ENV
+      - name: Run molecule tests
         run: |
           set -euo pipefail
           . .venv/bin/activate
-          export DOCKER_HOST="unix:///run/user/$(id -u)/docker.sock"
-          export ANSIBLE_INJECT_INVOCATION=1
-          PAIRS=$(python3 scripts/distribute_molecule.py --runner-index 0 --max-runners 1)
-          echo "Running all pairs in parallel: $PAIRS"
-          python3 scripts/run_molecule_parallel.py $PAIRS
+          python3 scripts/molecule_ci_guard.py $TEST_PAIRS
+        env:
+          GITEA_URL: ${{ github.server_url }}
+          REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
+          RUN_ID: ${{ github.run_id }}
+          JOB_NAME: ${{ github.job }}
+          MATRIX_INDEX: ${{ matrix.runner-index }}
+          GITEA_REPOSITORY: ${{ github.repository }}

scripts/molecule_ci_guard.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Run molecule tests while polling Gitea for other runner failures.
+
+Usage:
+    python3 scripts/molecule_ci_guard.py <pair1> <pair2> ...
+
+Environment variables:
+    GITEA_URL       Base URL of the Gitea instance.
+    REPO_TOKEN      API token with repo access.
+    RUN_ID          Workflow run ID (GITHUB_RUN_ID).
+    JOB_NAME        Base job name (GITHUB_JOB), e.g. "molecule-tests".
+    MATRIX_INDEX    Current matrix index (runner-index).
+    OWNER, REPO     Repository owner/name.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess  # nosec B404
+import sys
+import threading
+import time
+from pathlib import Path
+
+import click
+import requests
+
+from gitea_runner_manager.i18n import _
+
+POLL_INTERVAL = 10
+
+
+def get_running_jobs(
+    gitea_url: str, owner: str, repo: str, token: str, run_id: int
+) -> list[dict]:
+    """Return jobs for the given workflow run."""
+    url = f"{gitea_url}/api/v1/repos/{owner}/{repo}/actions/runs/{run_id}/jobs"
+    headers = {"Authorization": f"token {token}"}
+    response = requests.get(url, headers=headers, timeout=30)
+    response.raise_for_status()
+    data = response.json()
+    return data.get("jobs", [])
+
+
+def any_other_runner_failed(
+    jobs: list[dict], current_job_name: str, current_index: int
+) -> bool:
+    """Return True if any other molecule matrix job has failed."""
+    for job in jobs:
+        name = job.get("name", "")
+        if not name.startswith(current_job_name):
+            continue
+        # Skip the current runner instance
+        if name == f"{current_job_name} ({current_index})" or name == current_job_name:
+            continue
+        if job.get("conclusion") == "failure":
+            return True
+    return False
+
+
+def poll_for_other_failures(
+    gitea_url: str,
+    owner: str,
+    repo: str,
+    token: str,
+    run_id: int,
+    job_name: str,
+    current_index: int,
+    stop_event: threading.Event,
+    failed_event: threading.Event,
+) -> None:
+    """Background thread: poll API and signal if another runner fails."""
+    while not stop_event.is_set():
+        try:
+            jobs = get_running_jobs(gitea_url, owner, repo, token, run_id)
+            if any_other_runner_failed(jobs, job_name, current_index):
+                click.echo(
+                    _(
+                        "Another molecule runner failed. Stopping this runner early."
+                    )
+                )
+                failed_event.set()
+                return
+        except requests.RequestException as exc:
+            click.echo(_("API poll warning: {exc}", exc=exc))
+        stop_event.wait(POLL_INTERVAL)
+
+
+@click.command()
+@click.argument("pairs", nargs=-1, required=True)
+def cli(pairs: tuple[str, ...]) -> None:
+    """Run molecule pairs and stop if another CI runner fails."""
+    gitea_url = os.environ.get("GITEA_URL", "")
+    token = os.environ.get("REPO_TOKEN", "")
+    run_id = int(os.environ.get("RUN_ID", "0"))
+    job_name = os.environ.get("JOB_NAME", "molecule-tests")
+    current_index = int(os.environ.get("MATRIX_INDEX", "0"))
+    repository = os.environ.get("GITEA_REPOSITORY", "oblachno-oss/grm")
+    owner, sep, repo = repository.partition("/")
+    if not owner or not repo:
+        owner, repo = "oblachno-oss", "grm"
+
+    if not all([gitea_url, token, run_id]):
+        click.echo(
+            _(
+                "GITEA_URL/REPO_TOKEN/RUN_ID not set; running without cross-runner cancellation."
+            )
+        )
+
+    repo_root = Path(__file__).resolve().parent.parent
+    script_path = repo_root / "scripts" / "run_molecule_parallel.py"
+
+    env = os.environ.copy()
+    env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock")
+    env.setdefault("ANSIBLE_INJECT_INVOCATION", "1")
+
+    click.echo(_("Starting molecule runner with {count} pairs.", count=len(pairs)))
+
+    process = subprocess.Popen(  # nosec B603
+        [sys.executable, str(script_path), *pairs],
+        env=env,
+        stdout=None,
+        stderr=None,
+        preexec_fn=os.setsid,
+    )
+
+    stop_event = threading.Event()
+    failed_event = threading.Event()
+
+    if gitea_url and token and run_id:
+        poller = threading.Thread(
+            target=poll_for_other_failures,
+            args=(
+                gitea_url,
+                owner,
+                repo,
+                token,
+                run_id,
+                job_name,
+                current_index,
+                stop_event,
+                failed_event,
+            ),
+            daemon=True,
+        )
+        poller.start()
+
+    try:
+        while process.poll() is None:
+            if failed_event.is_set():
+                try:
+                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                except ProcessLookupError:
+                    pass
+                try:
+                    process.wait(timeout=10)
+                except subprocess.TimeoutExpired:
+                    try:
+                        os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+                    process.wait()
+                sys.exit(1)
+            time.sleep(1)
+    finally:
+        stop_event.set()
+
+    sys.exit(process.returncode)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

tests/unit/test_molecule_ci_guard.py

@@ -0,0 +1,325 @@
+"""Unit tests for scripts/molecule_ci_guard.py."""
+
+from __future__ import annotations
+
+import os
+import subprocess  # nosec B404
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from scripts.molecule_ci_guard import (
+    any_other_runner_failed,
+    cli,
+    get_running_jobs,
+    poll_for_other_failures,
+)
+
+
+class TestGetRunningJobs:
+    def test_returns_jobs(self) -> None:
+        with patch("scripts.molecule_ci_guard.requests.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = {
+                "jobs": [
+                    {"name": "molecule-tests (0)", "conclusion": "success"},
+                    {"name": "molecule-tests (1)", "conclusion": "failure"},
+                ]
+            }
+            mock_response.raise_for_status.return_value = None
+            mock_get.return_value = mock_response
+
+            jobs = get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
+            assert len(jobs) == 2
+            mock_get.assert_called_once()
+
+    def test_raises_on_request_error(self) -> None:
+        with patch("scripts.molecule_ci_guard.requests.get") as mock_get:
+            mock_get.side_effect = requests.RequestException("boom")
+            with pytest.raises(requests.RequestException):
+                get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
+
+
+class TestAnyOtherRunnerFailed:
+    def test_detects_other_failure(self) -> None:
+        jobs = [
+            {"name": "molecule-tests (0)", "conclusion": "success"},
+            {"name": "molecule-tests (1)", "conclusion": "failure"},
+            {"name": "molecule-tests (2)", "conclusion": "running"},
+        ]
+        assert any_other_runner_failed(jobs, "molecule-tests", 0) is True
+
+    def test_ignores_current_runner(self) -> None:
+        jobs = [
+            {"name": "molecule-tests (0)", "conclusion": "failure"},
+            {"name": "molecule-tests (1)", "conclusion": "success"},
+        ]
+        assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
+
+    def test_ignores_non_molecule_jobs(self) -> None:
+        jobs = [
+            {"name": "quality", "conclusion": "failure"},
+            {"name": "molecule-tests (1)", "conclusion": "success"},
+        ]
+        assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
+
+
+class TestPollForOtherFailures:
+    def test_sets_failed_event_when_other_runner_fails(self) -> None:
+        stop_event = MagicMock()
+        failed_event = MagicMock()
+
+        def side_effect(*args, **kwargs):
+            if stop_event.wait.call_count < 1:
+                return [
+                    {"name": "molecule-tests (0)", "conclusion": "success"},
+                    {"name": "molecule-tests (1)", "conclusion": "failure"},
+                ]
+            return []
+
+        with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
+            mock_get_jobs.side_effect = side_effect
+            stop_event.is_set.side_effect = [False, False]
+            stop_event.wait.return_value = True
+
+            poll_for_other_failures(
+                "https://gitea.example",
+                "owner",
+                "repo",
+                "token",
+                123,
+                "molecule-tests",
+                0,
+                stop_event,
+                failed_event,
+            )
+
+            failed_event.set.assert_called_once()
+
+    def test_poll_warns_on_api_error(self) -> None:
+        stop_event = MagicMock()
+        failed_event = MagicMock()
+
+        with patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
+            mock_get_jobs.side_effect = requests.RequestException("boom")
+            stop_event.is_set.side_effect = [False, True]
+            stop_event.wait.return_value = True
+
+            poll_for_other_failures(
+                "https://gitea.example",
+                "owner",
+                "repo",
+                "token",
+                123,
+                "molecule-tests",
+                0,
+                stop_event,
+                failed_event,
+            )
+
+            failed_event.set.assert_not_called()
+
+
+class TestCli:
+    def test_runs_without_api_env(self) -> None:
+        from click.testing import CliRunner
+
+        env = {"PATH": os.environ.get("PATH", "")}
+        with (
+            patch.dict(os.environ, env, clear=True),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+        ):
+            proc = MagicMock()
+            proc.poll.return_value = 0
+            proc.returncode = 0
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 0
+
+    def test_exits_when_other_runner_fails(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+            mock_get_jobs.return_value = [
+                {"name": "molecule-tests (0)", "conclusion": "running"},
+                {"name": "molecule-tests (1)", "conclusion": "failure"},
+            ]
+
+            proc = MagicMock()
+            proc.poll.return_value = None
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+            mock_killpg.assert_called()
+
+    def test_default_owner_repo_fallback(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "invalid",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+        ):
+            mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
+            proc = MagicMock()
+            proc.poll.return_value = 0
+            proc.returncode = 0
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 0
+
+    def test_sleeps_while_waiting_for_process(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("time.sleep") as mock_sleep,
+        ):
+            mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
+            proc = MagicMock()
+            proc.poll.side_effect = [None, 0]
+            proc.returncode = 0
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 0
+            mock_sleep.assert_called()
+
+    def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+            mock_killpg.side_effect = ProcessLookupError("no such process")
+            mock_get_jobs.return_value = [
+                {"name": "molecule-tests (0)", "conclusion": "running"},
+                {"name": "molecule-tests (1)", "conclusion": "failure"},
+            ]
+
+            proc = MagicMock()
+            proc.poll.return_value = None
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+
+    def test_exits_when_other_runner_fails_with_timeout(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+            mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
+            mock_get_jobs.return_value = [
+                {"name": "molecule-tests (0)", "conclusion": "running"},
+                {"name": "molecule-tests (1)", "conclusion": "failure"},
+            ]
+
+            proc = MagicMock()
+            proc.poll.return_value = None
+            proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+
+
+def test_main_module_block() -> None:
+    import scripts.molecule_ci_guard as mg
+
+    with open(mg.__file__) as f:
+        source = f.read()
+    source = source.replace('if __name__ == "__main__":\n    cli()\n', "")
+    namespace = dict(mg.__dict__)
+    exec(compile(source, mg.__file__, "exec"), namespace)
+    assert callable(namespace["cli"])