fix: run molecule pairs sequentially within each CI runner

Parallel molecule execution within a single runner caused conflicts
(shared temp directories, Docker network collisions). Rewrote
molecule_ci_guard.py to run pairs sequentially while still polling
the Gitea API for cross-runner cancellation.

Each pair now gets its own subprocess with proper environment setup
(MOLECULE_PLATFORM_NAME/IMAGE/COMMAND), and output streams directly
to CI logs for debugging.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

scripts/molecule_ci_guard.py

@@ -1,20 +1,29 @@
 #!/usr/bin/env python3
-"""Run molecule tests while polling Gitea for other runner failures.
+"""Run molecule tests sequentially while polling Gitea for other runner failures.
+
+Each pair is encoded as ``scenario|platform_name|platform_image|platform_command``.
+Pairs are executed one at a time (molecule scenarios share temp directories and
+Docker networks, so parallel execution within a single runner is unsafe).
+
+A background thread polls the Gitea API. If any other molecule matrix runner
+reports failure, the current molecule subprocess is killed and this runner
+exits early with code 1.
 
 Usage:
     python3 scripts/molecule_ci_guard.py <pair1> <pair2> ...
 
 Environment variables:
-    GITEA_URL       Base URL of the Gitea instance.
+    GITEA_URL           Base URL of the Gitea instance.
-    REPO_TOKEN      API token with repo access.
+    REPO_TOKEN          API token with repo access.
-    RUN_ID          Workflow run ID (GITHUB_RUN_ID).
+    RUN_ID              Workflow run ID (GITHUB_RUN_ID).
-    JOB_NAME        Base job name (GITHUB_JOB), e.g. "molecule-tests".
+    JOB_NAME            Base job name (GITHUB_JOB), e.g. "molecule-tests".
-    MATRIX_INDEX    Current matrix index (runner-index).
+    MATRIX_INDEX        Current matrix index (runner-index).
-    OWNER, REPO     Repository owner/name.
+    GITEA_REPOSITORY    Repository in "owner/repo" format.
 """
 
 from __future__ import annotations
 
+import contextlib
 import os
 import signal
 import subprocess  # nosec B404
@@ -51,7 +60,6 @@ def any_other_runner_failed(
         name = job.get("name", "")
         if not name.startswith(current_job_name):
             continue
-        # Skip the current runner instance
         if name == f"{current_job_name} ({current_index})" or name == current_job_name:
             continue
         if job.get("conclusion") == "failure":
@@ -87,10 +95,32 @@ def poll_for_other_failures(
         stop_event.wait(POLL_INTERVAL)
 
 
+def build_molecule_cmd(scenario: str) -> list[str]:
+    """Build the molecule command for a scenario."""
+    cmd = ["molecule", "test"]
+    if scenario != "default":
+        cmd.extend(["-s", scenario])
+    return cmd
+
+
+def build_env_for_pair(pair: str, base_env: dict[str, str]) -> dict[str, str]:
+    """Build environment for a single molecule pair."""
+    scenario, platform_name, platform_image, platform_command = pair.split("|")
+    env = base_env.copy()
+    env["MOLECULE_PLATFORM_NAME"] = platform_name
+    env["MOLECULE_PLATFORM_IMAGE"] = platform_image
+    if platform_command:
+        env["MOLECULE_PLATFORM_COMMAND"] = platform_command
+    elif "MOLECULE_PLATFORM_COMMAND" in env:
+        del env["MOLECULE_PLATFORM_COMMAND"]
+    env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true"
+    return env
+
+
 @click.command()
 @click.argument("pairs", nargs=-1, required=True)
 def cli(pairs: tuple[str, ...]) -> None:
-    """Run molecule pairs and stop if another CI runner fails."""
+    """Run molecule pairs sequentially, stop if another CI runner fails."""
     gitea_url = os.environ.get("GITEA_URL", "")
     token = os.environ.get("REPO_TOKEN", "")
     run_id = int(os.environ.get("RUN_ID", "0"))
@@ -109,21 +139,11 @@ def cli(pairs: tuple[str, ...]) -> None:
         )
 
     repo_root = Path(__file__).resolve().parent.parent
-    script_path = repo_root / "scripts" / "run_molecule_parallel.py"
+    role_dir = repo_root / "ansible" / "roles" / "gitea-runner"
 
-    env = os.environ.copy()
+    base_env = os.environ.copy()
-    env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock")
+    base_env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock")
-    env.setdefault("ANSIBLE_INJECT_INVOCATION", "1")
+    base_env.setdefault("ANSIBLE_INJECT_INVOCATION", "1")
-
-    click.echo(_("Starting molecule runner with {count} pairs.", count=len(pairs)))
-
-    process = subprocess.Popen(  # nosec B603
-        [sys.executable, str(script_path), *pairs],
-        env=env,
-        stdout=None,
-        stderr=None,
-        preexec_fn=os.setsid,
-    )
 
     stop_event = threading.Event()
     failed_event = threading.Event()
@@ -147,26 +167,59 @@ def cli(pairs: tuple[str, ...]) -> None:
         poller.start()
 
     try:
-        while process.poll() is None:
+        for pair in pairs:
             if failed_event.is_set():
-                try:
-                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                except ProcessLookupError:
-                    pass
-                try:
-                    process.wait(timeout=10)
-                except subprocess.TimeoutExpired:
-                    try:
-                        os.killpg(os.getpgid(process.pid), signal.SIGKILL)
-                    except ProcessLookupError:
-                        pass
-                    process.wait()
                 sys.exit(1)
-            time.sleep(1)
+
+            scenario = pair.split("|")[0]
+            platform_name = pair.split("|")[1]
+            click.echo(
+                _("Running: {scenario} on {platform}", scenario=scenario, platform=platform_name)
+            )
+
+            cmd = build_molecule_cmd(scenario)
+            env = build_env_for_pair(pair, base_env)
+
+            process = subprocess.Popen(  # nosec B603
+                cmd,
+                cwd=str(role_dir),
+                env=env,
+                preexec_fn=os.setsid,
+            )
+
+            try:
+                while process.poll() is None:
+                    if failed_event.is_set():
+                        with contextlib.suppress(ProcessLookupError):
+                            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                        try:
+                            process.wait(timeout=10)
+                        except subprocess.TimeoutExpired:
+                            with contextlib.suppress(ProcessLookupError):
+                                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+                            process.wait()
+                        sys.exit(1)
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                with contextlib.suppress(ProcessLookupError):
+                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                process.wait()
+                sys.exit(1)
+
+            rc = process.returncode
+            if rc != 0:
+                click.echo(
+                    _("FAILED: {pair} exited with code {code}", pair=pair, code=rc)
+                )
+                sys.exit(rc)
+
+            click.echo(_("PASSED: {pair}", pair=pair))
+
+        click.echo(_("All molecule tests passed."))
     finally:
         stop_event.set()
 
-    sys.exit(process.returncode)
+    sys.exit(0)
 
 
 if __name__ == "__main__":  # pragma: no cover

tests/unit/test_molecule_ci_guard.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import os
 import subprocess  # nosec B404
+import time
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -11,6 +12,8 @@ import requests
 
 from scripts.molecule_ci_guard import (
     any_other_runner_failed,
+    build_env_for_pair,
+    build_molecule_cmd,
     cli,
     get_running_jobs,
     poll_for_other_failures,
@@ -65,6 +68,32 @@ class TestAnyOtherRunnerFailed:
         assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
 
 
+class TestBuildMoleculeCmd:
+    def test_default_scenario(self) -> None:
+        assert build_molecule_cmd("default") == ["molecule", "test"]
+
+    def test_named_scenario(self) -> None:
+        assert build_molecule_cmd("lifecycle") == ["molecule", "test", "-s", "lifecycle"]
+
+
+class TestBuildEnvForPair:
+    def test_with_command(self) -> None:
+        env = build_env_for_pair("default|ubuntu-2204|img:latest|/lib/systemd/systemd", {})
+        assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
+        assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest"
+        assert env["MOLECULE_PLATFORM_COMMAND"] == "/lib/systemd/systemd"
+        assert env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] == "true"
+
+    def test_without_command(self) -> None:
+        env = build_env_for_pair("default|ubuntu-2204|img:latest|", {})
+        assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
+        assert "MOLECULE_PLATFORM_COMMAND" not in env
+
+    def test_without_command_removes_existing(self) -> None:
+        env = build_env_for_pair("default|ubuntu-2204|img:latest|", {"MOLECULE_PLATFORM_COMMAND": "old"})
+        assert "MOLECULE_PLATFORM_COMMAND" not in env
+
+
 class TestPollForOtherFailures:
     def test_sets_failed_event_when_other_runner_fails(self) -> None:
         stop_event = MagicMock()
@@ -122,13 +151,12 @@ class TestPollForOtherFailures:
 
 
 class TestCli:
-    def test_runs_without_api_env(self) -> None:
+    def test_all_pass(self) -> None:
         from click.testing import CliRunner
 
-        env = {"PATH": os.environ.get("PATH", "")}
         with (
-            patch.dict(os.environ, env, clear=True),
             patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("time.sleep"),
         ):
             proc = MagicMock()
             proc.poll.return_value = 0
@@ -138,8 +166,26 @@ class TestCli:
             runner = CliRunner()
             result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
             assert result.exit_code == 0
+            assert "All molecule tests passed" in result.output
 
-    def test_exits_when_other_runner_fails(self) -> None:
+    def test_failure_exits_nonzero(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("time.sleep"),
+        ):
+            proc = MagicMock()
+            proc.poll.return_value = 1
+            proc.returncode = 1
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+            assert "FAILED" in result.output
+
+    def test_exits_before_starting_when_already_failed(self) -> None:
         from click.testing import CliRunner
 
         with (
@@ -156,19 +202,85 @@ class TestCli:
                 },
                 clear=True,
             ),
+            patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
             patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
             patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
-            patch("os.killpg") as mock_killpg,
+            patch("time.sleep"),
-            patch("os.getpgid") as mock_getpgid,
         ):
-            mock_getpgid.return_value = 123
             mock_get_jobs.return_value = [
                 {"name": "molecule-tests (0)", "conclusion": "running"},
                 {"name": "molecule-tests (1)", "conclusion": "failure"},
             ]
+            proc = MagicMock()
+            proc.poll.return_value = 0
+            proc.returncode = 0
+            mock_popen.return_value = proc
 
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+            assert "Another molecule runner failed" in result.output
+
+    def test_keyboard_interrupt_kills_process(self) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("time.sleep", side_effect=KeyboardInterrupt),
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
             proc = MagicMock()
             proc.poll.return_value = None
+            proc.wait.return_value = 0
+            mock_popen.return_value = proc
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 1
+            mock_killpg.assert_called()
+
+    def test_exits_when_other_runner_fails(self) -> None:
+        from click.testing import CliRunner
+
+        real_sleep = time.sleep
+        call_count = [0]
+
+        def get_jobs_side_effect(*args, **kwargs):
+            call_count[0] += 1
+            if call_count[0] < 2:
+                return [{"name": "molecule-tests (1)", "conclusion": "running"}]
+            return [
+                {"name": "molecule-tests (0)", "conclusion": "running"},
+                {"name": "molecule-tests (1)", "conclusion": "failure"},
+            ]
+
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+            patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
+        ):
+            mock_getpgid.return_value = 123
+            proc = MagicMock()
+            proc.poll.return_value = None
+            proc.wait.return_value = 0
             mock_popen.return_value = proc
 
             runner = CliRunner()
@@ -179,6 +291,8 @@ class TestCli:
     def test_default_owner_repo_fallback(self) -> None:
         from click.testing import CliRunner
 
+        real_sleep = time.sleep
+
         with (
             patch.dict(
                 os.environ,
@@ -193,8 +307,10 @@ class TestCli:
                 },
                 clear=True,
             ),
+            patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
             patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
             patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("time.sleep", side_effect=lambda x: real_sleep(0.05)),
         ):
             mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
             proc = MagicMock()
@@ -206,69 +322,47 @@ class TestCli:
             result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
             assert result.exit_code == 0
 
-    def test_sleeps_while_waiting_for_process(self) -> None:
-        from click.testing import CliRunner
-
-        with (
-            patch.dict(
-                os.environ,
-                {
-                    "GITEA_URL": "https://gitea.example",
-                    "REPO_TOKEN": "token",
-                    "RUN_ID": "123",
-                    "JOB_NAME": "molecule-tests",
-                    "MATRIX_INDEX": "0",
-                    "GITEA_REPOSITORY": "oblachno-oss/grm",
-                    "PATH": os.environ.get("PATH", ""),
-                },
-                clear=True,
-            ),
-            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
-            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
-            patch("time.sleep") as mock_sleep,
-        ):
-            mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
-            proc = MagicMock()
-            proc.poll.side_effect = [None, 0]
-            proc.returncode = 0
-            mock_popen.return_value = proc
-
-            runner = CliRunner()
-            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
-            assert result.exit_code == 0
-            mock_sleep.assert_called()
-
     def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None:
         from click.testing import CliRunner
 
-        with (
+        real_sleep = time.sleep
-            patch.dict(
+        call_count = [0]
-                os.environ,
+
-                {
+        def get_jobs_side_effect(*args, **kwargs):
-                    "GITEA_URL": "https://gitea.example",
+            call_count[0] += 1
-                    "REPO_TOKEN": "token",
+            if call_count[0] < 2:
-                    "RUN_ID": "123",
+                return [{"name": "molecule-tests (1)", "conclusion": "running"}]
-                    "JOB_NAME": "molecule-tests",
+            return [
-                    "MATRIX_INDEX": "0",
-                    "GITEA_REPOSITORY": "oblachno-oss/grm",
-                    "PATH": os.environ.get("PATH", ""),
-                },
-                clear=True,
-            ),
-            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
-            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
-            patch("os.killpg") as mock_killpg,
-            patch("os.getpgid") as mock_getpgid,
-        ):
-            mock_getpgid.return_value = 123
-            mock_killpg.side_effect = ProcessLookupError("no such process")
-            mock_get_jobs.return_value = [
                 {"name": "molecule-tests (0)", "conclusion": "running"},
                 {"name": "molecule-tests (1)", "conclusion": "failure"},
             ]
 
+        with (
+            patch.dict(
+                os.environ,
+                {
+                    "GITEA_URL": "https://gitea.example",
+                    "REPO_TOKEN": "token",
+                    "RUN_ID": "123",
+                    "JOB_NAME": "molecule-tests",
+                    "MATRIX_INDEX": "0",
+                    "GITEA_REPOSITORY": "oblachno-oss/grm",
+                    "PATH": os.environ.get("PATH", ""),
+                },
+                clear=True,
+            ),
+            patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
+            patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
+            patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+            patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
+        ):
+            mock_getpgid.return_value = 123
+            mock_killpg.side_effect = ProcessLookupError("no such process")
             proc = MagicMock()
             proc.poll.return_value = None
+            proc.wait.return_value = 0
             mock_popen.return_value = proc
 
             runner = CliRunner()
@@ -278,6 +372,18 @@ class TestCli:
     def test_exits_when_other_runner_fails_with_timeout(self) -> None:
         from click.testing import CliRunner
 
+        real_sleep = time.sleep
+        call_count = [0]
+
+        def get_jobs_side_effect(*args, **kwargs):
+            call_count[0] += 1
+            if call_count[0] < 2:
+                return [{"name": "molecule-tests (1)", "conclusion": "running"}]
+            return [
+                {"name": "molecule-tests (0)", "conclusion": "running"},
+                {"name": "molecule-tests (1)", "conclusion": "failure"},
+            ]
+
         with (
             patch.dict(
                 os.environ,
@@ -292,18 +398,15 @@ class TestCli:
                 },
                 clear=True,
             ),
+            patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
             patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
-            patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
+            patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
             patch("os.killpg") as mock_killpg,
             patch("os.getpgid") as mock_getpgid,
+            patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
         ):
             mock_getpgid.return_value = 123
             mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
-            mock_get_jobs.return_value = [
-                {"name": "molecule-tests (0)", "conclusion": "running"},
-                {"name": "molecule-tests (1)", "conclusion": "failure"},
-            ]
-
             proc = MagicMock()
             proc.poll.return_value = None
             proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]