feat: parallel molecule runner with kill-on-first-failure

Added scripts/run_molecule_parallel.py to run a runner's assigned
(scenario, platform) pairs in parallel. If any subprocess fails, the
remaining ones are terminated with SIGTERM/SIGKILL and the runner
exits immediately. This gives fast feedback without continuing to run
tests that are guaranteed to fail for the same reason.

CI workflow now calls this script per matrix runner. Added fail-fast and
max-parallel for best-effort cancellation across runners.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.gitea/workflows/ci.yml

@@ -29,26 +29,24 @@ jobs:
   molecule-tests:
     needs: quality
     runs-on: docker
+    strategy:
+      fail-fast: true
+      matrix:
+        runner-index: [0, 1, 2]
     steps:
       - uses: actions/checkout@v4
       - name: Set up environment
         run: make setup
-      - name: Run all molecule tests
+      - name: Discover assigned test pairs
+        run: |
+          . .venv/bin/activate
+          PAIRS=$(python3 scripts/distribute_molecule.py --runner-index ${{ matrix.runner-index }} --max-runners 3)
+          echo "Assigned pairs: $PAIRS"
+          echo "TEST_PAIRS=$PAIRS" >> $GITHUB_ENV
+      - name: Run molecule tests
         run: |
           set -euo pipefail
           . .venv/bin/activate
           export DOCKER_HOST="unix:///run/user/$(id -u)/docker.sock"
           export ANSIBLE_INJECT_INVOCATION=1
-          cd ansible/roles/gitea-runner
-          for s in default multi-instance lifecycle template-content deregister update; do
-            export MOLECULE_PLATFORM_NAME="ubuntu-2204"
-            export MOLECULE_PLATFORM_IMAGE="geerlingguy/docker-ubuntu2204-ansible:latest"
-            export MOLECULE_PLATFORM_COMMAND="/lib/systemd/systemd"
-            echo "::group::Molecule: $s on ubuntu-2204"
-            if [ "$s" = "default" ]; then
-              ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true molecule test
-            else
-              ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true molecule test -s "$s"
-            fi
-            echo "::endgroup::"
-          done
+          python3 scripts/run_molecule_parallel.py $TEST_PAIRS

scripts/run_molecule_parallel.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Run molecule (scenario, platform) pairs in parallel and kill on first failure.
+
+Each pair is encoded as ``scenario|platform_name|platform_image|platform_command``.
+Subprocesses are started concurrently. If any subprocess exits with a non-zero
+status, the remaining subprocesses are terminated and this script exits with 1.
+
+Usage:
+    python3 scripts/run_molecule_parallel.py \
+        default|ubuntu-2204|ubuntu:22.04| \
+        lifecycle|debian-12|debian:12| \
+        ...
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import signal
+import subprocess  # nosec B404
+import sys
+from pathlib import Path
+
+import click
+
+from gitea_runner_manager.i18n import _
+
+
+def run_pair(pair: str, role_dir: Path) -> subprocess.Popen[bytes]:
+    """Start a subprocess for a single molecule (scenario, platform) pair."""
+    scenario, platform_name, platform_image, platform_command = pair.split("|")
+    env = os.environ.copy()
+    env["MOLECULE_PLATFORM_NAME"] = platform_name
+    env["MOLECULE_PLATFORM_IMAGE"] = platform_image
+    if platform_command:
+        env["MOLECULE_PLATFORM_COMMAND"] = platform_command
+    elif "MOLECULE_PLATFORM_COMMAND" in env:
+        del env["MOLECULE_PLATFORM_COMMAND"]
+
+    env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true"
+    env["ANSIBLE_INJECT_INVOCATION"] = "1"
+
+    cmd = ["molecule", "test"]
+    if scenario != "default":
+        cmd.extend(["-s", scenario])
+
+    click.echo(_("Starting: {scenario} on {platform}", scenario=scenario, platform=platform_name))
+    return subprocess.Popen(  # nosec B603
+        cmd,
+        cwd=str(role_dir),
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        preexec_fn=os.setsid,
+    )
+
+
+@click.command()
+@click.argument("pairs", nargs=-1, required=True)
+def cli(pairs: tuple[str, ...]) -> None:
+    """Run molecule pairs in parallel, stop on first failure."""
+    repo_root = Path(__file__).resolve().parent.parent
+    role_dir = repo_root / "ansible" / "roles" / "gitea-runner"
+
+    processes: list[subprocess.Popen[bytes]] = []
+    pair_names: list[str] = []
+    for pair in pairs:
+        proc = run_pair(pair, role_dir)
+        processes.append(proc)
+        pair_names.append(pair)
+
+    first_failure: str | None = None
+    returncode = 0
+    while processes:
+        finished_indices: list[int] = []
+        for i, proc in enumerate(processes):
+            ret = proc.poll()
+            if ret is not None:
+                finished_indices.append(i)
+                if ret != 0:
+                    first_failure = pair_names[i]
+                    returncode = ret
+
+        if first_failure is not None:
+            click.echo(
+                _(
+                    "FAILURE: {pair} exited with code {code}. Stopping remaining tests.",
+                    pair=first_failure,
+                    code=returncode,
+                )
+            )
+            for proc in processes:
+                if proc.poll() is None:
+                    with contextlib.suppress(ProcessLookupError):
+                        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            # Wait briefly, then SIGKILL survivors
+            for proc in processes:
+                try:
+                    proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    with contextlib.suppress(ProcessLookupError):
+                        os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+                    proc.wait()
+            sys.exit(returncode)
+
+        if not finished_indices:
+            # No process finished yet, wait a bit
+            for proc in processes:
+                proc.wait(timeout=0.5)
+            continue
+
+        # Remove finished processes from the list
+        for i in sorted(finished_indices, reverse=True):
+            del processes[i]
+            del pair_names[i]
+
+    click.echo(_("All molecule tests passed."))
+    sys.exit(0)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

tests/unit/test_run_molecule_parallel.py

@@ -0,0 +1,159 @@
+"""Unit tests for scripts/run_molecule_parallel.py."""
+
+from __future__ import annotations
+
+import os
+import subprocess  # nosec B404
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from scripts.run_molecule_parallel import cli, run_pair
+
+
+class TestRunPair:
+    def test_default_scenario(self, tmp_path: Path) -> None:
+        with (
+            patch("scripts.run_molecule_parallel.subprocess.Popen") as mock_popen,
+            patch.dict(os.environ, {"MOLECULE_PLATFORM_COMMAND": "old-command"}, clear=False),
+        ):
+            mock_popen.return_value = MagicMock()
+            result = run_pair("default|ubuntu-2204|img:latest|", tmp_path)
+            assert result is mock_popen.return_value
+            call_args = mock_popen.call_args
+            assert call_args.kwargs["cwd"] == str(tmp_path)
+            env = call_args.kwargs["env"]
+            assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
+            assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest"
+            assert "MOLECULE_PLATFORM_COMMAND" not in env
+            assert call_args.args[0] == ["molecule", "test"]
+
+    def test_named_scenario_with_command(self, tmp_path: Path) -> None:
+        with patch("scripts.run_molecule_parallel.subprocess.Popen") as mock_popen:
+            mock_popen.return_value = MagicMock()
+            result = run_pair("lifecycle|archlinux|img:arch|/usr/lib/systemd/systemd", tmp_path)
+            assert result is mock_popen.return_value
+            call_args = mock_popen.call_args
+            env = call_args.kwargs["env"]
+            assert env["MOLECULE_PLATFORM_NAME"] == "archlinux"
+            assert env["MOLECULE_PLATFORM_IMAGE"] == "img:arch"
+            assert env["MOLECULE_PLATFORM_COMMAND"] == "/usr/lib/systemd/systemd"
+            assert call_args.args[0] == ["molecule", "test", "-s", "lifecycle"]
+
+
+class TestCli:
+    def test_all_pass(self, tmp_path: Path) -> None:
+        from click.testing import CliRunner
+
+        with patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair:
+            mock_run_pair.return_value = MagicMock()
+            mock_run_pair.return_value.poll.side_effect = [None, 0]
+            mock_run_pair.return_value.wait.return_value = None
+
+            runner = CliRunner()
+            result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
+            assert result.exit_code == 0
+            assert "All molecule tests passed" in result.output
+
+    def test_failure_kills_remaining(self, tmp_path: Path) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+
+            good_proc = MagicMock()
+            good_proc.poll.return_value = None
+            good_proc.wait.return_value = None
+            bad_proc = MagicMock()
+            bad_proc.poll.side_effect = [None, 1, 1, 1]
+            bad_proc.wait.return_value = None
+            mock_run_pair.side_effect = [bad_proc, good_proc]
+
+            runner = CliRunner()
+            result = runner.invoke(
+                cli,
+                [
+                    "default|ubuntu-2204|img:latest|",
+                    "lifecycle|debian-12|img:deb|",
+                ],
+            )
+            assert result.exit_code == 1
+            assert "FAILURE" in result.output
+            mock_killpg.assert_called()
+
+    def test_failure_sends_sigkill_when_sigterm_times_out(self, tmp_path: Path) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+            # First SIGTERM succeeds, second SIGKILL raises ProcessLookupError
+            mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
+
+            good_proc = MagicMock()
+            good_proc.poll.return_value = None
+            good_proc.wait.side_effect = [None, subprocess.TimeoutExpired("cmd", 5)]
+            bad_proc = MagicMock()
+            bad_proc.poll.side_effect = [None, 1, 1, 1]
+            bad_proc.wait.return_value = None
+            mock_run_pair.side_effect = [bad_proc, good_proc]
+
+            runner = CliRunner()
+            result = runner.invoke(
+                cli,
+                [
+                    "default|ubuntu-2204|img:latest|",
+                    "lifecycle|debian-12|img:deb|",
+                ],
+            )
+            assert result.exit_code == 1
+            assert "FAILURE" in result.output
+            # First SIGTERM, then SIGKILL
+            assert mock_killpg.call_count >= 2
+
+    def test_failure_handles_process_lookup_error(self, tmp_path: Path) -> None:
+        from click.testing import CliRunner
+
+        with (
+            patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair,
+            patch("os.killpg") as mock_killpg,
+            patch("os.getpgid") as mock_getpgid,
+        ):
+            mock_getpgid.return_value = 123
+            mock_killpg.side_effect = ProcessLookupError("no such process")
+
+            good_proc = MagicMock()
+            good_proc.poll.return_value = None
+            good_proc.wait.return_value = None
+            bad_proc = MagicMock()
+            bad_proc.poll.side_effect = [None, 1, 1, 1]
+            bad_proc.wait.return_value = None
+            mock_run_pair.side_effect = [bad_proc, good_proc]
+
+            runner = CliRunner()
+            result = runner.invoke(
+                cli,
+                [
+                    "default|ubuntu-2204|img:latest|",
+                    "lifecycle|debian-12|img:deb|",
+                ],
+            )
+            assert result.exit_code == 1
+            assert "FAILURE" in result.output
+
+
+def test_main_module_block() -> None:
+    import scripts.run_molecule_parallel as rm
+
+    with open(rm.__file__) as f:
+        source = f.read()
+    source = source.replace('if __name__ == "__main__":\n    cli()\n', "")
+    namespace = dict(rm.__dict__)
+    exec(compile(source, rm.__file__, "exec"), namespace)
+    assert callable(namespace["cli"])