diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9782601..265cae6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -29,26 +29,24 @@ jobs: molecule-tests: needs: quality runs-on: docker + strategy: + fail-fast: true + matrix: + runner-index: [0, 1, 2] steps: - uses: actions/checkout@v4 - name: Set up environment run: make setup - - name: Run all molecule tests + - name: Discover assigned test pairs + run: | + . .venv/bin/activate + PAIRS=$(python3 scripts/distribute_molecule.py --runner-index ${{ matrix.runner-index }} --max-runners 3) + echo "Assigned pairs: $PAIRS" + echo "TEST_PAIRS=$PAIRS" >> $GITHUB_ENV + - name: Run molecule tests run: | set -euo pipefail . .venv/bin/activate export DOCKER_HOST="unix:///run/user/$(id -u)/docker.sock" export ANSIBLE_INJECT_INVOCATION=1 - cd ansible/roles/gitea-runner - for s in default multi-instance lifecycle template-content deregister update; do - export MOLECULE_PLATFORM_NAME="ubuntu-2204" - export MOLECULE_PLATFORM_IMAGE="geerlingguy/docker-ubuntu2204-ansible:latest" - export MOLECULE_PLATFORM_COMMAND="/lib/systemd/systemd" - echo "::group::Molecule: $s on ubuntu-2204" - if [ "$s" = "default" ]; then - ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true molecule test - else - ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true molecule test -s "$s" - fi - echo "::endgroup::" - done + python3 scripts/run_molecule_parallel.py $TEST_PAIRS diff --git a/scripts/run_molecule_parallel.py b/scripts/run_molecule_parallel.py new file mode 100644 index 0000000..645530b --- /dev/null +++ b/scripts/run_molecule_parallel.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Run molecule (scenario, platform) pairs in parallel and kill on first failure. + +Each pair is encoded as ``scenario|platform_name|platform_image|platform_command``. +Subprocesses are started concurrently. If any subprocess exits with a non-zero +status, the remaining subprocesses are terminated and this script exits with 1. + +Usage: + python3 scripts/run_molecule_parallel.py \ + default|ubuntu-2204|ubuntu:22.04| \ + lifecycle|debian-12|debian:12| \ + ... +""" + +from __future__ import annotations + +import contextlib +import os +import signal +import subprocess # nosec B404 +import sys +from pathlib import Path + +import click + +from gitea_runner_manager.i18n import _ + + +def run_pair(pair: str, role_dir: Path) -> subprocess.Popen[bytes]: + """Start a subprocess for a single molecule (scenario, platform) pair.""" + scenario, platform_name, platform_image, platform_command = pair.split("|") + env = os.environ.copy() + env["MOLECULE_PLATFORM_NAME"] = platform_name + env["MOLECULE_PLATFORM_IMAGE"] = platform_image + if platform_command: + env["MOLECULE_PLATFORM_COMMAND"] = platform_command + elif "MOLECULE_PLATFORM_COMMAND" in env: + del env["MOLECULE_PLATFORM_COMMAND"] + + env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true" + env["ANSIBLE_INJECT_INVOCATION"] = "1" + + cmd = ["molecule", "test"] + if scenario != "default": + cmd.extend(["-s", scenario]) + + click.echo(_("Starting: {scenario} on {platform}", scenario=scenario, platform=platform_name)) + return subprocess.Popen( # nosec B603 + cmd, + cwd=str(role_dir), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + preexec_fn=os.setsid, + ) + + +@click.command() +@click.argument("pairs", nargs=-1, required=True) +def cli(pairs: tuple[str, ...]) -> None: + """Run molecule pairs in parallel, stop on first failure.""" + repo_root = Path(__file__).resolve().parent.parent + role_dir = repo_root / "ansible" / "roles" / "gitea-runner" + + processes: list[subprocess.Popen[bytes]] = [] + pair_names: list[str] = [] + for pair in pairs: + proc = run_pair(pair, role_dir) + processes.append(proc) + pair_names.append(pair) + + first_failure: str | None = None + returncode = 0 + while processes: + finished_indices: list[int] = [] + for i, proc in enumerate(processes): + ret = proc.poll() + if ret is not None: + finished_indices.append(i) + if ret != 0: + first_failure = pair_names[i] + returncode = ret + + if first_failure is not None: + click.echo( + _( + "FAILURE: {pair} exited with code {code}. Stopping remaining tests.", + pair=first_failure, + code=returncode, + ) + ) + for proc in processes: + if proc.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + # Wait briefly, then SIGKILL survivors + for proc in processes: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + proc.wait() + sys.exit(returncode) + + if not finished_indices: + # No process finished yet, wait a bit + for proc in processes: + proc.wait(timeout=0.5) + continue + + # Remove finished processes from the list + for i in sorted(finished_indices, reverse=True): + del processes[i] + del pair_names[i] + + click.echo(_("All molecule tests passed.")) + sys.exit(0) + + +if __name__ == "__main__": # pragma: no cover + cli() diff --git a/tests/unit/test_run_molecule_parallel.py b/tests/unit/test_run_molecule_parallel.py new file mode 100644 index 0000000..c9a2fc1 --- /dev/null +++ b/tests/unit/test_run_molecule_parallel.py @@ -0,0 +1,159 @@ +"""Unit tests for scripts/run_molecule_parallel.py.""" + +from __future__ import annotations + +import os +import subprocess # nosec B404 +from pathlib import Path +from unittest.mock import MagicMock, patch + +from scripts.run_molecule_parallel import cli, run_pair + + +class TestRunPair: + def test_default_scenario(self, tmp_path: Path) -> None: + with ( + patch("scripts.run_molecule_parallel.subprocess.Popen") as mock_popen, + patch.dict(os.environ, {"MOLECULE_PLATFORM_COMMAND": "old-command"}, clear=False), + ): + mock_popen.return_value = MagicMock() + result = run_pair("default|ubuntu-2204|img:latest|", tmp_path) + assert result is mock_popen.return_value + call_args = mock_popen.call_args + assert call_args.kwargs["cwd"] == str(tmp_path) + env = call_args.kwargs["env"] + assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204" + assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest" + assert "MOLECULE_PLATFORM_COMMAND" not in env + assert call_args.args[0] == ["molecule", "test"] + + def test_named_scenario_with_command(self, tmp_path: Path) -> None: + with patch("scripts.run_molecule_parallel.subprocess.Popen") as mock_popen: + mock_popen.return_value = MagicMock() + result = run_pair("lifecycle|archlinux|img:arch|/usr/lib/systemd/systemd", tmp_path) + assert result is mock_popen.return_value + call_args = mock_popen.call_args + env = call_args.kwargs["env"] + assert env["MOLECULE_PLATFORM_NAME"] == "archlinux" + assert env["MOLECULE_PLATFORM_IMAGE"] == "img:arch" + assert env["MOLECULE_PLATFORM_COMMAND"] == "/usr/lib/systemd/systemd" + assert call_args.args[0] == ["molecule", "test", "-s", "lifecycle"] + + +class TestCli: + def test_all_pass(self, tmp_path: Path) -> None: + from click.testing import CliRunner + + with patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair: + mock_run_pair.return_value = MagicMock() + mock_run_pair.return_value.poll.side_effect = [None, 0] + mock_run_pair.return_value.wait.return_value = None + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 0 + assert "All molecule tests passed" in result.output + + def test_failure_kills_remaining(self, tmp_path: Path) -> None: + from click.testing import CliRunner + + with ( + patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + + good_proc = MagicMock() + good_proc.poll.return_value = None + good_proc.wait.return_value = None + bad_proc = MagicMock() + bad_proc.poll.side_effect = [None, 1, 1, 1] + bad_proc.wait.return_value = None + mock_run_pair.side_effect = [bad_proc, good_proc] + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "default|ubuntu-2204|img:latest|", + "lifecycle|debian-12|img:deb|", + ], + ) + assert result.exit_code == 1 + assert "FAILURE" in result.output + mock_killpg.assert_called() + + def test_failure_sends_sigkill_when_sigterm_times_out(self, tmp_path: Path) -> None: + from click.testing import CliRunner + + with ( + patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + # First SIGTERM succeeds, second SIGKILL raises ProcessLookupError + mock_killpg.side_effect = [None, ProcessLookupError("no such process")] + + good_proc = MagicMock() + good_proc.poll.return_value = None + good_proc.wait.side_effect = [None, subprocess.TimeoutExpired("cmd", 5)] + bad_proc = MagicMock() + bad_proc.poll.side_effect = [None, 1, 1, 1] + bad_proc.wait.return_value = None + mock_run_pair.side_effect = [bad_proc, good_proc] + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "default|ubuntu-2204|img:latest|", + "lifecycle|debian-12|img:deb|", + ], + ) + assert result.exit_code == 1 + assert "FAILURE" in result.output + # First SIGTERM, then SIGKILL + assert mock_killpg.call_count >= 2 + + def test_failure_handles_process_lookup_error(self, tmp_path: Path) -> None: + from click.testing import CliRunner + + with ( + patch("scripts.run_molecule_parallel.run_pair") as mock_run_pair, + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 + mock_killpg.side_effect = ProcessLookupError("no such process") + + good_proc = MagicMock() + good_proc.poll.return_value = None + good_proc.wait.return_value = None + bad_proc = MagicMock() + bad_proc.poll.side_effect = [None, 1, 1, 1] + bad_proc.wait.return_value = None + mock_run_pair.side_effect = [bad_proc, good_proc] + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "default|ubuntu-2204|img:latest|", + "lifecycle|debian-12|img:deb|", + ], + ) + assert result.exit_code == 1 + assert "FAILURE" in result.output + + +def test_main_module_block() -> None: + import scripts.run_molecule_parallel as rm + + with open(rm.__file__) as f: + source = f.read() + source = source.replace('if __name__ == "__main__":\n cli()\n', "") + namespace = dict(rm.__dict__) + exec(compile(source, rm.__file__, "exec"), namespace) + assert callable(namespace["cli"])