feat: parallel molecule runner with kill-on-first-failure
Added scripts/run_molecule_parallel.py to run a runner's assigned (scenario, platform) pairs in parallel. If any subprocess fails, the remaining ones are terminated with SIGTERM/SIGKILL and the runner exits immediately. This gives fast feedback without continuing to run tests that are guaranteed to fail for the same reason. CI workflow now calls this script per matrix runner. Added fail-fast and max-parallel for best-effort cancellation across runners. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
a57a5c328c
commit
69b96e4785
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run molecule (scenario, platform) pairs in parallel and kill on first failure.
|
||||
|
||||
Each pair is encoded as ``scenario|platform_name|platform_image|platform_command``.
|
||||
Subprocesses are started concurrently. If any subprocess exits with a non-zero
|
||||
status, the remaining subprocesses are terminated and this script exits with 1.
|
||||
|
||||
Usage:
|
||||
python3 scripts/run_molecule_parallel.py \
|
||||
default|ubuntu-2204|ubuntu:22.04| \
|
||||
lifecycle|debian-12|debian:12| \
|
||||
...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from gitea_runner_manager.i18n import _
|
||||
|
||||
|
||||
def run_pair(pair: str, role_dir: Path) -> subprocess.Popen[bytes]:
|
||||
"""Start a subprocess for a single molecule (scenario, platform) pair."""
|
||||
scenario, platform_name, platform_image, platform_command = pair.split("|")
|
||||
env = os.environ.copy()
|
||||
env["MOLECULE_PLATFORM_NAME"] = platform_name
|
||||
env["MOLECULE_PLATFORM_IMAGE"] = platform_image
|
||||
if platform_command:
|
||||
env["MOLECULE_PLATFORM_COMMAND"] = platform_command
|
||||
elif "MOLECULE_PLATFORM_COMMAND" in env:
|
||||
del env["MOLECULE_PLATFORM_COMMAND"]
|
||||
|
||||
env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true"
|
||||
env["ANSIBLE_INJECT_INVOCATION"] = "1"
|
||||
|
||||
cmd = ["molecule", "test"]
|
||||
if scenario != "default":
|
||||
cmd.extend(["-s", scenario])
|
||||
|
||||
click.echo(_("Starting: {scenario} on {platform}", scenario=scenario, platform=platform_name))
|
||||
return subprocess.Popen( # nosec B603
|
||||
cmd,
|
||||
cwd=str(role_dir),
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid,
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("pairs", nargs=-1, required=True)
|
||||
def cli(pairs: tuple[str, ...]) -> None:
|
||||
"""Run molecule pairs in parallel, stop on first failure."""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
role_dir = repo_root / "ansible" / "roles" / "gitea-runner"
|
||||
|
||||
processes: list[subprocess.Popen[bytes]] = []
|
||||
pair_names: list[str] = []
|
||||
for pair in pairs:
|
||||
proc = run_pair(pair, role_dir)
|
||||
processes.append(proc)
|
||||
pair_names.append(pair)
|
||||
|
||||
first_failure: str | None = None
|
||||
returncode = 0
|
||||
while processes:
|
||||
finished_indices: list[int] = []
|
||||
for i, proc in enumerate(processes):
|
||||
ret = proc.poll()
|
||||
if ret is not None:
|
||||
finished_indices.append(i)
|
||||
if ret != 0:
|
||||
first_failure = pair_names[i]
|
||||
returncode = ret
|
||||
|
||||
if first_failure is not None:
|
||||
click.echo(
|
||||
_(
|
||||
"FAILURE: {pair} exited with code {code}. Stopping remaining tests.",
|
||||
pair=first_failure,
|
||||
code=returncode,
|
||||
)
|
||||
)
|
||||
for proc in processes:
|
||||
if proc.poll() is None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
# Wait briefly, then SIGKILL survivors
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
||||
proc.wait()
|
||||
sys.exit(returncode)
|
||||
|
||||
if not finished_indices:
|
||||
# No process finished yet, wait a bit
|
||||
for proc in processes:
|
||||
proc.wait(timeout=0.5)
|
||||
continue
|
||||
|
||||
# Remove finished processes from the list
|
||||
for i in sorted(finished_indices, reverse=True):
|
||||
del processes[i]
|
||||
del pair_names[i]
|
||||
|
||||
click.echo(_("All molecule tests passed."))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
cli()
|
||||
Reference in New Issue
Block a user