fix: run molecule pairs sequentially within each CI runner
CI / quality (pull_request) Successful in 1m5s
CI / molecule-tests (2) (pull_request) Failing after 3m8s
CI / molecule-tests (0) (pull_request) Failing after 3m15s
CI / molecule-tests (1) (pull_request) Failing after 3m18s

Parallel molecule execution within a single runner caused conflicts
(shared temp directories, Docker network collisions). Rewrote
molecule_ci_guard.py to run pairs sequentially while still polling
the Gitea API for cross-runner cancellation.

Each pair now gets its own subprocess with proper environment setup
(MOLECULE_PLATFORM_NAME/IMAGE/COMMAND), and output streams directly
to CI logs for debugging.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Emil Simeonov
2026-06-20 23:53:13 +02:00
co-authored by Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent 6ef38631fc
commit c271293b2d
2 changed files with 261 additions and 105 deletions
+170 -67
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import os
import subprocess # nosec B404
import time
from unittest.mock import MagicMock, patch
import pytest
@@ -11,6 +12,8 @@ import requests
from scripts.molecule_ci_guard import (
any_other_runner_failed,
build_env_for_pair,
build_molecule_cmd,
cli,
get_running_jobs,
poll_for_other_failures,
@@ -65,6 +68,32 @@ class TestAnyOtherRunnerFailed:
assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
class TestBuildMoleculeCmd:
def test_default_scenario(self) -> None:
assert build_molecule_cmd("default") == ["molecule", "test"]
def test_named_scenario(self) -> None:
assert build_molecule_cmd("lifecycle") == ["molecule", "test", "-s", "lifecycle"]
class TestBuildEnvForPair:
def test_with_command(self) -> None:
env = build_env_for_pair("default|ubuntu-2204|img:latest|/lib/systemd/systemd", {})
assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest"
assert env["MOLECULE_PLATFORM_COMMAND"] == "/lib/systemd/systemd"
assert env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] == "true"
def test_without_command(self) -> None:
env = build_env_for_pair("default|ubuntu-2204|img:latest|", {})
assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
assert "MOLECULE_PLATFORM_COMMAND" not in env
def test_without_command_removes_existing(self) -> None:
env = build_env_for_pair("default|ubuntu-2204|img:latest|", {"MOLECULE_PLATFORM_COMMAND": "old"})
assert "MOLECULE_PLATFORM_COMMAND" not in env
class TestPollForOtherFailures:
def test_sets_failed_event_when_other_runner_fails(self) -> None:
stop_event = MagicMock()
@@ -122,13 +151,12 @@ class TestPollForOtherFailures:
class TestCli:
def test_runs_without_api_env(self) -> None:
def test_all_pass(self) -> None:
from click.testing import CliRunner
env = {"PATH": os.environ.get("PATH", "")}
with (
patch.dict(os.environ, env, clear=True),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("time.sleep"),
):
proc = MagicMock()
proc.poll.return_value = 0
@@ -138,8 +166,26 @@ class TestCli:
runner = CliRunner()
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 0
assert "All molecule tests passed" in result.output
def test_exits_when_other_runner_fails(self) -> None:
def test_failure_exits_nonzero(self) -> None:
from click.testing import CliRunner
with (
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("time.sleep"),
):
proc = MagicMock()
proc.poll.return_value = 1
proc.returncode = 1
mock_popen.return_value = proc
runner = CliRunner()
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 1
assert "FAILED" in result.output
def test_exits_before_starting_when_already_failed(self) -> None:
from click.testing import CliRunner
with (
@@ -156,19 +202,85 @@ class TestCli:
},
clear=True,
),
patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
patch("time.sleep"),
):
mock_getpgid.return_value = 123
mock_get_jobs.return_value = [
{"name": "molecule-tests (0)", "conclusion": "running"},
{"name": "molecule-tests (1)", "conclusion": "failure"},
]
proc = MagicMock()
proc.poll.return_value = 0
proc.returncode = 0
mock_popen.return_value = proc
runner = CliRunner()
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 1
assert "Another molecule runner failed" in result.output
def test_keyboard_interrupt_kills_process(self) -> None:
from click.testing import CliRunner
with (
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("time.sleep", side_effect=KeyboardInterrupt),
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
):
mock_getpgid.return_value = 123
proc = MagicMock()
proc.poll.return_value = None
proc.wait.return_value = 0
mock_popen.return_value = proc
runner = CliRunner()
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 1
mock_killpg.assert_called()
def test_exits_when_other_runner_fails(self) -> None:
from click.testing import CliRunner
real_sleep = time.sleep
call_count = [0]
def get_jobs_side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] < 2:
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
return [
{"name": "molecule-tests (0)", "conclusion": "running"},
{"name": "molecule-tests (1)", "conclusion": "failure"},
]
with (
patch.dict(
os.environ,
{
"GITEA_URL": "https://gitea.example",
"REPO_TOKEN": "token",
"RUN_ID": "123",
"JOB_NAME": "molecule-tests",
"MATRIX_INDEX": "0",
"GITEA_REPOSITORY": "oblachno-oss/grm",
"PATH": os.environ.get("PATH", ""),
},
clear=True,
),
patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
):
mock_getpgid.return_value = 123
proc = MagicMock()
proc.poll.return_value = None
proc.wait.return_value = 0
mock_popen.return_value = proc
runner = CliRunner()
@@ -179,6 +291,8 @@ class TestCli:
def test_default_owner_repo_fallback(self) -> None:
from click.testing import CliRunner
real_sleep = time.sleep
with (
patch.dict(
os.environ,
@@ -193,8 +307,10 @@ class TestCli:
},
clear=True,
),
patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
patch("time.sleep", side_effect=lambda x: real_sleep(0.05)),
):
mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
proc = MagicMock()
@@ -206,69 +322,47 @@ class TestCli:
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 0
def test_sleeps_while_waiting_for_process(self) -> None:
from click.testing import CliRunner
with (
patch.dict(
os.environ,
{
"GITEA_URL": "https://gitea.example",
"REPO_TOKEN": "token",
"RUN_ID": "123",
"JOB_NAME": "molecule-tests",
"MATRIX_INDEX": "0",
"GITEA_REPOSITORY": "oblachno-oss/grm",
"PATH": os.environ.get("PATH", ""),
},
clear=True,
),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
patch("time.sleep") as mock_sleep,
):
mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
proc = MagicMock()
proc.poll.side_effect = [None, 0]
proc.returncode = 0
mock_popen.return_value = proc
runner = CliRunner()
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
assert result.exit_code == 0
mock_sleep.assert_called()
def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None:
from click.testing import CliRunner
with (
patch.dict(
os.environ,
{
"GITEA_URL": "https://gitea.example",
"REPO_TOKEN": "token",
"RUN_ID": "123",
"JOB_NAME": "molecule-tests",
"MATRIX_INDEX": "0",
"GITEA_REPOSITORY": "oblachno-oss/grm",
"PATH": os.environ.get("PATH", ""),
},
clear=True,
),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
):
mock_getpgid.return_value = 123
mock_killpg.side_effect = ProcessLookupError("no such process")
mock_get_jobs.return_value = [
real_sleep = time.sleep
call_count = [0]
def get_jobs_side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] < 2:
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
return [
{"name": "molecule-tests (0)", "conclusion": "running"},
{"name": "molecule-tests (1)", "conclusion": "failure"},
]
with (
patch.dict(
os.environ,
{
"GITEA_URL": "https://gitea.example",
"REPO_TOKEN": "token",
"RUN_ID": "123",
"JOB_NAME": "molecule-tests",
"MATRIX_INDEX": "0",
"GITEA_REPOSITORY": "oblachno-oss/grm",
"PATH": os.environ.get("PATH", ""),
},
clear=True,
),
patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
):
mock_getpgid.return_value = 123
mock_killpg.side_effect = ProcessLookupError("no such process")
proc = MagicMock()
proc.poll.return_value = None
proc.wait.return_value = 0
mock_popen.return_value = proc
runner = CliRunner()
@@ -278,6 +372,18 @@ class TestCli:
def test_exits_when_other_runner_fails_with_timeout(self) -> None:
from click.testing import CliRunner
real_sleep = time.sleep
call_count = [0]
def get_jobs_side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] < 2:
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
return [
{"name": "molecule-tests (0)", "conclusion": "running"},
{"name": "molecule-tests (1)", "conclusion": "failure"},
]
with (
patch.dict(
os.environ,
@@ -292,18 +398,15 @@ class TestCli:
},
clear=True,
),
patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01),
patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen,
patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
patch("os.killpg") as mock_killpg,
patch("os.getpgid") as mock_getpgid,
patch("time.sleep", side_effect=lambda x: real_sleep(0.1)),
):
mock_getpgid.return_value = 123
mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
mock_get_jobs.return_value = [
{"name": "molecule-tests (0)", "conclusion": "running"},
{"name": "molecule-tests (1)", "conclusion": "failure"},
]
proc = MagicMock()
proc.poll.return_value = None
proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]