From cdbee0a31712cc642095840003722fe27eb0f3ce Mon Sep 17 00:00:00 2001 From: emil Date: Sat, 15 Aug 2026 02:12:46 +0200 Subject: [PATCH] DEVX-155: fix: kill dockerd by PID when pkill fails, use /dev/shm for alive daemon Add pgrep diagnostics before/after pkill to identify lingering dockerd processes. If pkill fails and pgrep still finds dockerd, kill by PID directly via os.kill. When inner dockerd can't be killed (still alive), use /dev/shm as data root since the overlay is still full. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/devx/molecule/start_docker.py | 41 ++++++++++-- tests/unit/test_start_docker.py | 103 ++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 12 deletions(-) diff --git a/src/devx/molecule/start_docker.py b/src/devx/molecule/start_docker.py index 16c0665..c541671 100644 --- a/src/devx/molecule/start_docker.py +++ b/src/devx/molecule/start_docker.py @@ -224,8 +224,18 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: # overlay (38G, often 100% full). Killing it frees up the # socket and any space used by its containers/volumes. # Use SIGKILL (-9) since the inner dockerd may not respond to SIGTERM. - # Try multiple patterns to match different dockerd invocations. - for pattern in ["dockerd", "dockerd-entrypoint.sh"]: + # Try multiple approaches to ensure the inner dockerd is killed. + with contextlib.suppress(Exception): + result = subprocess.run( # nosec B603 B607 + ["pgrep", "-af", "dockerd"], + capture_output=True, + text=True, + timeout=5, + ) + if result.stdout.strip(): + click.echo(f" dockerd processes before kill: {result.stdout.strip()}") + + for pattern in ["dockerd", "dockerd-entrypoint.sh", "containerd"]: with contextlib.suppress(Exception): subprocess.run( # nosec B603 B607 ["pkill", "-9", "-f", pattern], @@ -234,6 +244,24 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: ) time.sleep(3) + # Check if dockerd processes are still alive + with contextlib.suppress(Exception): + result = subprocess.run( # nosec B603 B607 + ["pgrep", "-af", "dockerd"], + capture_output=True, + text=True, + timeout=5, + ) + if result.stdout.strip(): + click.echo(f" dockerd processes after kill: {result.stdout.strip()}") + # Try killing by PID directly + for pid_str in result.stdout.split("\n"): + pid = pid_str.split()[0] if pid_str.strip() else "" + if pid: + with contextlib.suppress(Exception): + os.kill(int(pid), 9) + time.sleep(2) + # Verify the inner dockerd is actually dead. If we can still # connect to /var/run/docker.sock, the old daemon is still running # and we need to use a different socket path. @@ -265,10 +293,11 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: timeout=30, ) - # Use a fresh data root on the container's overlay. - # /dev/shm is a small tmpfs (16G) — too small for images. - # The container's overlay (38G) has more space after cleanup. - docker_data_root = "/tmp/docker-data" # nosec B108 + # Use a fresh data root. If the inner dockerd is dead, use the + # container's overlay (38G, with freed space). If the inner + # dockerd is still alive, use /dev/shm (16G tmpfs) — the overlay + # is still full because the inner dockerd's data can't be cleaned. + docker_data_root = "/dev/shm/docker" if old_daemon_alive else "/tmp/docker-data" # nosec B108 # Remove stale socket if present with contextlib.suppress(OSError): diff --git a/tests/unit/test_start_docker.py b/tests/unit/test_start_docker.py index 695fe12..44419a1 100644 --- a/tests/unit/test_start_docker.py +++ b/tests/unit/test_start_docker.py @@ -18,6 +18,21 @@ from devx.molecule.start_docker import ( ) +def _pgrep_empty() -> MagicMock: + """Mock for pgrep returning no dockerd processes.""" + return MagicMock(stdout="", returncode=1) + + +def _docker_info_alive() -> MagicMock: + """Mock for docker info showing daemon alive.""" + return MagicMock(returncode=0) + + +def _docker_info_dead() -> MagicMock: + """Mock for docker info showing daemon dead.""" + return MagicMock(returncode=1) + + class TestIsDockerReady: @patch("devx.molecule.start_docker.subprocess.run") def test_ready(self, mock_run: MagicMock) -> None: @@ -190,6 +205,16 @@ class TestStartDockerDaemon: mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") # is_docker_ready: first check (inner dockerd) True, then local daemon checks mock_ready.side_effect = [True, False, False, False, False, True] + # subprocess.run calls: pgrep(before), pkill x3, pgrep(after), docker info, rm + mock_run.side_effect = [ + _pgrep_empty(), + MagicMock(), + MagicMock(), + MagicMock(), + _pgrep_empty(), + _docker_info_dead(), + MagicMock(), + ] assert start_docker_daemon(timeout=5) is True mock_popen.assert_called_once() @@ -218,11 +243,14 @@ class TestStartDockerDaemon: mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") mock_ready.side_effect = [True, False, False, False, False, True] mock_free.return_value = 5 * 1024**3 - # mock_run: pkill calls (x2), docker info check (returncode=1 = dead), rm call + # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm mock_run.side_effect = [ + _pgrep_empty(), MagicMock(), MagicMock(), - MagicMock(returncode=1), + MagicMock(), + _pgrep_empty(), + _docker_info_dead(), MagicMock(), ] assert start_docker_daemon(timeout=5) is True @@ -253,17 +281,61 @@ class TestStartDockerDaemon: mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") mock_ready.side_effect = [True, False, False, False, False, True] mock_free.return_value = 5 * 1024**3 - # mock_run: pkill calls (x2), docker info check (returncode=0 = alive), rm call + # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(alive), rm mock_run.side_effect = [ + _pgrep_empty(), MagicMock(), MagicMock(), - MagicMock(returncode=0), + MagicMock(), + _pgrep_empty(), + _docker_info_alive(), MagicMock(), ] assert start_docker_daemon(timeout=5) is True mock_popen.assert_called_once() assert os.environ.get("DOCKER_HOST") == "unix:///dev/shm/docker.sock" + @patch("devx.molecule.start_docker.os.kill") + @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes") + @patch("devx.molecule.start_docker.is_docker_ready") + @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) + @patch("devx.molecule.start_docker.glob.glob", return_value=[]) + @patch("devx.molecule.start_docker.subprocess.run") + @patch("devx.molecule.start_docker.subprocess.Popen") + @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") + @patch("devx.molecule.start_docker.time.sleep") + def test_inner_dockerd_killed_by_pid( + self, + mock_sleep: MagicMock, + mock_ntf: MagicMock, + mock_popen: MagicMock, + mock_run: MagicMock, + mock_glob: MagicMock, + mock_exists: MagicMock, + mock_ready: MagicMock, + mock_free: MagicMock, + mock_diag: MagicMock, + mock_kill: MagicMock, + ) -> None: + """Should kill dockerd by PID when pkill fails and pgrep finds processes.""" + mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") + mock_ready.side_effect = [True, False, False, False, False, True] + mock_free.return_value = 5 * 1024**3 + # pgrep(before) empty, pkill x3, pgrep(after) finds PID 12345, docker info(dead), rm + mock_run.side_effect = [ + _pgrep_empty(), + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(stdout="12345 /usr/bin/dockererd\n", returncode=0), + _docker_info_dead(), + MagicMock(), + ] + assert start_docker_daemon(timeout=5) is True + mock_popen.assert_called_once() + mock_kill.assert_called_once_with(12345, 9) + @patch("devx.molecule.start_docker._diagnose_socket") @patch("devx.molecule.start_docker._get_docker_free_bytes") @patch("devx.molecule.start_docker.is_docker_ready") @@ -367,6 +439,16 @@ class TestStartDockerDaemon: mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") mock_ready.side_effect = [True, False, False, False, False, True] mock_free.return_value = 5 * 1024**3 + # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm + mock_run.side_effect = [ + _pgrep_empty(), + MagicMock(), + MagicMock(), + MagicMock(), + _pgrep_empty(), + _docker_info_dead(), + MagicMock(), + ] assert start_docker_daemon(timeout=5) is True mock_popen.assert_called_once() @@ -394,6 +476,15 @@ class TestStartDockerDaemon: mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") # No sockets exist, local daemon starts mock_ready.side_effect = [False, False, False, False, True] + # subprocess.run: pgrep(before), pkill x3, pgrep(after), rm + mock_run.side_effect = [ + _pgrep_empty(), + MagicMock(), + MagicMock(), + MagicMock(), + _pgrep_empty(), + MagicMock(), + ] assert start_docker_daemon(timeout=5) is True mock_popen.assert_called_once() popen_args = mock_popen.call_args.args[0] @@ -427,7 +518,7 @@ class TestStartDockerDaemon: with patch("builtins.open", mock_open(read_data="dockerd error log")): assert start_docker_daemon(timeout=3) is False mock_popen.assert_called_once() - assert mock_sleep.call_count == 4 # 1 after pkill + 3 timeout retries + assert mock_sleep.call_count == 5 # 1 after pkill + 1 after pgrep + 3 timeout retries @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") @@ -481,7 +572,7 @@ class TestStartDockerDaemon: mock_ready.side_effect = [False, False, True] assert start_docker_daemon(timeout=5) is True assert mock_popen.call_count == 1 - assert mock_sleep.call_count == 3 # 1 after pkill + 2 loop retries + assert mock_sleep.call_count == 4 # 1 after pkill + 1 after pgrep + 2 loop retries @patch("devx.molecule.start_docker._diagnose_socket") @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)