From a2d47b7efc77ec6a4cc284cd1c0bb681c0721765 Mon Sep 17 00:00:00 2001 From: emil Date: Sat, 15 Aug 2026 00:41:27 +0200 Subject: [PATCH] DEVX-155: fix: start local dockerd instead of using low-space inner dockerd When no socket has sufficient space, don't fall back to the low-space inner dockerd (which will fail on image pulls). Instead, kill the inner dockerd, clean up its data root to free space, and start a local dockerd on /dev/shm with vfs storage driver. Also adds pkill of the inner dockerd and cleanup of its data root (overlay2, image, volumes, containers) before starting the local dockerd, to free up the 2.4GB used by the inner dockerd's data. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/devx/molecule/start_docker.py | 32 ++++++++++++++++++++++++------- tests/unit/test_start_docker.py | 26 +++++++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/devx/molecule/start_docker.py b/src/devx/molecule/start_docker.py index 61588d2..565492e 100644 --- a/src/devx/molecule/start_docker.py +++ b/src/devx/molecule/start_docker.py @@ -212,15 +212,33 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: return True click.echo(f" Insufficient space ({free_gb:.1f} GB), trying next...") - # If we found a working socket but with low space, use the last - # one as a fallback (better than nothing). - for sock in reversed(candidates): - if _try_socket(sock): - click.echo(f"Using low-space fallback: {sock}") - return True - + # No socket with sufficient space found. + # Don't fall back to the low-space inner dockerd — it will fail + # on image pulls. Instead, start a local dockerd on /dev/shm. click.echo(_("Host Docker not available, starting local dockerd...")) + # Kill the inner dockerd (started by the CI image) to free its + # data root and socket. The inner dockerd uses the container's + # overlay (38G, often 100% full). Killing it frees up the + # socket and any space used by its containers/volumes. + with contextlib.suppress(Exception): + subprocess.run( # nosec B603 B607 + ["pkill", "-f", "dockerd.*--host fd://"], + check=False, + timeout=5, + ) + time.sleep(2) + + # Clean up the inner dockerd's data root to free space + inner_data_root = "/home/grm-ci-runner-*/.local/share/docker" + rm_paths = " ".join(f"{inner_data_root}/{d}" for d in ("overlay2", "image", "volumes", "containers")) + with contextlib.suppress(Exception): + subprocess.run( # nosec B603 B607 + ["sh", "-c", f"rm -rf {rm_paths}"], + check=False, + timeout=30, + ) + # The existing dockerd (started by the CI image) uses the default data # root on the CI container's overlay (38G, often 100% full). We can't # kill it (different PID namespace) or bind to the same socket. Use a diff --git a/tests/unit/test_start_docker.py b/tests/unit/test_start_docker.py index 8bddea4..33ee0c9 100644 --- a/tests/unit/test_start_docker.py +++ b/tests/unit/test_start_docker.py @@ -170,6 +170,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) + @patch("devx.molecule.start_docker.subprocess.run") @patch("devx.molecule.start_docker.subprocess.Popen") @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") @patch("devx.molecule.start_docker.time.sleep") @@ -178,6 +179,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ntf: MagicMock, mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, @@ -196,6 +198,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready") @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) + @patch("devx.molecule.start_docker.subprocess.run") @patch("devx.molecule.start_docker.subprocess.Popen") @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") @patch("devx.molecule.start_docker.time.sleep") @@ -204,6 +207,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ntf: MagicMock, mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, @@ -300,18 +304,28 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready") @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) - def test_all_sockets_low_space_falls_back( + @patch("devx.molecule.start_docker.subprocess.run") + @patch("devx.molecule.start_docker.subprocess.Popen") + @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") + @patch("devx.molecule.start_docker.time.sleep") + def test_all_sockets_low_space_starts_local( self, + mock_sleep: MagicMock, + mock_ntf: MagicMock, + mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock, ) -> None: - """Should fall back to a low-space socket if no better option exists.""" - mock_ready.side_effect = [True, True, True, True] - mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3] + """Should start local dockerd if all sockets have low space.""" + mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") + mock_ready.side_effect = [True, False, False, False, False, True] + mock_free.return_value = 5 * 1024**3 assert start_docker_daemon(timeout=5) is True + mock_popen.assert_called_once() @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") @@ -370,7 +384,7 @@ class TestStartDockerDaemon: with patch("builtins.open", mock_open(read_data="dockerd error log")): assert start_docker_daemon(timeout=3) is False mock_popen.assert_called_once() - assert mock_sleep.call_count == 3 + assert mock_sleep.call_count == 4 # 1 after pkill + 3 timeout retries @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") @@ -424,7 +438,7 @@ class TestStartDockerDaemon: mock_ready.side_effect = [False, False, True] assert start_docker_daemon(timeout=5) is True assert mock_popen.call_count == 1 - assert mock_sleep.call_count == 2 + assert mock_sleep.call_count == 3 # 1 after pkill + 2 loop retries @patch("devx.molecule.start_docker._diagnose_socket") @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)