diff --git a/src/devx/molecule/start_docker.py b/src/devx/molecule/start_docker.py index 61588d2..565492e 100644 --- a/src/devx/molecule/start_docker.py +++ b/src/devx/molecule/start_docker.py @@ -212,15 +212,33 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: return True click.echo(f" Insufficient space ({free_gb:.1f} GB), trying next...") - # If we found a working socket but with low space, use the last - # one as a fallback (better than nothing). - for sock in reversed(candidates): - if _try_socket(sock): - click.echo(f"Using low-space fallback: {sock}") - return True - + # No socket with sufficient space found. + # Don't fall back to the low-space inner dockerd — it will fail + # on image pulls. Instead, start a local dockerd on /dev/shm. click.echo(_("Host Docker not available, starting local dockerd...")) + # Kill the inner dockerd (started by the CI image) to free its + # data root and socket. The inner dockerd uses the container's + # overlay (38G, often 100% full). Killing it frees up the + # socket and any space used by its containers/volumes. + with contextlib.suppress(Exception): + subprocess.run( # nosec B603 B607 + ["pkill", "-f", "dockerd.*--host fd://"], + check=False, + timeout=5, + ) + time.sleep(2) + + # Clean up the inner dockerd's data root to free space + inner_data_root = "/home/grm-ci-runner-*/.local/share/docker" + rm_paths = " ".join(f"{inner_data_root}/{d}" for d in ("overlay2", "image", "volumes", "containers")) + with contextlib.suppress(Exception): + subprocess.run( # nosec B603 B607 + ["sh", "-c", f"rm -rf {rm_paths}"], + check=False, + timeout=30, + ) + # The existing dockerd (started by the CI image) uses the default data # root on the CI container's overlay (38G, often 100% full). We can't # kill it (different PID namespace) or bind to the same socket. Use a diff --git a/tests/unit/test_start_docker.py b/tests/unit/test_start_docker.py index 8bddea4..33ee0c9 100644 --- a/tests/unit/test_start_docker.py +++ b/tests/unit/test_start_docker.py @@ -170,6 +170,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) + @patch("devx.molecule.start_docker.subprocess.run") @patch("devx.molecule.start_docker.subprocess.Popen") @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") @patch("devx.molecule.start_docker.time.sleep") @@ -178,6 +179,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ntf: MagicMock, mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, @@ -196,6 +198,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready") @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) + @patch("devx.molecule.start_docker.subprocess.run") @patch("devx.molecule.start_docker.subprocess.Popen") @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") @patch("devx.molecule.start_docker.time.sleep") @@ -204,6 +207,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ntf: MagicMock, mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, @@ -300,18 +304,28 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.is_docker_ready") @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK})) @patch("devx.molecule.start_docker.glob.glob", return_value=[]) - def test_all_sockets_low_space_falls_back( + @patch("devx.molecule.start_docker.subprocess.run") + @patch("devx.molecule.start_docker.subprocess.Popen") + @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") + @patch("devx.molecule.start_docker.time.sleep") + def test_all_sockets_low_space_starts_local( self, + mock_sleep: MagicMock, + mock_ntf: MagicMock, + mock_popen: MagicMock, + mock_run: MagicMock, mock_glob: MagicMock, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock, ) -> None: - """Should fall back to a low-space socket if no better option exists.""" - mock_ready.side_effect = [True, True, True, True] - mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3] + """Should start local dockerd if all sockets have low space.""" + mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") + mock_ready.side_effect = [True, False, False, False, False, True] + mock_free.return_value = 5 * 1024**3 assert start_docker_daemon(timeout=5) is True + mock_popen.assert_called_once() @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") @@ -370,7 +384,7 @@ class TestStartDockerDaemon: with patch("builtins.open", mock_open(read_data="dockerd error log")): assert start_docker_daemon(timeout=3) is False mock_popen.assert_called_once() - assert mock_sleep.call_count == 3 + assert mock_sleep.call_count == 4 # 1 after pkill + 3 timeout retries @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") @@ -424,7 +438,7 @@ class TestStartDockerDaemon: mock_ready.side_effect = [False, False, True] assert start_docker_daemon(timeout=5) is True assert mock_popen.call_count == 1 - assert mock_sleep.call_count == 2 + assert mock_sleep.call_count == 3 # 1 after pkill + 2 loop retries @patch("devx.molecule.start_docker._diagnose_socket") @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)