diff --git a/src/devx/molecule/start_docker.py b/src/devx/molecule/start_docker.py index 7d4b9be..61f85ae 100644 --- a/src/devx/molecule/start_docker.py +++ b/src/devx/molecule/start_docker.py @@ -10,6 +10,12 @@ If the host socket is not available, it tries the rootless socket, then starts a local ``dockerd`` with the vfs storage driver (requires privileged container). +When the host socket IS available but has limited disk space (e.g. an +inner DinD daemon writing to a 38 GB container overlay), the script +prefers a rootless socket that has more available space. This prevents +"no space left on device" errors during molecule tests that pull images +and create containers via the Docker daemon. + Usage:: python3 -m devx.molecule.start_docker [--timeout 30] @@ -19,6 +25,7 @@ from __future__ import annotations import glob import os +import shutil import subprocess # nosec B404 import sys import tempfile @@ -32,6 +39,10 @@ DEFAULT_TIMEOUT = 30 DOCKER_SOCK = "/var/run/docker.sock" # Rootless socket fallback (e.g. /run/user/994/docker.sock) ROOTLESS_SOCK = f"/run/user/{os.getuid()}/docker.sock" +# Minimum free bytes for a Docker daemon to be considered usable. +# Below this, image pulls and container creation will fail with ENOSPC. +# 20 GB leaves room for molecule-test-base (~500 MB) + a few containers. +MIN_FREE_BYTES = 20 * 1024**3 # 20 GB def is_docker_ready() -> bool: @@ -46,6 +57,46 @@ def is_docker_ready() -> bool: return result.returncode == 0 +def _get_docker_free_bytes() -> int: + """Get free disk space (bytes) at the Docker daemon's data root. + + Returns 0 if the daemon is not reachable or the data root cannot be + determined. + """ + docker_host = os.environ.get("DOCKER_HOST", f"unix://{DOCKER_SOCK}") + try: + result = subprocess.run( # nosec B603 B607 + [ + "docker", + "info", + "--format", + "{{.DockerRootDir}}", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + env={**os.environ, "DOCKER_HOST": docker_host}, + ) + if result.returncode != 0 or not result.stdout.strip(): + return 0 + data_root = result.stdout.strip() + if not os.path.exists(data_root): + return 0 + return shutil.disk_usage(data_root).free + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + return 0 + + +def _try_socket(sock_path: str) -> bool: + """Set DOCKER_HOST to *sock_path* and check if the daemon is ready. + + Returns ``True`` if the daemon responds, ``False`` otherwise. + """ + os.environ["DOCKER_HOST"] = f"unix://{sock_path}" + return is_docker_ready() + + def _diagnose_socket() -> None: """Print diagnostic info about the Docker socket.""" click.echo(f"DOCKER_HOST = {os.environ.get('DOCKER_HOST', '(not set)')}") @@ -97,10 +148,14 @@ def _diagnose_socket() -> None: def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: """Ensure Docker is ready for molecule tests. - First tries the host socket. If that works, sets ``DOCKER_HOST`` and - returns immediately. If not, tries the rootless socket. If neither - works, starts a local ``dockerd`` with vfs storage driver (requires - privileged container). + Tries sockets in this order, preferring ones with enough disk space: + + 1. Host socket (``/var/run/docker.sock``) — if it has ≥ ``MIN_FREE_BYTES`` + free space, use it immediately. + 2. Rootless sockets (``/run/user/*/docker.sock``) — if the host socket + has insufficient space, try rootless sockets which may have access + to the host's full filesystem. + 3. Local ``dockerd`` with vfs storage driver — last resort. Returns ``True`` if Docker is ready, ``False`` if it failed to start within the timeout. @@ -115,25 +170,45 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool: # Check if host Docker is already available if is_docker_ready(): - click.echo(_("Docker daemon already running")) - return True + free_bytes = _get_docker_free_bytes() + free_gb = free_bytes / 1024**3 + click.echo(f"Docker daemon already running (free space: {free_gb:.1f} GB)") + if free_bytes >= MIN_FREE_BYTES: + return True + click.echo( + f"Host Docker has only {free_gb:.1f} GB free " + f"(need ≥ {MIN_FREE_BYTES / 1024**3:.0f} GB), trying rootless sockets..." + ) + else: + click.echo("Host Docker not available, trying rootless sockets...") - # Try rootless socket (e.g. /run/user/994/docker.sock) - click.echo(f"Trying rootless socket: {ROOTLESS_SOCK}") - os.environ["DOCKER_HOST"] = f"unix://{ROOTLESS_SOCK}" - if os.path.exists(ROOTLESS_SOCK) and is_docker_ready(): - click.echo(_("Docker daemon already running")) - return True - - # Scan for any rootless sockets at other UIDs + # Collect candidate rootless sockets + candidates: list[str] = [] + if os.path.exists(ROOTLESS_SOCK): + candidates.append(ROOTLESS_SOCK) for sock in sorted(glob.glob("/run/user/*/docker.sock")): - if sock == ROOTLESS_SOCK: + if sock not in candidates: + candidates.append(sock) + + # Try each rootless socket — prefer one with enough free space + for sock in candidates: + click.echo(f"Trying rootless socket: {sock}") + if not _try_socket(sock): continue - click.echo(f"Trying alternative rootless socket: {sock}") - os.environ["DOCKER_HOST"] = f"unix://{sock}" - if is_docker_ready(): + free_bytes = _get_docker_free_bytes() + free_gb = free_bytes / 1024**3 + click.echo(f" Docker daemon ready (free space: {free_gb:.1f} GB)") + if free_bytes >= MIN_FREE_BYTES: click.echo(_("Docker daemon already running")) return True + click.echo(f" Insufficient space ({free_gb:.1f} GB), trying next...") + + # If we found a working rootless socket but with low space, use it + # as a fallback (better than nothing). + for sock in reversed(candidates): + if _try_socket(sock): + click.echo(f"Using low-space fallback: {sock}") + return True click.echo(_("Host Docker not available, starting local dockerd...")) diff --git a/tests/unit/test_start_docker.py b/tests/unit/test_start_docker.py index f02ffb6..35a7eb2 100644 --- a/tests/unit/test_start_docker.py +++ b/tests/unit/test_start_docker.py @@ -8,6 +8,8 @@ from click.testing import CliRunner from devx.molecule.start_docker import ( DOCKER_SOCK, _diagnose_socket, + _get_docker_free_bytes, + _try_socket, is_docker_ready, main, start_docker_daemon, @@ -40,6 +42,42 @@ class TestIsDockerReady: assert call_kwargs.kwargs["env"]["DOCKER_HOST"] == rootless +class TestGetDockerFreeBytes: + @patch("devx.molecule.start_docker.shutil.disk_usage") + @patch("devx.molecule.start_docker.os.path.exists", return_value=True) + @patch("devx.molecule.start_docker.subprocess.run") + def test_returns_free_bytes(self, mock_run: MagicMock, mock_exists: MagicMock, mock_du: MagicMock) -> None: + mock_run.return_value = MagicMock(stdout="/var/lib/docker\n", returncode=0, text="") + mock_du.return_value = MagicMock(free=50 * 1024**3) + with patch.dict("os.environ", {"DOCKER_HOST": f"unix://{DOCKER_SOCK}"}, clear=False): + assert _get_docker_free_bytes() == 50 * 1024**3 + + @patch("devx.molecule.start_docker.subprocess.run") + def test_daemon_not_reachable(self, mock_run: MagicMock) -> None: + mock_run.return_value = MagicMock(returncode=1, stdout="", text="") + assert _get_docker_free_bytes() == 0 + + @patch("devx.molecule.start_docker.os.path.exists", return_value=False) + @patch("devx.molecule.start_docker.subprocess.run") + def test_data_root_not_accessible(self, mock_run: MagicMock, mock_exists: MagicMock) -> None: + mock_run.return_value = MagicMock(stdout="/some/path\n", returncode=0, text="") + assert _get_docker_free_bytes() == 0 + + @patch("devx.molecule.start_docker.subprocess.run", side_effect=FileNotFoundError) + def test_subprocess_not_found(self, mock_run: MagicMock) -> None: + assert _get_docker_free_bytes() == 0 + + +class TestTrySocket: + @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) + def test_ready(self, mock_ready: MagicMock) -> None: + assert _try_socket("/run/user/999/docker.sock") is True + + @patch("devx.molecule.start_docker.is_docker_ready", return_value=False) + def test_not_ready(self, mock_ready: MagicMock) -> None: + assert _try_socket("/run/user/999/docker.sock") is False + + class TestDiagnoseSocket: @patch("devx.molecule.start_docker.os.stat") @patch("devx.molecule.start_docker.os.path.exists", return_value=True) @@ -88,61 +126,79 @@ class TestDiagnoseSocket: class TestStartDockerDaemon: @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3) @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) - def test_host_socket_available(self, mock_ready: MagicMock, mock_diag: MagicMock) -> None: - """Should return immediately if host Docker is available.""" + def test_host_socket_available_with_space( + self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock + ) -> None: + """Should return immediately if host Docker has enough space.""" assert start_docker_daemon(timeout=5) is True mock_ready.assert_called_once() mock_diag.assert_called_once() @patch("devx.molecule.start_docker._diagnose_socket") - @patch("devx.molecule.start_docker.os.path.exists", return_value=True) - @patch("devx.molecule.start_docker.is_docker_ready") - def test_rootless_socket_available( - self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=5 * 1024**3) + @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) + def test_host_socket_low_space_tries_rootless( + self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock ) -> None: - """Should use rootless socket if host socket fails.""" - # First check (host) fails, second check (rootless) succeeds - mock_ready.side_effect = [False, True] - with patch("devx.molecule.start_docker.glob.glob", return_value=[]): + """Should try rootless sockets if host Docker has low space.""" + with ( + patch("devx.molecule.start_docker.glob.glob", return_value=[]), + patch("devx.molecule.start_docker.os.path.exists", return_value=False), + ): + # Host ready but low space, no rootless sockets, falls back to host assert start_docker_daemon(timeout=5) is True @patch("devx.molecule.start_docker._diagnose_socket") - @patch("devx.molecule.start_docker.os.path.exists", return_value=True) + @patch("devx.molecule.start_docker._get_docker_free_bytes") @patch("devx.molecule.start_docker.is_docker_ready") + @patch("devx.molecule.start_docker.os.path.exists", return_value=True) + def test_rootless_socket_with_space( + self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock + ) -> None: + """Should use rootless socket if host has low space and rootless has enough.""" + # Host ready but low space, rootless ready with enough space + mock_ready.side_effect = [True, True] + mock_free.side_effect = [5 * 1024**3, 200 * 1024**3] + own_sock = f"/run/user/{os.getuid()}/docker.sock" + with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]): + assert start_docker_daemon(timeout=5) is True + + @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes") + @patch("devx.molecule.start_docker.is_docker_ready") + @patch("devx.molecule.start_docker.os.path.exists", return_value=True) def test_alt_rootless_socket_found( - self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock + self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock ) -> None: """Should find rootless socket at a different UID via glob scan.""" - # Host fails, own rootless fails, alt rootless succeeds + # Host not ready, own rootless not ready, alt rootless ready with space mock_ready.side_effect = [False, False, True] + mock_free.side_effect = [200 * 1024**3] + own_sock = f"/run/user/{os.getuid()}/docker.sock" alt_sock = "/run/user/999/docker.sock" - with patch("devx.molecule.start_docker.glob.glob", return_value=[alt_sock]): + with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]): assert start_docker_daemon(timeout=5) is True @patch("devx.molecule.start_docker._diagnose_socket") - @patch("devx.molecule.start_docker.os.path.exists", return_value=True) + @patch("devx.molecule.start_docker._get_docker_free_bytes") @patch("devx.molecule.start_docker.is_docker_ready") - def test_alt_rootless_socket_skips_own( - self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock + @patch("devx.molecule.start_docker.os.path.exists", return_value=True) + def test_all_sockets_low_space_falls_back( + self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock ) -> None: - """Should skip the own rootless socket in glob scan (already tried).""" - # Host fails, own rootless fails, alt rootless also fails, dockerd fails - mock_ready.side_effect = [False, False, False, False, False, False] + """Should fall back to a low-space socket if no better option exists.""" + # Host ready low space, rootless ready low space, falls back to rootless + mock_ready.side_effect = [True, True, True, True] + mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3] own_sock = f"/run/user/{os.getuid()}/docker.sock" - alt_sock = "/run/user/999/docker.sock" - with ( - patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]), - patch("devx.molecule.start_docker.time.sleep"), - patch("devx.molecule.start_docker.subprocess.Popen"), - patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") as mock_ntf, - patch("builtins.open", mock_open(read_data="err")), - ): - mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log") - assert start_docker_daemon(timeout=2) is False + with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]): + assert start_docker_daemon(timeout=5) is True @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0) @patch("devx.molecule.start_docker.os.path.exists", return_value=False) @patch("devx.molecule.start_docker.is_docker_ready", return_value=False) @patch("devx.molecule.start_docker.time.sleep") @@ -155,6 +211,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ready: MagicMock, mock_exists: MagicMock, + mock_free: MagicMock, mock_diag: MagicMock, mock_glob: MagicMock, ) -> None: @@ -171,6 +228,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0) @patch("devx.molecule.start_docker.os.path.exists", return_value=False) @patch("devx.molecule.start_docker.is_docker_ready", return_value=False) @patch("devx.molecule.start_docker.time.sleep") @@ -183,6 +241,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ready: MagicMock, mock_exists: MagicMock, + mock_free: MagicMock, mock_diag: MagicMock, mock_glob: MagicMock, ) -> None: @@ -194,6 +253,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0) @patch("devx.molecule.start_docker.os.path.exists", return_value=False) @patch("devx.molecule.start_docker.is_docker_ready", return_value=False) @patch("devx.molecule.start_docker.time.sleep") @@ -206,6 +266,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ready: MagicMock, mock_exists: MagicMock, + mock_free: MagicMock, mock_diag: MagicMock, mock_glob: MagicMock, ) -> None: @@ -216,6 +277,7 @@ class TestStartDockerDaemon: @patch("devx.molecule.start_docker.glob.glob", return_value=[]) @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0) @patch("devx.molecule.start_docker.os.path.exists", return_value=False) @patch("devx.molecule.start_docker.is_docker_ready") @patch("devx.molecule.start_docker.time.sleep") @@ -228,6 +290,7 @@ class TestStartDockerDaemon: mock_sleep: MagicMock, mock_ready: MagicMock, mock_exists: MagicMock, + mock_free: MagicMock, mock_diag: MagicMock, mock_glob: MagicMock, ) -> None: @@ -239,12 +302,14 @@ class TestStartDockerDaemon: assert mock_sleep.call_count == 1 @patch("devx.molecule.start_docker._diagnose_socket") + @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3) @patch("devx.molecule.start_docker.os.environ") @patch("devx.molecule.start_docker.is_docker_ready", return_value=True) def test_sets_docker_host( self, mock_ready: MagicMock, mock_environ: MagicMock, + mock_free: MagicMock, mock_diag: MagicMock, ) -> None: """DOCKER_HOST must be set so molecule connects to correct socket."""