DEVX-155: fix: prefer rootless Docker socket over low-space inner DinD daemon

When a CI container has an inner dockerd (DinD) writing to the
container's overlay (e.g. 38 GB), image pulls fail with ENOSPC.
start_docker.py now checks the daemon's free disk space and tries
rootless sockets (which have access to the host's full filesystem)
when the default socket has less than 20 GB free.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

src/devx/molecule/start_docker.py

@@ -10,6 +10,12 @@ If the host socket is not available, it tries the rootless socket, then
 starts a local ``dockerd`` with the vfs storage driver (requires
 privileged container).
 
+When the host socket IS available but has limited disk space (e.g. an
+inner DinD daemon writing to a 38 GB container overlay), the script
+prefers a rootless socket that has more available space.  This prevents
+"no space left on device" errors during molecule tests that pull images
+and create containers via the Docker daemon.
+
 Usage::
 
     python3 -m devx.molecule.start_docker [--timeout 30]
@@ -19,6 +25,7 @@ from __future__ import annotations
 
 import glob
 import os
+import shutil
 import subprocess  # nosec B404
 import sys
 import tempfile
@@ -32,6 +39,10 @@ DEFAULT_TIMEOUT = 30
 DOCKER_SOCK = "/var/run/docker.sock"
 # Rootless socket fallback (e.g. /run/user/994/docker.sock)
 ROOTLESS_SOCK = f"/run/user/{os.getuid()}/docker.sock"
+# Minimum free bytes for a Docker daemon to be considered usable.
+# Below this, image pulls and container creation will fail with ENOSPC.
+# 20 GB leaves room for molecule-test-base (~500 MB) + a few containers.
+MIN_FREE_BYTES = 20 * 1024**3  # 20 GB
 
 
 def is_docker_ready() -> bool:
@@ -46,6 +57,46 @@ def is_docker_ready() -> bool:
     return result.returncode == 0
 
 
+def _get_docker_free_bytes() -> int:
+    """Get free disk space (bytes) at the Docker daemon's data root.
+
+    Returns 0 if the daemon is not reachable or the data root cannot be
+    determined.
+    """
+    docker_host = os.environ.get("DOCKER_HOST", f"unix://{DOCKER_SOCK}")
+    try:
+        result = subprocess.run(  # nosec B603 B607
+            [
+                "docker",
+                "info",
+                "--format",
+                "{{.DockerRootDir}}",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=False,
+            env={**os.environ, "DOCKER_HOST": docker_host},
+        )
+        if result.returncode != 0 or not result.stdout.strip():
+            return 0
+        data_root = result.stdout.strip()
+        if not os.path.exists(data_root):
+            return 0
+        return shutil.disk_usage(data_root).free
+    except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
+        return 0
+
+
+def _try_socket(sock_path: str) -> bool:
+    """Set DOCKER_HOST to *sock_path* and check if the daemon is ready.
+
+    Returns ``True`` if the daemon responds, ``False`` otherwise.
+    """
+    os.environ["DOCKER_HOST"] = f"unix://{sock_path}"
+    return is_docker_ready()
+
+
 def _diagnose_socket() -> None:
     """Print diagnostic info about the Docker socket."""
     click.echo(f"DOCKER_HOST = {os.environ.get('DOCKER_HOST', '(not set)')}")
@@ -97,10 +148,14 @@ def _diagnose_socket() -> None:
 def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
     """Ensure Docker is ready for molecule tests.
 
-    First tries the host socket. If that works, sets ``DOCKER_HOST`` and
-    returns immediately. If not, tries the rootless socket. If neither
-    works, starts a local ``dockerd`` with vfs storage driver (requires
-    privileged container).
+    Tries sockets in this order, preferring ones with enough disk space:
+
+    1. Host socket (``/var/run/docker.sock``) — if it has ≥ ``MIN_FREE_BYTES``
+       free space, use it immediately.
+    2. Rootless sockets (``/run/user/*/docker.sock``) — if the host socket
+       has insufficient space, try rootless sockets which may have access
+       to the host's full filesystem.
+    3. Local ``dockerd`` with vfs storage driver — last resort.
 
     Returns ``True`` if Docker is ready, ``False`` if it failed to
     start within the timeout.
@@ -115,25 +170,45 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
 
     # Check if host Docker is already available
     if is_docker_ready():
-        click.echo(_("Docker daemon already running"))
-        return True
+        free_bytes = _get_docker_free_bytes()
+        free_gb = free_bytes / 1024**3
+        click.echo(f"Docker daemon already running (free space: {free_gb:.1f} GB)")
+        if free_bytes >= MIN_FREE_BYTES:
+            return True
+        click.echo(
+            f"Host Docker has only {free_gb:.1f} GB free "
+            f"(need ≥ {MIN_FREE_BYTES / 1024**3:.0f} GB), trying rootless sockets..."
+        )
+    else:
+        click.echo("Host Docker not available, trying rootless sockets...")
 
-    # Try rootless socket (e.g. /run/user/994/docker.sock)
-    click.echo(f"Trying rootless socket: {ROOTLESS_SOCK}")
-    os.environ["DOCKER_HOST"] = f"unix://{ROOTLESS_SOCK}"
-    if os.path.exists(ROOTLESS_SOCK) and is_docker_ready():
-        click.echo(_("Docker daemon already running"))
-        return True
-
-    # Scan for any rootless sockets at other UIDs
+    # Collect candidate rootless sockets
+    candidates: list[str] = []
+    if os.path.exists(ROOTLESS_SOCK):
+        candidates.append(ROOTLESS_SOCK)
     for sock in sorted(glob.glob("/run/user/*/docker.sock")):
-        if sock == ROOTLESS_SOCK:
+        if sock not in candidates:
+            candidates.append(sock)
+
+    # Try each rootless socket — prefer one with enough free space
+    for sock in candidates:
+        click.echo(f"Trying rootless socket: {sock}")
+        if not _try_socket(sock):
             continue
-        click.echo(f"Trying alternative rootless socket: {sock}")
-        os.environ["DOCKER_HOST"] = f"unix://{sock}"
-        if is_docker_ready():
+        free_bytes = _get_docker_free_bytes()
+        free_gb = free_bytes / 1024**3
+        click.echo(f"  Docker daemon ready (free space: {free_gb:.1f} GB)")
+        if free_bytes >= MIN_FREE_BYTES:
             click.echo(_("Docker daemon already running"))
             return True
+        click.echo(f"  Insufficient space ({free_gb:.1f} GB), trying next...")
+
+    # If we found a working rootless socket but with low space, use it
+    # as a fallback (better than nothing).
+    for sock in reversed(candidates):
+        if _try_socket(sock):
+            click.echo(f"Using low-space fallback: {sock}")
+            return True
 
     click.echo(_("Host Docker not available, starting local dockerd..."))
 

tests/unit/test_start_docker.py

@@ -8,6 +8,8 @@ from click.testing import CliRunner
 from devx.molecule.start_docker import (
     DOCKER_SOCK,
     _diagnose_socket,
+    _get_docker_free_bytes,
+    _try_socket,
     is_docker_ready,
     main,
     start_docker_daemon,
@@ -40,6 +42,42 @@ class TestIsDockerReady:
         assert call_kwargs.kwargs["env"]["DOCKER_HOST"] == rootless
 
 
+class TestGetDockerFreeBytes:
+    @patch("devx.molecule.start_docker.shutil.disk_usage")
+    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    @patch("devx.molecule.start_docker.subprocess.run")
+    def test_returns_free_bytes(self, mock_run: MagicMock, mock_exists: MagicMock, mock_du: MagicMock) -> None:
+        mock_run.return_value = MagicMock(stdout="/var/lib/docker\n", returncode=0, text="")
+        mock_du.return_value = MagicMock(free=50 * 1024**3)
+        with patch.dict("os.environ", {"DOCKER_HOST": f"unix://{DOCKER_SOCK}"}, clear=False):
+            assert _get_docker_free_bytes() == 50 * 1024**3
+
+    @patch("devx.molecule.start_docker.subprocess.run")
+    def test_daemon_not_reachable(self, mock_run: MagicMock) -> None:
+        mock_run.return_value = MagicMock(returncode=1, stdout="", text="")
+        assert _get_docker_free_bytes() == 0
+
+    @patch("devx.molecule.start_docker.os.path.exists", return_value=False)
+    @patch("devx.molecule.start_docker.subprocess.run")
+    def test_data_root_not_accessible(self, mock_run: MagicMock, mock_exists: MagicMock) -> None:
+        mock_run.return_value = MagicMock(stdout="/some/path\n", returncode=0, text="")
+        assert _get_docker_free_bytes() == 0
+
+    @patch("devx.molecule.start_docker.subprocess.run", side_effect=FileNotFoundError)
+    def test_subprocess_not_found(self, mock_run: MagicMock) -> None:
+        assert _get_docker_free_bytes() == 0
+
+
+class TestTrySocket:
+    @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
+    def test_ready(self, mock_ready: MagicMock) -> None:
+        assert _try_socket("/run/user/999/docker.sock") is True
+
+    @patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
+    def test_not_ready(self, mock_ready: MagicMock) -> None:
+        assert _try_socket("/run/user/999/docker.sock") is False
+
+
 class TestDiagnoseSocket:
     @patch("devx.molecule.start_docker.os.stat")
     @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
@@ -88,61 +126,79 @@ class TestDiagnoseSocket:
 
 class TestStartDockerDaemon:
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
-    def test_host_socket_available(self, mock_ready: MagicMock, mock_diag: MagicMock) -> None:
-        """Should return immediately if host Docker is available."""
+    def test_host_socket_available_with_space(
+        self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+    ) -> None:
+        """Should return immediately if host Docker has enough space."""
         assert start_docker_daemon(timeout=5) is True
         mock_ready.assert_called_once()
         mock_diag.assert_called_once()
 
     @patch("devx.molecule.start_docker._diagnose_socket")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
-    @patch("devx.molecule.start_docker.is_docker_ready")
-    def test_rootless_socket_available(
-        self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=5 * 1024**3)
+    @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
+    def test_host_socket_low_space_tries_rootless(
+        self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
     ) -> None:
-        """Should use rootless socket if host socket fails."""
-        # First check (host) fails, second check (rootless) succeeds
-        mock_ready.side_effect = [False, True]
-        with patch("devx.molecule.start_docker.glob.glob", return_value=[]):
+        """Should try rootless sockets if host Docker has low space."""
+        with (
+            patch("devx.molecule.start_docker.glob.glob", return_value=[]),
+            patch("devx.molecule.start_docker.os.path.exists", return_value=False),
+        ):
+            # Host ready but low space, no rootless sockets, falls back to host
             assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker._diagnose_socket")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    def test_rootless_socket_with_space(
+        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+    ) -> None:
+        """Should use rootless socket if host has low space and rootless has enough."""
+        # Host ready but low space, rootless ready with enough space
+        mock_ready.side_effect = [True, True]
+        mock_free.side_effect = [5 * 1024**3, 200 * 1024**3]
+        own_sock = f"/run/user/{os.getuid()}/docker.sock"
+        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]):
+            assert start_docker_daemon(timeout=5) is True
+
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
     def test_alt_rootless_socket_found(
-        self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
+        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
     ) -> None:
         """Should find rootless socket at a different UID via glob scan."""
-        # Host fails, own rootless fails, alt rootless succeeds
+        # Host not ready, own rootless not ready, alt rootless ready with space
         mock_ready.side_effect = [False, False, True]
+        mock_free.side_effect = [200 * 1024**3]
+        own_sock = f"/run/user/{os.getuid()}/docker.sock"
         alt_sock = "/run/user/999/docker.sock"
-        with patch("devx.molecule.start_docker.glob.glob", return_value=[alt_sock]):
+        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]):
             assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker._diagnose_socket")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
-    def test_alt_rootless_socket_skips_own(
-        self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
+    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    def test_all_sockets_low_space_falls_back(
+        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
     ) -> None:
-        """Should skip the own rootless socket in glob scan (already tried)."""
-        # Host fails, own rootless fails, alt rootless also fails, dockerd fails
-        mock_ready.side_effect = [False, False, False, False, False, False]
+        """Should fall back to a low-space socket if no better option exists."""
+        # Host ready low space, rootless ready low space, falls back to rootless
+        mock_ready.side_effect = [True, True, True, True]
+        mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3]
         own_sock = f"/run/user/{os.getuid()}/docker.sock"
-        alt_sock = "/run/user/999/docker.sock"
-        with (
-            patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]),
-            patch("devx.molecule.start_docker.time.sleep"),
-            patch("devx.molecule.start_docker.subprocess.Popen"),
-            patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") as mock_ntf,
-            patch("builtins.open", mock_open(read_data="err")),
-        ):
-            mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
-            assert start_docker_daemon(timeout=2) is False
+        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]):
+            assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.os.path.exists", return_value=False)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
     @patch("devx.molecule.start_docker.time.sleep")
@@ -155,6 +211,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ready: MagicMock,
         mock_exists: MagicMock,
+        mock_free: MagicMock,
         mock_diag: MagicMock,
         mock_glob: MagicMock,
     ) -> None:
@@ -171,6 +228,7 @@ class TestStartDockerDaemon:
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.os.path.exists", return_value=False)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
     @patch("devx.molecule.start_docker.time.sleep")
@@ -183,6 +241,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ready: MagicMock,
         mock_exists: MagicMock,
+        mock_free: MagicMock,
         mock_diag: MagicMock,
         mock_glob: MagicMock,
     ) -> None:
@@ -194,6 +253,7 @@ class TestStartDockerDaemon:
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.os.path.exists", return_value=False)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
     @patch("devx.molecule.start_docker.time.sleep")
@@ -206,6 +266,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ready: MagicMock,
         mock_exists: MagicMock,
+        mock_free: MagicMock,
         mock_diag: MagicMock,
         mock_glob: MagicMock,
     ) -> None:
@@ -216,6 +277,7 @@ class TestStartDockerDaemon:
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.os.path.exists", return_value=False)
     @patch("devx.molecule.start_docker.is_docker_ready")
     @patch("devx.molecule.start_docker.time.sleep")
@@ -228,6 +290,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ready: MagicMock,
         mock_exists: MagicMock,
+        mock_free: MagicMock,
         mock_diag: MagicMock,
         mock_glob: MagicMock,
     ) -> None:
@@ -239,12 +302,14 @@ class TestStartDockerDaemon:
         assert mock_sleep.call_count == 1
 
     @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
     @patch("devx.molecule.start_docker.os.environ")
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
     def test_sets_docker_host(
         self,
         mock_ready: MagicMock,
         mock_environ: MagicMock,
+        mock_free: MagicMock,
         mock_diag: MagicMock,
     ) -> None:
         """DOCKER_HOST must be set so molecule connects to correct socket."""