DEVX-155: fix: prefer /run/host-docker.sock over inner dockerd

The start_docker.py script was checking /var/run/docker.sock first,
which inside CI containers is an inner dockerd (v29.5.3) with data
root on the container's 38G overlay (often 100% full). When
_get_docker_free_bytes() returned 0 (data root path not accessible
from inside container), the script assumed it was the host Docker
with plenty of space and returned True immediately — without trying
the host's rootless Docker socket at /run/host-docker.sock.

Fix: try /run/host-docker.sock FIRST (before /var/run/docker.sock).
The host socket is mounted by the gitea runner config and has access
to the host's full filesystem (455G). Only trust free_bytes == 0
(= data root not accessible from container) for /run/host-docker.sock,
since the host's root dir is genuinely outside the container. For
other sockets (inner dockerd), free_bytes == 0 means the path doesn't
exist inside the container — don't trust it.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

src/devx/molecule/start_docker.py

@@ -157,60 +157,41 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
 
     Tries sockets in this order, preferring ones with enough disk space:
 
-    1. Host socket (``/var/run/docker.sock``) — if it has ≥ ``MIN_FREE_BYTES``
+    1. Host rootless socket (``/run/host-docker.sock``) — mounted by the
-       free space, use it immediately.
+       gitea runner config, has access to the host's full filesystem
-    2. Rootless sockets (``/run/user/*/docker.sock``) — if the host socket
+       (e.g. 455 GB).  Preferred over the inner dockerd.
-       has insufficient space, try rootless sockets which may have access
+    2. Default socket (``/var/run/docker.sock``) — may be an inner dockerd
-       to the host's full filesystem.
+       started by the CI image (v29.5.3) with data root on the container's
-    3. Local ``dockerd`` with vfs storage driver — last resort.
+       limited overlay (e.g. 38 GB, often 100 % full).
+    3. Other rootless sockets (``/run/user/*/docker.sock``).
+    4. Local ``dockerd`` with vfs storage driver — last resort.
 
     Returns ``True`` if Docker is ready, ``False`` if it failed to
     start within the timeout.
     """
-    # Point Docker CLI and Python library to the socket explicitly
-    os.environ["DOCKER_HOST"] = f"unix://{DOCKER_SOCK}"
-
     # Diagnose socket state
     click.echo("--- Docker socket diagnostics ---")
     _diagnose_socket()
     click.echo("--- End diagnostics ---")
 
-    # Check if host Docker is already available
+    # Collect candidate sockets in priority order.
-    if is_docker_ready():
+    # The host's rootless Docker socket (mounted at /run/host-docker.sock
-        free_bytes = _get_docker_free_bytes()
+    # by the gitea runner config) is preferred — it has access to the
-        free_gb = free_bytes / 1024**3
+    # host's full filesystem instead of the container's limited overlay.
-        click.echo(f"Docker daemon already running (free space: {free_gb:.1f} GB)")
-        if free_bytes >= MIN_FREE_BYTES:
-            return True
-        # If free_bytes is 0, the Docker root dir is on the host filesystem
-        # (not accessible from inside the container). The host Docker has
-        # access to the full 455G disk — always use it in that case.
-        if free_bytes == 0:
-            click.echo("Host Docker root dir not accessible from container, using host Docker")
-            return True
-        click.echo(
-            f"Host Docker has only {free_gb:.1f} GB free "
-            f"(need ≥ {MIN_FREE_BYTES / 1024**3:.0f} GB), trying rootless sockets..."
-        )
-    else:
-        click.echo("Host Docker not available, trying rootless sockets...")
-
-    # Collect candidate rootless sockets
     candidates: list[str] = []
-    # The host Docker socket is mounted by the gitea_runner config.
-    # This is the preferred candidate — it has access to the host's
-    # full filesystem instead of the container's limited overlay.
     if os.path.exists(HOST_DOCKER_SOCK):
         candidates.append(HOST_DOCKER_SOCK)
+    if os.path.exists(DOCKER_SOCK):
+        candidates.append(DOCKER_SOCK)
     if os.path.exists(ROOTLESS_SOCK):
         candidates.append(ROOTLESS_SOCK)
     for sock in sorted(glob.glob("/run/user/*/docker.sock")):
         if sock not in candidates:
             candidates.append(sock)
 
-    # Try each rootless socket — prefer one with enough free space
+    # Try each candidate socket — prefer one with enough free space
     for sock in candidates:
-        click.echo(f"Trying rootless socket: {sock}")
+        click.echo(f"Trying socket: {sock}")
         if not _try_socket(sock):
             continue
         free_bytes = _get_docker_free_bytes()
@@ -219,10 +200,20 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
         if free_bytes >= MIN_FREE_BYTES:
             click.echo(_("Docker daemon already running"))
             return True
+        # If free_bytes is 0, the Docker root dir is on the host filesystem
+        # (not accessible from inside the container). This is expected for
+        # the host's rootless Docker — it has the full host disk.
+        # Only trust this for /run/host-docker.sock (known host socket).
+        # For other sockets (e.g. inner dockerd), free_bytes == 0 means
+        # the data root path doesn't exist inside the container — the
+        # inner dockerd may be using the container's full overlay.
+        if free_bytes == 0 and sock == HOST_DOCKER_SOCK:
+            click.echo("Host rootless Docker root dir not accessible from container, using it")
+            return True
         click.echo(f"  Insufficient space ({free_gb:.1f} GB), trying next...")
 
-    # If we found a working rootless socket but with low space, use it
+    # If we found a working socket but with low space, use the last
-    # as a fallback (better than nothing).
+    # one as a fallback (better than nothing).
     for sock in reversed(candidates):
         if _try_socket(sock):
             click.echo(f"Using low-space fallback: {sock}")

tests/unit/test_start_docker.py

@@ -7,6 +7,8 @@ from click.testing import CliRunner
 
 from devx.molecule.start_docker import (
     DOCKER_SOCK,
+    HOST_DOCKER_SOCK,
+    ROOTLESS_SOCK,
     _diagnose_socket,
     _get_docker_free_bytes,
     _try_socket,
@@ -124,91 +126,192 @@ class TestDiagnoseSocket:
         mock_exists.assert_called_with(DOCKER_SOCK)
 
 
+def _exists_map(paths: set[str]) -> MagicMock:
+    """Return a mock os.path.exists that returns True only for *paths*."""
+    return MagicMock(side_effect=lambda p: p in paths)
+
+
 class TestStartDockerDaemon:
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     def test_host_socket_available_with_space(
-        self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
     ) -> None:
-        """Should return immediately if host Docker has enough space."""
+        """Should return immediately if /var/run/docker.sock has enough space."""
         assert start_docker_daemon(timeout=5) is True
-        mock_ready.assert_called_once()
         mock_diag.assert_called_once()
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
-    def test_host_socket_inaccessible_root_uses_host(
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({HOST_DOCKER_SOCK}))
-        self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    def test_host_rootless_inaccessible_root_uses_host(
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
     ) -> None:
-        """Should use host Docker when root dir is inaccessible (free=0)."""
+        """Should use host rootless Docker when root dir is inaccessible (free=0)."""
         assert start_docker_daemon(timeout=5) is True
-        mock_ready.assert_called_once()
 
     @patch("devx.molecule.start_docker._diagnose_socket")
-    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=5 * 1024**3)
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
-    def test_host_socket_low_space_tries_rootless(
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
-        self, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    @patch("devx.molecule.start_docker.subprocess.Popen")
+    @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
+    @patch("devx.molecule.start_docker.time.sleep")
+    def test_inner_dockerd_free_zero_starts_local(
+        self,
+        mock_sleep: MagicMock,
+        mock_ntf: MagicMock,
+        mock_popen: MagicMock,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
     ) -> None:
-        """Should try rootless sockets if host Docker has low space."""
+        """Inner dockerd with free=0 (not host socket) should NOT be trusted — start local."""
-        with (
+        mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
-            patch("devx.molecule.start_docker.glob.glob", return_value=[]),
+        # is_docker_ready: first check (inner dockerd) True, then local daemon checks
-            patch("devx.molecule.start_docker.os.path.exists", return_value=False),
+        mock_ready.side_effect = [True, False, False, False, False, True]
-            patch("devx.molecule.start_docker.subprocess.run"),
+        assert start_docker_daemon(timeout=5) is True
-            patch("devx.molecule.start_docker.subprocess.Popen"),
+        mock_popen.assert_called_once()
-            patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile"),
-            patch("devx.molecule.start_docker.time.sleep"),
-        ):
-            # Host ready but low space, no rootless sockets, starts local dockerd
-            assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
-    def test_rootless_socket_with_space(
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
-        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+    @patch("devx.molecule.start_docker.subprocess.Popen")
+    @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
+    @patch("devx.molecule.start_docker.time.sleep")
+    def test_host_socket_low_space_starts_local(
+        self,
+        mock_sleep: MagicMock,
+        mock_ntf: MagicMock,
+        mock_popen: MagicMock,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
     ) -> None:
-        """Should use rootless socket if host has low space and rootless has enough."""
+        """Should start local dockerd if host Docker has low space and no rootless sockets."""
-        # Host ready but low space, rootless ready with enough space
+        mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
+        mock_ready.side_effect = [True, False, False, False, False, True]
+        mock_free.return_value = 5 * 1024**3
+        assert start_docker_daemon(timeout=5) is True
+        mock_popen.assert_called_once()
+
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    def test_host_rootless_preferred_over_inner_dockerd(
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+    ) -> None:
+        """Should prefer /run/host-docker.sock over /var/run/docker.sock."""
+        # host rootless ready with space, inner dockerd never tried
+        mock_ready.side_effect = [True]
+        mock_free.side_effect = [200 * 1024**3]
+        assert start_docker_daemon(timeout=5) is True
+        # DOCKER_HOST should be set to host socket
+        assert os.environ.get("DOCKER_HOST") == f"unix://{HOST_DOCKER_SOCK}"
+
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    def test_host_rootless_not_ready_falls_to_inner(
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+    ) -> None:
+        """Should fall through to inner dockerd if host rootless socket is not ready."""
+        mock_ready.side_effect = [False, True]
+        assert start_docker_daemon(timeout=5) is True
+        assert os.environ.get("DOCKER_HOST") == f"unix://{DOCKER_SOCK}"
+
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
+    @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
+    @patch(
+        "devx.molecule.start_docker.os.path.exists",
+        side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK, ROOTLESS_SOCK, "/run/user/999/docker.sock"}),
+    )
+    @patch("devx.molecule.start_docker.glob.glob", return_value=["/run/user/999/docker.sock"])
+    def test_glob_finds_extra_rootless_socket(
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+    ) -> None:
+        """Should include rootless sockets found via glob scan."""
+        assert start_docker_daemon(timeout=5) is True
+
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    def test_inner_dockerd_fallback_when_host_rootless_low_space(
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+    ) -> None:
+        """Should fall back to inner dockerd if host rootless has low space."""
+        # host rootless ready low space, inner dockerd ready with space
         mock_ready.side_effect = [True, True]
         mock_free.side_effect = [5 * 1024**3, 200 * 1024**3]
-        own_sock = f"/run/user/{os.getuid()}/docker.sock"
+        assert start_docker_daemon(timeout=5) is True
-        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]):
+        assert os.environ.get("DOCKER_HOST") == f"unix://{DOCKER_SOCK}"
-            assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
-    def test_alt_rootless_socket_found(
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
-        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
-    ) -> None:
-        """Should find rootless socket at a different UID via glob scan."""
-        # Host not ready, own rootless not ready, alt rootless ready with space
-        mock_ready.side_effect = [False, False, True]
-        mock_free.side_effect = [200 * 1024**3]
-        own_sock = f"/run/user/{os.getuid()}/docker.sock"
-        alt_sock = "/run/user/999/docker.sock"
-        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]):
-            assert start_docker_daemon(timeout=5) is True
-
-    @patch("devx.molecule.start_docker._diagnose_socket")
-    @patch("devx.molecule.start_docker._get_docker_free_bytes")
-    @patch("devx.molecule.start_docker.is_docker_ready")
-    @patch("devx.molecule.start_docker.os.path.exists", return_value=True)
     def test_all_sockets_low_space_falls_back(
-        self, mock_exists: MagicMock, mock_ready: MagicMock, mock_free: MagicMock, mock_diag: MagicMock
+        self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
     ) -> None:
         """Should fall back to a low-space socket if no better option exists."""
-        # Host ready low space, rootless ready low space, falls back to rootless
         mock_ready.side_effect = [True, True, True, True]
         mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3]
-        own_sock = f"/run/user/{os.getuid()}/docker.sock"
+        assert start_docker_daemon(timeout=5) is True
-        with patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock]):
-            assert start_docker_daemon(timeout=5) is True
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
@@ -232,7 +335,7 @@ class TestStartDockerDaemon:
         mock_glob: MagicMock,
     ) -> None:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
-        # Host fails, rootless doesn't exist, local daemon starts
+        # No sockets exist, local daemon starts
         mock_ready.side_effect = [False, False, False, False, True]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
@@ -317,18 +420,22 @@ class TestStartDockerDaemon:
         mock_glob: MagicMock,
     ) -> None:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
-        # Host fails, rootless doesn't exist, local ready on first loop check
+        # No sockets exist, local ready on second loop check
         mock_ready.side_effect = [False, False, True]
         assert start_docker_daemon(timeout=5) is True
         assert mock_popen.call_count == 1
-        assert mock_sleep.call_count == 1
+        assert mock_sleep.call_count == 2
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
     @patch("devx.molecule.start_docker.os.environ")
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     def test_sets_docker_host(
         self,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
         mock_ready: MagicMock,
         mock_environ: MagicMock,
         mock_free: MagicMock,