DEVX-155: fix: start local dockerd instead of using low-space inner dockerd

When no socket has sufficient space, don't fall back to the low-space
inner dockerd (which will fail on image pulls). Instead, kill the inner
dockerd, clean up its data root to free space, and start a local
dockerd on /dev/shm with vfs storage driver.

Also adds pkill of the inner dockerd and cleanup of its data root
(overlay2, image, volumes, containers) before starting the local
dockerd, to free up the 2.4GB used by the inner dockerd's data.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

src/devx/molecule/start_docker.py

@@ -212,15 +212,33 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
             return True
         click.echo(f"  Insufficient space ({free_gb:.1f} GB), trying next...")
 
-    # If we found a working socket but with low space, use the last
+    # No socket with sufficient space found.
-    # one as a fallback (better than nothing).
+    # Don't fall back to the low-space inner dockerd — it will fail
-    for sock in reversed(candidates):
+    # on image pulls.  Instead, start a local dockerd on /dev/shm.
-        if _try_socket(sock):
-            click.echo(f"Using low-space fallback: {sock}")
-            return True
-
     click.echo(_("Host Docker not available, starting local dockerd..."))
 
+    # Kill the inner dockerd (started by the CI image) to free its
+    # data root and socket.  The inner dockerd uses the container's
+    # overlay (38G, often 100% full).  Killing it frees up the
+    # socket and any space used by its containers/volumes.
+    with contextlib.suppress(Exception):
+        subprocess.run(  # nosec B603 B607
+            ["pkill", "-f", "dockerd.*--host fd://"],
+            check=False,
+            timeout=5,
+        )
+    time.sleep(2)
+
+    # Clean up the inner dockerd's data root to free space
+    inner_data_root = "/home/grm-ci-runner-*/.local/share/docker"
+    rm_paths = " ".join(f"{inner_data_root}/{d}" for d in ("overlay2", "image", "volumes", "containers"))
+    with contextlib.suppress(Exception):
+        subprocess.run(  # nosec B603 B607
+            ["sh", "-c", f"rm -rf {rm_paths}"],
+            check=False,
+            timeout=30,
+        )
+
     # The existing dockerd (started by the CI image) uses the default data
     # root on the CI container's overlay (38G, often 100% full). We can't
     # kill it (different PID namespace) or bind to the same socket. Use a

tests/unit/test_start_docker.py

@@ -170,6 +170,7 @@ class TestStartDockerDaemon:
     @patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
     @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    @patch("devx.molecule.start_docker.subprocess.run")
     @patch("devx.molecule.start_docker.subprocess.Popen")
     @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
     @patch("devx.molecule.start_docker.time.sleep")
@@ -178,6 +179,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ntf: MagicMock,
         mock_popen: MagicMock,
+        mock_run: MagicMock,
         mock_glob: MagicMock,
         mock_exists: MagicMock,
         mock_ready: MagicMock,
@@ -196,6 +198,7 @@ class TestStartDockerDaemon:
     @patch("devx.molecule.start_docker.is_docker_ready")
     @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    @patch("devx.molecule.start_docker.subprocess.run")
     @patch("devx.molecule.start_docker.subprocess.Popen")
     @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
     @patch("devx.molecule.start_docker.time.sleep")
@@ -204,6 +207,7 @@ class TestStartDockerDaemon:
         mock_sleep: MagicMock,
         mock_ntf: MagicMock,
         mock_popen: MagicMock,
+        mock_run: MagicMock,
         mock_glob: MagicMock,
         mock_exists: MagicMock,
         mock_ready: MagicMock,
@@ -300,18 +304,28 @@ class TestStartDockerDaemon:
     @patch("devx.molecule.start_docker.is_docker_ready")
     @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
-    def test_all_sockets_low_space_falls_back(
+    @patch("devx.molecule.start_docker.subprocess.run")
+    @patch("devx.molecule.start_docker.subprocess.Popen")
+    @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
+    @patch("devx.molecule.start_docker.time.sleep")
+    def test_all_sockets_low_space_starts_local(
         self,
+        mock_sleep: MagicMock,
+        mock_ntf: MagicMock,
+        mock_popen: MagicMock,
+        mock_run: MagicMock,
         mock_glob: MagicMock,
         mock_exists: MagicMock,
         mock_ready: MagicMock,
         mock_free: MagicMock,
         mock_diag: MagicMock,
     ) -> None:
-        """Should fall back to a low-space socket if no better option exists."""
+        """Should start local dockerd if all sockets have low space."""
-        mock_ready.side_effect = [True, True, True, True]
+        mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
-        mock_free.side_effect = [5 * 1024**3, 5 * 1024**3, 5 * 1024**3, 5 * 1024**3]
+        mock_ready.side_effect = [True, False, False, False, False, True]
+        mock_free.return_value = 5 * 1024**3
         assert start_docker_daemon(timeout=5) is True
+        mock_popen.assert_called_once()
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
@@ -370,7 +384,7 @@ class TestStartDockerDaemon:
         with patch("builtins.open", mock_open(read_data="dockerd error log")):
             assert start_docker_daemon(timeout=3) is False
         mock_popen.assert_called_once()
-        assert mock_sleep.call_count == 3
+        assert mock_sleep.call_count == 4  # 1 after pkill + 3 timeout retries
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
@@ -424,7 +438,7 @@ class TestStartDockerDaemon:
         mock_ready.side_effect = [False, False, True]
         assert start_docker_daemon(timeout=5) is True
         assert mock_popen.call_count == 1
-        assert mock_sleep.call_count == 2
+        assert mock_sleep.call_count == 3  # 1 after pkill + 2 loop retries
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)