DEVX-155: fix: kill dockerd by PID when pkill fails, use /dev/shm for alive daemon

Add pgrep diagnostics before/after pkill to identify lingering dockerd
processes. If pkill fails and pgrep still finds dockerd, kill by PID
directly via os.kill. When inner dockerd can't be killed (still alive),
use /dev/shm as data root since the overlay is still full.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

src/devx/molecule/start_docker.py

@@ -224,8 +224,18 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
     # overlay (38G, often 100% full).  Killing it frees up the
     # socket and any space used by its containers/volumes.
     # Use SIGKILL (-9) since the inner dockerd may not respond to SIGTERM.
-    # Try multiple patterns to match different dockerd invocations.
-    for pattern in ["dockerd", "dockerd-entrypoint.sh"]:
+    # Try multiple approaches to ensure the inner dockerd is killed.
+    with contextlib.suppress(Exception):
+        result = subprocess.run(  # nosec B603 B607
+            ["pgrep", "-af", "dockerd"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.stdout.strip():
+            click.echo(f"  dockerd processes before kill: {result.stdout.strip()}")
+
+    for pattern in ["dockerd", "dockerd-entrypoint.sh", "containerd"]:
         with contextlib.suppress(Exception):
             subprocess.run(  # nosec B603 B607
                 ["pkill", "-9", "-f", pattern],
@@ -234,6 +244,24 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
             )
     time.sleep(3)
 
+    # Check if dockerd processes are still alive
+    with contextlib.suppress(Exception):
+        result = subprocess.run(  # nosec B603 B607
+            ["pgrep", "-af", "dockerd"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.stdout.strip():
+            click.echo(f"  dockerd processes after kill: {result.stdout.strip()}")
+            # Try killing by PID directly
+            for pid_str in result.stdout.split("\n"):
+                pid = pid_str.split()[0] if pid_str.strip() else ""
+                if pid:
+                    with contextlib.suppress(Exception):
+                        os.kill(int(pid), 9)
+            time.sleep(2)
+
     # Verify the inner dockerd is actually dead.  If we can still
     # connect to /var/run/docker.sock, the old daemon is still running
     # and we need to use a different socket path.
@@ -265,10 +293,11 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
             timeout=30,
         )
 
-    # Use a fresh data root on the container's overlay.
-    # /dev/shm is a small tmpfs (16G) — too small for images.
-    # The container's overlay (38G) has more space after cleanup.
-    docker_data_root = "/tmp/docker-data"  # nosec B108
+    # Use a fresh data root.  If the inner dockerd is dead, use the
+    # container's overlay (38G, with freed space).  If the inner
+    # dockerd is still alive, use /dev/shm (16G tmpfs) — the overlay
+    # is still full because the inner dockerd's data can't be cleaned.
+    docker_data_root = "/dev/shm/docker" if old_daemon_alive else "/tmp/docker-data"  # nosec B108
 
     # Remove stale socket if present
     with contextlib.suppress(OSError):

tests/unit/test_start_docker.py

@@ -18,6 +18,21 @@ from devx.molecule.start_docker import (
 )
 
 
+def _pgrep_empty() -> MagicMock:
+    """Mock for pgrep returning no dockerd processes."""
+    return MagicMock(stdout="", returncode=1)
+
+
+def _docker_info_alive() -> MagicMock:
+    """Mock for docker info showing daemon alive."""
+    return MagicMock(returncode=0)
+
+
+def _docker_info_dead() -> MagicMock:
+    """Mock for docker info showing daemon dead."""
+    return MagicMock(returncode=1)
+
+
 class TestIsDockerReady:
     @patch("devx.molecule.start_docker.subprocess.run")
     def test_ready(self, mock_run: MagicMock) -> None:
@@ -190,6 +205,16 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         # is_docker_ready: first check (inner dockerd) True, then local daemon checks
         mock_ready.side_effect = [True, False, False, False, False, True]
+        # subprocess.run calls: pgrep(before), pkill x3, pgrep(after), docker info, rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
 
@@ -218,11 +243,14 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
-        # mock_run: pkill calls (x2), docker info check (returncode=1 = dead), rm call
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
         mock_run.side_effect = [
+            _pgrep_empty(),
             MagicMock(),
             MagicMock(),
-            MagicMock(returncode=1),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
             MagicMock(),
         ]
         assert start_docker_daemon(timeout=5) is True
@@ -253,17 +281,61 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
-        # mock_run: pkill calls (x2), docker info check (returncode=0 = alive), rm call
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(alive), rm
         mock_run.side_effect = [
+            _pgrep_empty(),
             MagicMock(),
             MagicMock(),
-            MagicMock(returncode=0),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_alive(),
             MagicMock(),
         ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
         assert os.environ.get("DOCKER_HOST") == "unix:///dev/shm/docker.sock"
 
+    @patch("devx.molecule.start_docker.os.kill")
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    @patch("devx.molecule.start_docker.subprocess.run")
+    @patch("devx.molecule.start_docker.subprocess.Popen")
+    @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
+    @patch("devx.molecule.start_docker.time.sleep")
+    def test_inner_dockerd_killed_by_pid(
+        self,
+        mock_sleep: MagicMock,
+        mock_ntf: MagicMock,
+        mock_popen: MagicMock,
+        mock_run: MagicMock,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+        mock_kill: MagicMock,
+    ) -> None:
+        """Should kill dockerd by PID when pkill fails and pgrep finds processes."""
+        mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
+        mock_ready.side_effect = [True, False, False, False, False, True]
+        mock_free.return_value = 5 * 1024**3
+        # pgrep(before) empty, pkill x3, pgrep(after) finds PID 12345, docker info(dead), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(stdout="12345 /usr/bin/dockererd\n", returncode=0),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
+        assert start_docker_daemon(timeout=5) is True
+        mock_popen.assert_called_once()
+        mock_kill.assert_called_once_with(12345, 9)
+
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
@@ -367,6 +439,16 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
 
@@ -394,6 +476,15 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         # No sockets exist, local daemon starts
         mock_ready.side_effect = [False, False, False, False, True]
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
         popen_args = mock_popen.call_args.args[0]
@@ -427,7 +518,7 @@ class TestStartDockerDaemon:
         with patch("builtins.open", mock_open(read_data="dockerd error log")):
             assert start_docker_daemon(timeout=3) is False
         mock_popen.assert_called_once()
-        assert mock_sleep.call_count == 4  # 1 after pkill + 3 timeout retries
+        assert mock_sleep.call_count == 5  # 1 after pkill + 1 after pgrep + 3 timeout retries
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
@@ -481,7 +572,7 @@ class TestStartDockerDaemon:
         mock_ready.side_effect = [False, False, True]
         assert start_docker_daemon(timeout=5) is True
         assert mock_popen.call_count == 1
-        assert mock_sleep.call_count == 3  # 1 after pkill + 2 loop retries
+        assert mock_sleep.call_count == 4  # 1 after pkill + 1 after pgrep + 2 loop retries
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)