DEVX-155: fix: kill dockerd by PID when pkill fails, use /dev/shm for alive daemon

Add pgrep diagnostics before/after pkill to identify lingering dockerd
processes. If pkill fails and pgrep still finds dockerd, kill by PID
directly via os.kill. When inner dockerd can't be killed (still alive),
use /dev/shm as data root since the overlay is still full.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/unit/test_start_docker.py

@@ -18,6 +18,21 @@ from devx.molecule.start_docker import (
 )
 
 
+def _pgrep_empty() -> MagicMock:
+    """Mock for pgrep returning no dockerd processes."""
+    return MagicMock(stdout="", returncode=1)
+
+
+def _docker_info_alive() -> MagicMock:
+    """Mock for docker info showing daemon alive."""
+    return MagicMock(returncode=0)
+
+
+def _docker_info_dead() -> MagicMock:
+    """Mock for docker info showing daemon dead."""
+    return MagicMock(returncode=1)
+
+
 class TestIsDockerReady:
     @patch("devx.molecule.start_docker.subprocess.run")
     def test_ready(self, mock_run: MagicMock) -> None:
@@ -190,6 +205,16 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         # is_docker_ready: first check (inner dockerd) True, then local daemon checks
         mock_ready.side_effect = [True, False, False, False, False, True]
+        # subprocess.run calls: pgrep(before), pkill x3, pgrep(after), docker info, rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
 
@@ -218,11 +243,14 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
-        # mock_run: pkill calls (x2), docker info check (returncode=1 = dead), rm call
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
         mock_run.side_effect = [
+            _pgrep_empty(),
             MagicMock(),
             MagicMock(),
-            MagicMock(returncode=1),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
             MagicMock(),
         ]
         assert start_docker_daemon(timeout=5) is True
@@ -253,17 +281,61 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
-        # mock_run: pkill calls (x2), docker info check (returncode=0 = alive), rm call
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(alive), rm
         mock_run.side_effect = [
+            _pgrep_empty(),
             MagicMock(),
             MagicMock(),
-            MagicMock(returncode=0),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_alive(),
             MagicMock(),
         ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
         assert os.environ.get("DOCKER_HOST") == "unix:///dev/shm/docker.sock"
 
+    @patch("devx.molecule.start_docker.os.kill")
+    @patch("devx.molecule.start_docker._diagnose_socket")
+    @patch("devx.molecule.start_docker._get_docker_free_bytes")
+    @patch("devx.molecule.start_docker.is_docker_ready")
+    @patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
+    @patch("devx.molecule.start_docker.glob.glob", return_value=[])
+    @patch("devx.molecule.start_docker.subprocess.run")
+    @patch("devx.molecule.start_docker.subprocess.Popen")
+    @patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
+    @patch("devx.molecule.start_docker.time.sleep")
+    def test_inner_dockerd_killed_by_pid(
+        self,
+        mock_sleep: MagicMock,
+        mock_ntf: MagicMock,
+        mock_popen: MagicMock,
+        mock_run: MagicMock,
+        mock_glob: MagicMock,
+        mock_exists: MagicMock,
+        mock_ready: MagicMock,
+        mock_free: MagicMock,
+        mock_diag: MagicMock,
+        mock_kill: MagicMock,
+    ) -> None:
+        """Should kill dockerd by PID when pkill fails and pgrep finds processes."""
+        mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
+        mock_ready.side_effect = [True, False, False, False, False, True]
+        mock_free.return_value = 5 * 1024**3
+        # pgrep(before) empty, pkill x3, pgrep(after) finds PID 12345, docker info(dead), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(stdout="12345 /usr/bin/dockererd\n", returncode=0),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
+        assert start_docker_daemon(timeout=5) is True
+        mock_popen.assert_called_once()
+        mock_kill.assert_called_once_with(12345, 9)
+
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes")
     @patch("devx.molecule.start_docker.is_docker_ready")
@@ -367,6 +439,16 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         mock_ready.side_effect = [True, False, False, False, False, True]
         mock_free.return_value = 5 * 1024**3
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            _docker_info_dead(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
 
@@ -394,6 +476,15 @@ class TestStartDockerDaemon:
         mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
         # No sockets exist, local daemon starts
         mock_ready.side_effect = [False, False, False, False, True]
+        # subprocess.run: pgrep(before), pkill x3, pgrep(after), rm
+        mock_run.side_effect = [
+            _pgrep_empty(),
+            MagicMock(),
+            MagicMock(),
+            MagicMock(),
+            _pgrep_empty(),
+            MagicMock(),
+        ]
         assert start_docker_daemon(timeout=5) is True
         mock_popen.assert_called_once()
         popen_args = mock_popen.call_args.args[0]
@@ -427,7 +518,7 @@ class TestStartDockerDaemon:
         with patch("builtins.open", mock_open(read_data="dockerd error log")):
             assert start_docker_daemon(timeout=3) is False
         mock_popen.assert_called_once()
-        assert mock_sleep.call_count == 4  # 1 after pkill + 3 timeout retries
+        assert mock_sleep.call_count == 5  # 1 after pkill + 1 after pgrep + 3 timeout retries
 
     @patch("devx.molecule.start_docker.glob.glob", return_value=[])
     @patch("devx.molecule.start_docker._diagnose_socket")
@@ -481,7 +572,7 @@ class TestStartDockerDaemon:
         mock_ready.side_effect = [False, False, True]
         assert start_docker_daemon(timeout=5) is True
         assert mock_popen.call_count == 1
-        assert mock_sleep.call_count == 3  # 1 after pkill + 2 loop retries
+        assert mock_sleep.call_count == 4  # 1 after pkill + 1 after pgrep + 2 loop retries
 
     @patch("devx.molecule.start_docker._diagnose_socket")
     @patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)