Public Access
DEVX-155: fix: kill dockerd by PID when pkill fails, use /dev/shm for alive daemon
Add pgrep diagnostics before/after pkill to identify lingering dockerd processes. If pkill fails and pgrep still finds dockerd, kill by PID directly via os.kill. When inner dockerd can't be killed (still alive), use /dev/shm as data root since the overlay is still full. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
3ac3e613e9
commit
cdbee0a317
@@ -224,8 +224,18 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
|
||||
# overlay (38G, often 100% full). Killing it frees up the
|
||||
# socket and any space used by its containers/volumes.
|
||||
# Use SIGKILL (-9) since the inner dockerd may not respond to SIGTERM.
|
||||
# Try multiple patterns to match different dockerd invocations.
|
||||
for pattern in ["dockerd", "dockerd-entrypoint.sh"]:
|
||||
# Try multiple approaches to ensure the inner dockerd is killed.
|
||||
with contextlib.suppress(Exception):
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
["pgrep", "-af", "dockerd"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
click.echo(f" dockerd processes before kill: {result.stdout.strip()}")
|
||||
|
||||
for pattern in ["dockerd", "dockerd-entrypoint.sh", "containerd"]:
|
||||
with contextlib.suppress(Exception):
|
||||
subprocess.run( # nosec B603 B607
|
||||
["pkill", "-9", "-f", pattern],
|
||||
@@ -234,6 +244,24 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
# Check if dockerd processes are still alive
|
||||
with contextlib.suppress(Exception):
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
["pgrep", "-af", "dockerd"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
click.echo(f" dockerd processes after kill: {result.stdout.strip()}")
|
||||
# Try killing by PID directly
|
||||
for pid_str in result.stdout.split("\n"):
|
||||
pid = pid_str.split()[0] if pid_str.strip() else ""
|
||||
if pid:
|
||||
with contextlib.suppress(Exception):
|
||||
os.kill(int(pid), 9)
|
||||
time.sleep(2)
|
||||
|
||||
# Verify the inner dockerd is actually dead. If we can still
|
||||
# connect to /var/run/docker.sock, the old daemon is still running
|
||||
# and we need to use a different socket path.
|
||||
@@ -265,10 +293,11 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
# Use a fresh data root on the container's overlay.
|
||||
# /dev/shm is a small tmpfs (16G) — too small for images.
|
||||
# The container's overlay (38G) has more space after cleanup.
|
||||
docker_data_root = "/tmp/docker-data" # nosec B108
|
||||
# Use a fresh data root. If the inner dockerd is dead, use the
|
||||
# container's overlay (38G, with freed space). If the inner
|
||||
# dockerd is still alive, use /dev/shm (16G tmpfs) — the overlay
|
||||
# is still full because the inner dockerd's data can't be cleaned.
|
||||
docker_data_root = "/dev/shm/docker" if old_daemon_alive else "/tmp/docker-data" # nosec B108
|
||||
|
||||
# Remove stale socket if present
|
||||
with contextlib.suppress(OSError):
|
||||
|
||||
Reference in New Issue
Block a user