From 4610e5e5c8c8ace31041a955f69ceda18f7c9daa Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Sat, 8 Aug 2026 14:47:27 +0200 Subject: [PATCH] fix: exclude CI job containers from prune to prevent self-destruction The prune service and healthcheck force-remove ALL containers via "docker ps -aq | xargs -r docker rm -f", but this includes the CI job container itself (named GITEA-ACTIONS-TASK-*). Removing it causes "RWLayer of container is unexpectedly nil" errors and kills the active CI job. Exclude containers whose name starts with GITEA-ACTIONS-TASK from the force-remove step. Use "docker ps -a --format" with name filtering instead of "docker ps -aq". Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../gitea_runner/molecule/template-content/verify.yml | 2 ++ .../roles/gitea_runner/templates/docker-prune.service.j2 | 6 ++++-- .../roles/gitea_runner/templates/runner-healthcheck.sh.j2 | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index cff0668..5cea828 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -48,6 +48,7 @@ that: - "'Type=oneshot' in prune_service.content | b64decode" - "'docker rm -f' in prune_service.content | b64decode" + - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode" - "'docker system prune -af' in prune_service.content | b64decode" - "'docker network prune' in prune_service.content | b64decode" - "'docker builder prune' in prune_service.content | b64decode" @@ -105,6 +106,7 @@ - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode" - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" - "'docker rm -f' in healthcheck_script.content | b64decode" + - "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode" - "'docker system prune -af' in healthcheck_script.content | b64decode" - "'docker network prune' in healthcheck_script.content | b64decode" - "'status=removing' in healthcheck_script.content | b64decode" diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index 22f5a1c..208c8c0 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -5,11 +5,13 @@ Description=Docker prune for Gitea runner resources Type=oneshot Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} -# Force-remove ALL containers (including running ones) left behind by failed +# Force-remove stale containers (including running ones) left behind by failed # molecule tests. "docker container prune -f" only removes stopped containers, # so running containers from crashed/interrupted CI jobs accumulate indefinitely, # consuming disk and memory. We stop+rm everything first, then prune the rest. -ExecStart=/bin/sh -c 'docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true' +# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing +# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors. +ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes ExecStart=/usr/bin/docker network prune -f ExecStart=/usr/bin/docker builder prune -f diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index d8f624b..1b47bfa 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -159,10 +159,14 @@ fi disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources" - # Force-remove ALL containers (including running ones from failed molecule tests). + # Force-remove stale containers (including running ones from failed molecule tests). # "docker container prune -f" only removes stopped containers, so running # containers from crashed CI jobs accumulate and consume disk/memory. - docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true + # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). + docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ + | grep -v 'GITEA-ACTIONS-TASK' \ + | awk '{print $1}' \ + | xargs -r docker rm -f 2>/dev/null || true docker system prune -af --filter "until=1h" --volumes || true docker network prune -f || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')