oblachno-oss/grm
3
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 21 Wiki Activity

fix: exclude CI job containers from prune to prevent self-destruction

Browse Source 
The prune service and healthcheck force-remove ALL containers via
"docker ps -aq | xargs -r docker rm -f", but this includes the CI
job container itself (named GITEA-ACTIONS-TASK-*). Removing it causes
"RWLayer of container is unexpectedly nil" errors and kills the
active CI job.

Exclude containers whose name starts with GITEA-ACTIONS-TASK from
the force-remove step. Use "docker ps -a --format" with name filtering
instead of "docker ps -aq".

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Emil Simeonov
2026-08-09 01:07:38 +02:00
co-authored by Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent a9e22819df
commit 4610e5e5c8
3 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/roles/gitea_runner/molecule/template-content/verify.yml
+2
View File
@@ -48,6 +48,7 @@
that:
- "'Type=oneshot' in prune_service.content | b64decode"
- "'docker rm -f' in prune_service.content | b64decode"
- "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode"
- "'docker system prune -af' in prune_service.content | b64decode"
- "'docker network prune' in prune_service.content | b64decode"
- "'docker builder prune' in prune_service.content | b64decode"
@@ -105,6 +106,7 @@
- "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
- "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
- "'docker rm -f' in healthcheck_script.content | b64decode"
- "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode"
- "'docker system prune -af' in healthcheck_script.content | b64decode"
- "'docker network prune' in healthcheck_script.content | b64decode"
- "'status=removing' in healthcheck_script.content | b64decode"
ansible/roles/gitea_runner/templates/docker-prune.service.j2
+4 -2
View File
@@ -5,11 +5,13 @@ Description=Docker prune for Gitea runner resources
Type=oneshot
Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
# Force-remove ALL containers (including running ones) left behind by failed
# Force-remove stale containers (including running ones) left behind by failed
# molecule tests. "docker container prune -f" only removes stopped containers,
# so running containers from crashed/interrupted CI jobs accumulate indefinitely,
# consuming disk and memory. We stop+rm everything first, then prune the rest.
ExecStart=/bin/sh -c 'docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true'
# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing
# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors.
ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true'
ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes
ExecStart=/usr/bin/docker network prune -f
ExecStart=/usr/bin/docker builder prune -f
ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2
+6 -2
View File
@@ -159,10 +159,14 @@ fi
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
# Force-remove ALL containers (including running ones from failed molecule tests).
# Force-remove stale containers (including running ones from failed molecule tests).
# "docker container prune -f" only removes stopped containers, so running
# containers from crashed CI jobs accumulate and consume disk/memory.
docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true
# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
| grep -v 'GITEA-ACTIONS-TASK' \
| awk '{print $1}' \
| xargs -r docker rm -f 2>/dev/null || true
docker system prune -af --filter "until=1h" --volumes || true
docker network prune -f || true
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')