diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index cff0668..5cea828 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -48,6 +48,7 @@ that: - "'Type=oneshot' in prune_service.content | b64decode" - "'docker rm -f' in prune_service.content | b64decode" + - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode" - "'docker system prune -af' in prune_service.content | b64decode" - "'docker network prune' in prune_service.content | b64decode" - "'docker builder prune' in prune_service.content | b64decode" @@ -105,6 +106,7 @@ - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode" - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" - "'docker rm -f' in healthcheck_script.content | b64decode" + - "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode" - "'docker system prune -af' in healthcheck_script.content | b64decode" - "'docker network prune' in healthcheck_script.content | b64decode" - "'status=removing' in healthcheck_script.content | b64decode" diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index 22f5a1c..208c8c0 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -5,11 +5,13 @@ Description=Docker prune for Gitea runner resources Type=oneshot Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} -# Force-remove ALL containers (including running ones) left behind by failed +# Force-remove stale containers (including running ones) left behind by failed # molecule tests. "docker container prune -f" only removes stopped containers, # so running containers from crashed/interrupted CI jobs accumulate indefinitely, # consuming disk and memory. We stop+rm everything first, then prune the rest. -ExecStart=/bin/sh -c 'docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true' +# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing +# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors. +ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes ExecStart=/usr/bin/docker network prune -f ExecStart=/usr/bin/docker builder prune -f diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index d8f624b..1b47bfa 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -159,10 +159,14 @@ fi disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources" - # Force-remove ALL containers (including running ones from failed molecule tests). + # Force-remove stale containers (including running ones from failed molecule tests). # "docker container prune -f" only removes stopped containers, so running # containers from crashed CI jobs accumulate and consume disk/memory. - docker ps -aq 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true + # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). + docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ + | grep -v 'GITEA-ACTIONS-TASK' \ + | awk '{print $1}' \ + | xargs -r docker rm -f 2>/dev/null || true docker system prune -af --filter "until=1h" --volumes || true docker network prune -f || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')