From 185e41c49e4a8ad28acd176cc8db412c4b972eb5 Mon Sep 17 00:00:00 2001 From: emo Date: Wed, 5 Aug 2026 13:50:34 +0000 Subject: [PATCH] GRM-156: fix: harden rootless Docker daemon resilience on CI runners --- ansible/roles/gitea_runner/defaults/main.yml | 20 ++++++++++++++++- .../molecule/template-content/verify.yml | 9 ++++++++ .../gitea_runner/tasks/rootless_docker.yml | 7 ++++++ .../templates/docker-prune.service.j2 | 3 +++ .../templates/runner-healthcheck.sh.j2 | 22 +++++++++++++++---- 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 08bb745..8ed07ca 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -25,11 +25,29 @@ gitea_runner_prune_label: "gitea-runner=true" gitea_runner_service_restart_sec: "5" # Health check configuration -gitea_runner_healthcheck_interval: "5min" +# 2min interval — catches hung daemons before multiple CI jobs fail between checks. +# The previous 5min interval was too coarse: a stuck daemon could fail 3+ molecule +# jobs in the window between healthcheck runs. +gitea_runner_healthcheck_interval: "2min" gitea_runner_healthcheck_boot_delay: "2min" gitea_runner_healthcheck_disk_threshold: 85 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" +# Docker daemon resilience settings (applied to daemon.json). +# live-restore: containers survive daemon restarts — prevents stuck container +# states when the healthcheck restarts a hung daemon. +# shutdown-timeout: grace period (seconds) for containers to stop on daemon +# shutdown/restart. Default 15s is too short for DinD containers with nested +# processes (molecule tests). 30s gives SIGTERM time to propagate. +# max-concurrent-downloads/uploads: limits parallel transfers to reduce daemon +# memory pressure when multiple CI jobs pull images simultaneously. +# default-ulimits: prevents FD exhaustion in container processes. +gitea_runner_docker_live_restore: true +gitea_runner_docker_shutdown_timeout: 30 +gitea_runner_docker_max_concurrent_downloads: 3 +gitea_runner_docker_max_concurrent_uploads: 3 +gitea_runner_docker_default_nofile: 1048576 + # Admin token for runner deregistration via Gitea API. # If not set, falls back to registration_token (which likely lacks admin scope). # Set this to a token with admin scope to enable automatic runner cleanup on removal. diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index a2c0558..f0ebdfa 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -49,6 +49,9 @@ - "'Type=oneshot' in prune_service.content | b64decode" - "'docker system prune' in prune_service.content | b64decode" - "'docker volume prune' in prune_service.content | b64decode" + - "'docker container prune' in prune_service.content | b64decode" + - "'docker network prune' in prune_service.content | b64decode" + - "'docker builder prune' in prune_service.content | b64decode" fail_msg: "Prune service template is missing expected directives" - name: Read rendered prune timer template @@ -99,9 +102,15 @@ ansible.builtin.assert: that: - "'docker info' in healthcheck_script.content | b64decode" + - "'timeout 10 docker info' in healthcheck_script.content | b64decode" - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode" - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" - "'docker system prune' in healthcheck_script.content | b64decode" + - "'docker container prune' in healthcheck_script.content | b64decode" + - "'docker network prune' in healthcheck_script.content | b64decode" + - "'status=removing' in healthcheck_script.content | b64decode" + - "'status=stopping' in healthcheck_script.content | b64decode" + - "'docker rm -f' in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" fail_msg: "Healthcheck script template is missing expected content" diff --git a/ansible/roles/gitea_runner/tasks/rootless_docker.yml b/ansible/roles/gitea_runner/tasks/rootless_docker.yml index 703cd8d..dd693bf 100644 --- a/ansible/roles/gitea_runner/tasks/rootless_docker.yml +++ b/ansible/roles/gitea_runner/tasks/rootless_docker.yml @@ -214,6 +214,13 @@ dest: "{{ gitea_runner_home }}/.config/docker/daemon.json" content: | { + "live-restore": {{ gitea_runner_docker_live_restore | to_json }}, + "shutdown-timeout": {{ gitea_runner_docker_shutdown_timeout }}, + "max-concurrent-downloads": {{ gitea_runner_docker_max_concurrent_downloads }}, + "max-concurrent-uploads": {{ gitea_runner_docker_max_concurrent_uploads }}, + "default-ulimits": { + "nofile": {"Name": "nofile", "Hard": {{ gitea_runner_docker_default_nofile }}, "Soft": {{ gitea_runner_docker_default_nofile }}} + }, {% if gitea_runner_docker_rootless_net_driver == 'pasta' %} "ipv6": true, "ip6tables": true, diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index d0ad2b4..5a3f1e3 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -7,3 +7,6 @@ Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} ExecStart=/usr/bin/docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until={{ gitea_runner_prune_until }}" ExecStart=/usr/bin/docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" +ExecStart=/usr/bin/docker container prune -f +ExecStart=/usr/bin/docker network prune -f +ExecStart=/usr/bin/docker builder prune -f diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index 80bf83f..d8b865e 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -7,18 +7,29 @@ DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock" XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}" export DOCKER_HOST XDG_RUNTIME_DIR -# 1. Check Docker daemon responsiveness -if ! docker info >/dev/null 2>&1; then - echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}" +# 1. Check Docker daemon responsiveness (with timeout — a bare `docker info` can +# hang indefinitely on a stuck daemon, blocking the healthcheck itself). +if ! timeout 10 docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon not responding at ${DOCKER_HOST} (timed out after 10s)" systemctl --user restart docker.service sleep 3 - if ! docker info >/dev/null 2>&1; then + if ! timeout 10 docker info >/dev/null 2>&1; then echo "CRITICAL: Docker daemon still down after restart" exit 1 fi echo "RECOVERED: Docker daemon restarted successfully" fi +# 1b. Clean up stuck containers — containers in "removing" or "stopping" state +# for too long cause "cannot kill container: did not receive an exit event" +# errors in molecule destroy phases. Force-remove them so subsequent CI jobs +# don't inherit the stuck state. +stuck_containers=$(timeout 10 docker ps -a --filter "status=removing" --filter "status=stopping" --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null || true) +if [[ -n "$stuck_containers" ]]; then + echo "WARN: Found stuck containers (removing/stopping), force-cleaning" + echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true +fi + # 2. Check gitea-runner service is active runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true) if [[ "$runner_state" != "active" ]]; then @@ -41,6 +52,9 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true # Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality) docker image prune -f || true + # Clean up stopped containers and dangling networks that accumulate from failed jobs + docker container prune -f || true + docker network prune -f || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') echo "INFO: Disk usage after prune: ${disk_pct}%" fi