GRM-156: fix: harden rootless Docker daemon resilience on CI runners

ansible/roles/gitea_runner/defaults/main.yml

@@ -25,11 +25,29 @@ gitea_runner_prune_label: "gitea-runner=true"
 gitea_runner_service_restart_sec: "5"
 
 # Health check configuration
-gitea_runner_healthcheck_interval: "5min"
+# 2min interval — catches hung daemons before multiple CI jobs fail between checks.
+# The previous 5min interval was too coarse: a stuck daemon could fail 3+ molecule
+# jobs in the window between healthcheck runs.
+gitea_runner_healthcheck_interval: "2min"
 gitea_runner_healthcheck_boot_delay: "2min"
 gitea_runner_healthcheck_disk_threshold: 85
 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
 
+# Docker daemon resilience settings (applied to daemon.json).
+# live-restore: containers survive daemon restarts — prevents stuck container
+#   states when the healthcheck restarts a hung daemon.
+# shutdown-timeout: grace period (seconds) for containers to stop on daemon
+#   shutdown/restart. Default 15s is too short for DinD containers with nested
+#   processes (molecule tests). 30s gives SIGTERM time to propagate.
+# max-concurrent-downloads/uploads: limits parallel transfers to reduce daemon
+#   memory pressure when multiple CI jobs pull images simultaneously.
+# default-ulimits: prevents FD exhaustion in container processes.
+gitea_runner_docker_live_restore: true
+gitea_runner_docker_shutdown_timeout: 30
+gitea_runner_docker_max_concurrent_downloads: 3
+gitea_runner_docker_max_concurrent_uploads: 3
+gitea_runner_docker_default_nofile: 1048576
+
 # Admin token for runner deregistration via Gitea API.
 # If not set, falls back to registration_token (which likely lacks admin scope).
 # Set this to a token with admin scope to enable automatic runner cleanup on removal.

ansible/roles/gitea_runner/molecule/template-content/verify.yml

@@ -49,6 +49,9 @@
           - "'Type=oneshot' in prune_service.content | b64decode"
           - "'docker system prune' in prune_service.content | b64decode"
           - "'docker volume prune' in prune_service.content | b64decode"
+          - "'docker container prune' in prune_service.content | b64decode"
+          - "'docker network prune' in prune_service.content | b64decode"
+          - "'docker builder prune' in prune_service.content | b64decode"
         fail_msg: "Prune service template is missing expected directives"
 
     - name: Read rendered prune timer template
@@ -99,9 +102,15 @@
       ansible.builtin.assert:
         that:
           - "'docker info' in healthcheck_script.content | b64decode"
+          - "'timeout 10 docker info' in healthcheck_script.content | b64decode"
           - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
           - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
           - "'docker system prune' in healthcheck_script.content | b64decode"
+          - "'docker container prune' in healthcheck_script.content | b64decode"
+          - "'docker network prune' in healthcheck_script.content | b64decode"
+          - "'status=removing' in healthcheck_script.content | b64decode"
+          - "'status=stopping' in healthcheck_script.content | b64decode"
+          - "'docker rm -f' in healthcheck_script.content | b64decode"
           - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
         fail_msg: "Healthcheck script template is missing expected content"
 

ansible/roles/gitea_runner/tasks/rootless_docker.yml

@@ -214,6 +214,13 @@
     dest: "{{ gitea_runner_home }}/.config/docker/daemon.json"
     content: |
       {
+        "live-restore": {{ gitea_runner_docker_live_restore | to_json }},
+        "shutdown-timeout": {{ gitea_runner_docker_shutdown_timeout }},
+        "max-concurrent-downloads": {{ gitea_runner_docker_max_concurrent_downloads }},
+        "max-concurrent-uploads": {{ gitea_runner_docker_max_concurrent_uploads }},
+        "default-ulimits": {
+          "nofile": {"Name": "nofile", "Hard": {{ gitea_runner_docker_default_nofile }}, "Soft": {{ gitea_runner_docker_default_nofile }}}
+        },
         {% if gitea_runner_docker_rootless_net_driver == 'pasta' %}
         "ipv6": true,
         "ip6tables": true,

ansible/roles/gitea_runner/templates/docker-prune.service.j2

@@ -7,3 +7,6 @@ Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
 Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
 ExecStart=/usr/bin/docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until={{ gitea_runner_prune_until }}"
 ExecStart=/usr/bin/docker volume prune -f --filter "label={{ gitea_runner_prune_label }}"
+ExecStart=/usr/bin/docker container prune -f
+ExecStart=/usr/bin/docker network prune -f
+ExecStart=/usr/bin/docker builder prune -f

ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2

@@ -7,18 +7,29 @@ DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
 XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
 export DOCKER_HOST XDG_RUNTIME_DIR
 
-# 1. Check Docker daemon responsiveness
-if ! docker info >/dev/null 2>&1; then
-    echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}"
+# 1. Check Docker daemon responsiveness (with timeout — a bare `docker info` can
+#    hang indefinitely on a stuck daemon, blocking the healthcheck itself).
+if ! timeout 10 docker info >/dev/null 2>&1; then
+    echo "ERROR: Docker daemon not responding at ${DOCKER_HOST} (timed out after 10s)"
     systemctl --user restart docker.service
     sleep 3
-    if ! docker info >/dev/null 2>&1; then
+    if ! timeout 10 docker info >/dev/null 2>&1; then
         echo "CRITICAL: Docker daemon still down after restart"
         exit 1
     fi
     echo "RECOVERED: Docker daemon restarted successfully"
 fi
 
+# 1b. Clean up stuck containers — containers in "removing" or "stopping" state
+#     for too long cause "cannot kill container: did not receive an exit event"
+#     errors in molecule destroy phases. Force-remove them so subsequent CI jobs
+#     don't inherit the stuck state.
+stuck_containers=$(timeout 10 docker ps -a --filter "status=removing" --filter "status=stopping" --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null || true)
+if [[ -n "$stuck_containers" ]]; then
+    echo "WARN: Found stuck containers (removing/stopping), force-cleaning"
+    echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
+fi
+
 # 2. Check gitea-runner service is active
 runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
 if [[ "$runner_state" != "active" ]]; then
@@ -41,6 +52,9 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
     docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true
     # Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality)
     docker image prune -f || true
+    # Clean up stopped containers and dangling networks that accumulate from failed jobs
+    docker container prune -f || true
+    docker network prune -f || true
     disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
     echo "INFO: Disk usage after prune: ${disk_pct}%"
 fi