GRM-159: feat(healthcheck): add two-tier disk prune with critical threshold

Co-authored-by: emil User <emil.simeonov@tutanota.com>

ansible/roles/gitea_runner/defaults/main.yml

@@ -41,7 +41,13 @@ gitea_runner_service_restart_sec: "5"
 # jobs in the window between healthcheck runs.
 gitea_runner_healthcheck_interval: "2min"
 gitea_runner_healthcheck_boot_delay: "2min"
-gitea_runner_healthcheck_disk_threshold: 75
+gitea_runner_healthcheck_disk_threshold: 70
+# When disk reaches this level, prune EVERYTHING (no until-filter) — the
+# runner is dangerously full and the gentle until=1h prune isn't enough.
+# This removes all stopped containers and unused images regardless of age.
+# At 75%+, molecule containers fail with "container is not running" because
+# overlay2 runs out of space under parallel DinD load.
+gitea_runner_healthcheck_disk_critical: 75
 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
 
 # Auto-recovery: when the healthcheck detects an unregistered runner, it

ansible/roles/gitea_runner/molecule/template-content/verify.yml

@@ -112,4 +112,5 @@
           - "'status=removing' in healthcheck_script.content | b64decode"
           - "'status=stopping' in healthcheck_script.content | b64decode"
           - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
+          - "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode"
         fail_msg: "Healthcheck script template is missing expected content"

ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2

@@ -223,8 +223,23 @@ fi
 
 # 3. Check disk space — prune aggressively if below threshold
 disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
-if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
-    echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
+if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then
+    echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune"
+    # Critical level: remove ALL stopped containers (no age filter) and ALL
+    # unused images/volumes. The until=1h gentle prune is insufficient here.
+    # Stop+rm stale non-CI containers regardless of age (failed molecule tests
+    # from the last 59 minutes also consume disk).
+    docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
+        | grep -v 'GITEA-ACTIONS-TASK' \
+        | awk '{print $1}' \
+        | xargs -r docker rm -f 2>/dev/null || true
+    docker system prune -af --volumes || true
+    docker network prune -f || true
+    docker builder prune -af || true
+    disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+    echo "INFO: Disk usage after full prune: ${disk_pct}%"
+elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
+    echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)"
     # Force-remove stale containers (including running ones from failed molecule tests)
     # that are older than 1 hour. "docker container prune -f" only removes stopped
     # containers, so running containers from crashed CI jobs accumulate and consume
@@ -236,7 +251,11 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
         | grep -E '(hour|day|week|month|year)s? ago' \
         | awk '{print $1}' \
         | xargs -r docker rm -f 2>/dev/null || true
-    docker system prune -af --filter "until=1h" --volumes || true
+    # Prune images and containers older than 1h (until filter is NOT
+    # supported with --volumes, so prune volumes separately without a filter).
+    docker image prune -af --filter "until=1h" 2>/dev/null || true
+    docker container prune -f --filter "until=1h" 2>/dev/null || true
+    docker volume prune -f 2>/dev/null || true
     # Prune networks older than 1 hour to avoid removing networks that
     # molecule tests are actively creating (e.g. 'traefik' network created
     # during molecule create phase before containers are attached).