From 9c786ec4df5bff50e3e82b37d01fd830910440ed Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 10 Aug 2026 11:54:41 +0200 Subject: [PATCH] revert: restore original healthcheck thresholds (2min, 70%, 75%) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1min interval + 70% critical threshold caused more problems than it solved. The host disk normally sits at ~74% — just under the original 75% critical threshold. Lowering to 70% triggered aggressive docker system prune -af --volumes every single minute, which: 1. Wiped all cached images, forcing all 6 parallel slots to re-pull 2-3GB each simultaneously — increasing disk pressure, not reducing it 2. Caused network prune to remove networks mid-job → "network not found" errors in Gitea Actions Reverting to 2min/70%warn/75%critical. Images stay cached, only gentle until=1h pruning runs at 70%, and full prune only triggers at 75% (above the normal 74% baseline). The network prune --filter "until=1h" fix from the previous commit is kept — that was a genuine bug fix. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ansible/roles/gitea_runner/defaults/main.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 18b55e2..d0fc2f2 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -36,20 +36,23 @@ gitea_runner_prune_label: "gitea-runner=true" gitea_runner_service_restart_sec: "5" # Health check configuration -# 1min interval — catches hung daemons and disk exhaustion before multiple CI -# jobs fail between checks. The previous 2min interval was too coarse under -# parallel DinD load: a single molecule scenario can fill 10+ GB in 2 minutes, -# causing subsequent scenarios on the same runner to fail with "Failed to -# create temporary directory" (disk full). -gitea_runner_healthcheck_interval: "1min" +# 2min interval — catches hung daemons before multiple CI jobs fail between +# checks. The 1min interval caused excessive pruning which removed cached +# images, forcing all 6 parallel slots to re-pull simultaneously and +# actually increasing disk pressure. +gitea_runner_healthcheck_interval: "2min" gitea_runner_healthcheck_boot_delay: "2min" -gitea_runner_healthcheck_disk_threshold: 60 +gitea_runner_healthcheck_disk_threshold: 70 # When disk reaches this level, prune EVERYTHING (no until-filter) — the # runner is dangerously full and the gentle until=1h prune isn't enough. # This removes all stopped containers and unused images regardless of age. # At 75%+, molecule containers fail with "container is not running" because # overlay2 runs out of space under parallel DinD load. -gitea_runner_healthcheck_disk_critical: 70 +# IMPORTANT: keep at 75 (not lower) — the host disk normally sits at ~74%. +# Lowering to 70 triggers full prune every cycle, wiping cached images and +# forcing all parallel slots to re-pull simultaneously, which increases +# disk pressure rather than reducing it. +gitea_runner_healthcheck_disk_critical: 75 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" # Auto-recovery: when the healthcheck detects an unregistered runner, it