From 3f22910a7dca9989663c9f036eed365d6abda2ac Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 10 Aug 2026 09:04:24 +0200 Subject: [PATCH] fix: reduce healthcheck interval to 1min and disk thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runners are consistently running out of disk during parallel molecule tests. The 2min healthcheck interval is too coarse — a single molecule scenario can fill 10+ GB in 2 minutes. Reduce to 1min and lower disk thresholds (60% warn, 70% critical) so pruning kicks in earlier. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ansible/roles/gitea_runner/defaults/main.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 1d991ce..cb8bd27 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -36,18 +36,20 @@ gitea_runner_prune_label: "gitea-runner=true" gitea_runner_service_restart_sec: "5" # Health check configuration -# 2min interval — catches hung daemons before multiple CI jobs fail between checks. -# The previous 5min interval was too coarse: a stuck daemon could fail 3+ molecule -# jobs in the window between healthcheck runs. -gitea_runner_healthcheck_interval: "2min" +# 1min interval — catches hung daemons and disk exhaustion before multiple CI +# jobs fail between checks. The previous 2min interval was too coarse under +# parallel DinD load: a single molecule scenario can fill 10+ GB in 2 minutes, +# causing subsequent scenarios on the same runner to fail with "Failed to +# create temporary directory" (disk full). +gitea_runner_healthcheck_interval: "1min" gitea_runner_healthcheck_boot_delay: "2min" -gitea_runner_healthcheck_disk_threshold: 70 +gitea_runner_healthcheck_disk_threshold: 60 # When disk reaches this level, prune EVERYTHING (no until-filter) — the # runner is dangerously full and the gentle until=1h prune isn't enough. # This removes all stopped containers and unused images regardless of age. # At 75%+, molecule containers fail with "container is not running" because # overlay2 runs out of space under parallel DinD load. -gitea_runner_healthcheck_disk_critical: 75 +gitea_runner_healthcheck_disk_critical: 70 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" # Auto-recovery: when the healthcheck detects an unregistered runner, it