feat: scoped runner cleanup with ownership leases and disk admission

Extend stopped-container protection to an explicit lease contract
(org.oblachno.lease-until / org.oblachno.owner) honored by every cleanup
path. Consolidate the duplicated inline prune logic from docker-prune
and the healthcheck into a single tiered runner-cleanup.sh; remove the
unfiltered `system prune -af --volumes` / `volume prune` paths that could
wipe a job's volumes mid-run, and keep warm base images under pressure.

At critical disk usage the healthcheck now stops admitting new work
(stops gitea-runner.service once no CI job is in flight) and resumes it
automatically after recovery. The runner config declares capacity, and
the role refuses to install on production-marked hosts.
This commit is contained in:
Emil Simeonov
2026-09-22 18:34:12 +02:00
parent 49646c38db
commit 3c0696f3f2
10 changed files with 385 additions and 106 deletions
+26 -5
View File
@@ -31,6 +31,13 @@ gitea_runner_prune_until: "24h"
# container operations).
gitea_runner_prune_schedule: "*-*-* 00/6:00:00"
gitea_runner_prune_label: "gitea-runner=true"
# Shared scoped cleanup script (runner-cleanup.sh) used by the prune timer
# and the healthcheck disk-pressure tiers (GRM-173).
gitea_runner_cleanup_script_path: "{{ gitea_runner_config_dir }}/cleanup.sh"
# Regex alternation of image refs never removed by cleanup — warm base
# layers stay warm even under critical disk pressure.
gitea_runner_keep_images:
- "runner-images/"
# Service configuration
gitea_runner_service_restart_sec: "5"
@@ -42,13 +49,20 @@ gitea_runner_service_restart_sec: "5"
gitea_runner_healthcheck_interval: "2min"
gitea_runner_healthcheck_boot_delay: "2min"
gitea_runner_healthcheck_disk_threshold: 70
# When disk reaches this level, prune EVERYTHING (no until-filter) — the
# runner is dangerously full and the gentle until=1h prune isn't enough.
# This removes all stopped containers and unused images regardless of age.
# At 75%+, molecule containers fail with "container is not running" because
# overlay2 runs out of space under parallel DinD load.
# At this level the cleanup script drops age limits — the runner is
# dangerously full and the gentle until=1h prune isn't enough. Leases,
# keep-images, running containers and CI job containers are still honored
# (GRM-173). At 75%+, molecule containers fail with "container is not
# running" because overlay2 runs out of space under parallel DinD load.
gitea_runner_healthcheck_disk_critical: 75
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
# Disk-pressure admission control (GRM-173): at critical disk usage the
# healthcheck stops gitea-runner.service (no new jobs are fetched) once no
# CI job container is running, and resumes it automatically after recovery.
gitea_runner_disk_admission_enabled: true
# Physical-host admission (GRM-173): act_runner capacity — max parallel
# tasks per runner. Declared explicitly (upstream default is 1).
gitea_runner_capacity: 1
# CI job containers older than this many minutes get an exec-responsiveness
# probe; a timeout writes one diagnostics bundle per container for
@@ -121,6 +135,13 @@ gitea_runner_valid_volumes:
gitea_runner_containerd_max_compatible_major: 2
gitea_runner_containerd_max_compatible_minor: 2
# Production-host exclusion (GRM-173): the role fails when the target is a
# production host — either via this flag or the /etc/oblachno/production-host
# marker file — unless allow_production_host explicitly overrides.
gitea_runner_on_production_host: false
gitea_runner_allow_production_host: false
gitea_runner_production_marker_path: "/etc/oblachno/production-host"
# Docker installation (for rootless dependencies)
gitea_runner_docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg"
gitea_runner_docker_apt_arch: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else ansible_facts['architecture'] }}"