feat: scoped runner cleanup with ownership leases and disk admission
Extend stopped-container protection to an explicit lease contract (org.oblachno.lease-until / org.oblachno.owner) honored by every cleanup path. Consolidate the duplicated inline prune logic from docker-prune and the healthcheck into a single tiered runner-cleanup.sh; remove the unfiltered `system prune -af --volumes` / `volume prune` paths that could wipe a job's volumes mid-run, and keep warm base images under pressure. At critical disk usage the healthcheck now stops admitting new work (stops gitea-runner.service once no CI job is in flight) and resumes it automatically after recovery. The runner config declares capacity, and the role refuses to install on production-marked hosts.
This commit is contained in:
@@ -31,6 +31,13 @@ gitea_runner_prune_until: "24h"
|
||||
# container operations).
|
||||
gitea_runner_prune_schedule: "*-*-* 00/6:00:00"
|
||||
gitea_runner_prune_label: "gitea-runner=true"
|
||||
# Shared scoped cleanup script (runner-cleanup.sh) used by the prune timer
|
||||
# and the healthcheck disk-pressure tiers (GRM-173).
|
||||
gitea_runner_cleanup_script_path: "{{ gitea_runner_config_dir }}/cleanup.sh"
|
||||
# Regex alternation of image refs never removed by cleanup — warm base
|
||||
# layers stay warm even under critical disk pressure.
|
||||
gitea_runner_keep_images:
|
||||
- "runner-images/"
|
||||
|
||||
# Service configuration
|
||||
gitea_runner_service_restart_sec: "5"
|
||||
@@ -42,13 +49,20 @@ gitea_runner_service_restart_sec: "5"
|
||||
gitea_runner_healthcheck_interval: "2min"
|
||||
gitea_runner_healthcheck_boot_delay: "2min"
|
||||
gitea_runner_healthcheck_disk_threshold: 70
|
||||
# When disk reaches this level, prune EVERYTHING (no until-filter) — the
|
||||
# runner is dangerously full and the gentle until=1h prune isn't enough.
|
||||
# This removes all stopped containers and unused images regardless of age.
|
||||
# At 75%+, molecule containers fail with "container is not running" because
|
||||
# overlay2 runs out of space under parallel DinD load.
|
||||
# At this level the cleanup script drops age limits — the runner is
|
||||
# dangerously full and the gentle until=1h prune isn't enough. Leases,
|
||||
# keep-images, running containers and CI job containers are still honored
|
||||
# (GRM-173). At 75%+, molecule containers fail with "container is not
|
||||
# running" because overlay2 runs out of space under parallel DinD load.
|
||||
gitea_runner_healthcheck_disk_critical: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
# Disk-pressure admission control (GRM-173): at critical disk usage the
|
||||
# healthcheck stops gitea-runner.service (no new jobs are fetched) once no
|
||||
# CI job container is running, and resumes it automatically after recovery.
|
||||
gitea_runner_disk_admission_enabled: true
|
||||
# Physical-host admission (GRM-173): act_runner capacity — max parallel
|
||||
# tasks per runner. Declared explicitly (upstream default is 1).
|
||||
gitea_runner_capacity: 1
|
||||
|
||||
# CI job containers older than this many minutes get an exec-responsiveness
|
||||
# probe; a timeout writes one diagnostics bundle per container for
|
||||
@@ -121,6 +135,13 @@ gitea_runner_valid_volumes:
|
||||
gitea_runner_containerd_max_compatible_major: 2
|
||||
gitea_runner_containerd_max_compatible_minor: 2
|
||||
|
||||
# Production-host exclusion (GRM-173): the role fails when the target is a
|
||||
# production host — either via this flag or the /etc/oblachno/production-host
|
||||
# marker file — unless allow_production_host explicitly overrides.
|
||||
gitea_runner_on_production_host: false
|
||||
gitea_runner_allow_production_host: false
|
||||
gitea_runner_production_marker_path: "/etc/oblachno/production-host"
|
||||
|
||||
# Docker installation (for rootless dependencies)
|
||||
gitea_runner_docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg"
|
||||
gitea_runner_docker_apt_arch: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else ansible_facts['architecture'] }}"
|
||||
|
||||
Reference in New Issue
Block a user