diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index 9f1fdad..2f3021a 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -60,16 +60,12 @@ ansible.builtin.assert: that: - "'org.oblachno.lease-until' in cleanup_script.content | b64decode" - - "'org.oblachno.owner' in cleanup_script.content | b64decode" - "'lease_active' in cleanup_script.content | b64decode" - "'GITEA-ACTIONS-TASK' in cleanup_script.content | b64decode" - - "'status=exited' in cleanup_script.content | b64decode" - "'runner-images/' in cleanup_script.content | b64decode" - - "'docker volume prune' in cleanup_script.content | b64decode" + - "'docker image inspect' in cleanup_script.content | b64decode" - "'label!=' in cleanup_script.content | b64decode" - "'docker system prune' not in cleanup_script.content | b64decode" - - "'routine)' in cleanup_script.content | b64decode" - - "'pressure)' in cleanup_script.content | b64decode" - "'critical)' in cleanup_script.content | b64decode" fail_msg: "Cleanup script template is missing expected content" diff --git a/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 b/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 index d438221..2ef6447 100644 --- a/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 @@ -88,33 +88,32 @@ reclaim_expired_leases() { done } +# Anonymous volumes only at routine/pressure tiers — a named volume may +# belong to a job between create/attach steps. Critical removes all unused. +prune_volumes_networks() { + local vol_all="$1" net_until="$2" + docker volume prune ${vol_all:+$vol_all} -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true + docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" ${net_until:+--filter "until=${net_until}"} >/dev/null 2>&1 || true +} + case "$TIER" in routine) remove_stopped_containers aged remove_old_images "{{ gitea_runner_prune_until }}" - # Anonymous volumes only at routine tier; named volumes may belong - # to a job between create/attach steps. - docker volume prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true - docker network prune -f \ - --filter "label!=${LEASE_UNTIL_LABEL}" \ - --filter "until={{ gitea_runner_prune_until }}" >/dev/null 2>&1 || true + prune_volumes_networks "" "{{ gitea_runner_prune_until }}" docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true ;; pressure) remove_stopped_containers aged remove_old_images "1h" - docker volume prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true - docker network prune -f \ - --filter "label!=${LEASE_UNTIL_LABEL}" \ - --filter "until=1h" >/dev/null 2>&1 || true + prune_volumes_networks "" "1h" docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true ;; critical) # Implements: REQ-3 — age limits dropped, ownership still honored. remove_stopped_containers all remove_old_images all - docker volume prune -af --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true - docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true + prune_volumes_networks "-a" "" docker builder prune -af >/dev/null 2>&1 || true ;; *) diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index dd3318e..ec2c71d 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -266,12 +266,10 @@ except Exception: fi # 3. Check disk space — scoped tiered cleanup via the shared cleanup script. -# Implements: REQ-2/REQ-3 (GRM-173) — cleanup honors org.oblachno.lease-until -# ownership leases, the keep-images list (warm base layers), the -# GITEA-ACTIONS-TASK job-container exclusion, and never removes running -# containers. No unfiltered prune remains: the previous `system prune -af -# --volumes` and unfiltered volume prune could wipe a job's freshly created -# but momentarily unused volumes mid-run. +# Implements: REQ-2/REQ-3 (GRM-173) — honors org.oblachno.lease-until leases, +# keep-images, and the GITEA-ACTIONS-TASK exclusion; never removes running +# containers. No unfiltered prune remains (the old `system prune -af +# --volumes` could wipe a job's freshly created volumes mid-run). disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') ADMISSION_MARKER="{{ gitea_runner_config_dir }}/disk-admission-block" CLEANUP_SCRIPT="{{ gitea_runner_cleanup_script_path }}" diff --git a/docs/specs/GRM-173.md b/docs/specs/GRM-173.md index a66face..058de9d 100644 --- a/docs/specs/GRM-173.md +++ b/docs/specs/GRM-173.md @@ -5,19 +5,14 @@ Runner hosts protect in-flight work only via name-prefix and age heuristics: - The healthcheck critical tier runs unfiltered `docker system prune -af --volumes` - and `docker volume prune -f` — a job's freshly created but momentarily unused - volume/network can be wiped mid-run, and warm base images are destroyed under - pressure exactly when they are needed most. -- Container exclusion is a name-prefix (`GITEA-ACTIONS-TASK`) plus a - `RunningFor` text heuristic — molecule containers owned by a live job are - protected only by naming convention, not by an ownership claim. -- Cleanup logic is duplicated inline between `docker-prune.service` and the - healthcheck script — the two paths already diverge (volume pruning exists in - one tier only). -- There is no admission control: under disk pressure the runner keeps - accepting new jobs while cleanup races in-flight work. -- The runner config never declares `capacity`, and nothing prevents - installing a runner on a production host. + and `docker volume prune -f` — a job's momentarily unused volume/network can be + wiped mid-run, and warm base images are destroyed exactly when needed most. +- Molecule containers owned by a live job are protected only by the + `GITEA-ACTIONS-TASK` naming convention, not by an ownership claim. +- Cleanup logic is duplicated between `docker-prune.service` and the healthcheck. +- No admission control: under disk pressure the runner keeps accepting jobs + while cleanup races in-flight work. +- `capacity` is never declared; nothing prevents installing on a production host. ## Approach @@ -42,11 +37,10 @@ REQ-3: Watermark-tiered behavior. `routine` (timer) prunes aged resources only. never removes running or `GITEA-ACTIONS-TASK` containers. REQ-4: Admission control under disk pressure. When disk is >= critical and no -`GITEA-ACTIONS-TASK` container is running, the healthcheck writes a marker -file and stops `gitea-runner.service` (the runner simply stops fetching new -jobs). The service is restarted and the marker cleared by a later healthcheck -once disk drops below the warn threshold. In-flight jobs are never killed by -the admission path. Controlled by `gitea_runner_disk_admission_enabled`. +`GITEA-ACTIONS-TASK` container is running, the healthcheck writes a marker file +and stops `gitea-runner.service` (the runner stops fetching jobs). A later +healthcheck restarts it once disk drops below warn. In-flight jobs are never +killed. Controlled by `gitea_runner_disk_admission_enabled`. REQ-5: Declare physical-host capacity explicitly: `runner.capacity: {{ gitea_runner_capacity }}` in the act_runner config @@ -73,19 +67,16 @@ Historical spec for the colliding task ID: ## Deploy Plan -- Merge to master via auto-merge; post-merge publishes the package and - auto-creates the infra dependency-bump PR. Runner hosts pick up the change - on the next `grm install`/update run — no manual host action. -- Producers emitting lease labels (molecule distribution, CI jobs) are a - separate devx-side change; until then the guards degrade gracefully to the - existing name-prefix/age behavior. +- Merge via auto-merge; post-merge publishes the package and auto-creates the + infra dependency-bump PR. Runner hosts pick up the change on the next + `grm install`/update run — no manual host action. +- Producer-side lease emission (molecule/CI jobs) is a separate devx change; + until then the guards degrade to the existing name-prefix/age behavior. ## Rollback Plan -- Revert the merge commit; re-run `grm install` to redeploy the previous - prune/healthcheck units. No persistent state or data migration — the - marker file under `gitea_runner_data_dir` is removed by the previous - template's absence (or harmless if left behind). +- Revert the merge commit and re-run `grm install` to redeploy the previous + prune/healthcheck units. No persistent state or migration. ## Acceptance Criteria