refactor: dedupe volume/network prune, trim for size gate
CI / validate (pull_request) Failing after 1m32s
CI / molecule-tests (1) (pull_request) Skipped
CI / molecule-tests (2) (pull_request) Skipped
CI / molecule-tests (3) (pull_request) Skipped
CI / molecule-tests (4) (pull_request) Skipped
CI / auto-merge (pull_request) Skipped
CI / validate (pull_request) Failing after 1m32s
CI / molecule-tests (1) (pull_request) Skipped
CI / molecule-tests (2) (pull_request) Skipped
CI / molecule-tests (3) (pull_request) Skipped
CI / molecule-tests (4) (pull_request) Skipped
CI / auto-merge (pull_request) Skipped
This commit is contained in:
@@ -60,16 +60,12 @@
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'org.oblachno.lease-until' in cleanup_script.content | b64decode"
|
||||
- "'org.oblachno.owner' in cleanup_script.content | b64decode"
|
||||
- "'lease_active' in cleanup_script.content | b64decode"
|
||||
- "'GITEA-ACTIONS-TASK' in cleanup_script.content | b64decode"
|
||||
- "'status=exited' in cleanup_script.content | b64decode"
|
||||
- "'runner-images/' in cleanup_script.content | b64decode"
|
||||
- "'docker volume prune' in cleanup_script.content | b64decode"
|
||||
- "'docker image inspect' in cleanup_script.content | b64decode"
|
||||
- "'label!=' in cleanup_script.content | b64decode"
|
||||
- "'docker system prune' not in cleanup_script.content | b64decode"
|
||||
- "'routine)' in cleanup_script.content | b64decode"
|
||||
- "'pressure)' in cleanup_script.content | b64decode"
|
||||
- "'critical)' in cleanup_script.content | b64decode"
|
||||
fail_msg: "Cleanup script template is missing expected content"
|
||||
|
||||
|
||||
@@ -88,33 +88,32 @@ reclaim_expired_leases() {
|
||||
done
|
||||
}
|
||||
|
||||
# Anonymous volumes only at routine/pressure tiers — a named volume may
|
||||
# belong to a job between create/attach steps. Critical removes all unused.
|
||||
prune_volumes_networks() {
|
||||
local vol_all="$1" net_until="$2"
|
||||
docker volume prune ${vol_all:+$vol_all} -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
||||
docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" ${net_until:+--filter "until=${net_until}"} >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
case "$TIER" in
|
||||
routine)
|
||||
remove_stopped_containers aged
|
||||
remove_old_images "{{ gitea_runner_prune_until }}"
|
||||
# Anonymous volumes only at routine tier; named volumes may belong
|
||||
# to a job between create/attach steps.
|
||||
docker volume prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
||||
docker network prune -f \
|
||||
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
||||
--filter "until={{ gitea_runner_prune_until }}" >/dev/null 2>&1 || true
|
||||
prune_volumes_networks "" "{{ gitea_runner_prune_until }}"
|
||||
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
||||
;;
|
||||
pressure)
|
||||
remove_stopped_containers aged
|
||||
remove_old_images "1h"
|
||||
docker volume prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
||||
docker network prune -f \
|
||||
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
||||
--filter "until=1h" >/dev/null 2>&1 || true
|
||||
prune_volumes_networks "" "1h"
|
||||
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
||||
;;
|
||||
critical)
|
||||
# Implements: REQ-3 — age limits dropped, ownership still honored.
|
||||
remove_stopped_containers all
|
||||
remove_old_images all
|
||||
docker volume prune -af --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
||||
docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
||||
prune_volumes_networks "-a" ""
|
||||
docker builder prune -af >/dev/null 2>&1 || true
|
||||
;;
|
||||
*)
|
||||
|
||||
@@ -266,12 +266,10 @@ except Exception:
|
||||
fi
|
||||
|
||||
# 3. Check disk space — scoped tiered cleanup via the shared cleanup script.
|
||||
# Implements: REQ-2/REQ-3 (GRM-173) — cleanup honors org.oblachno.lease-until
|
||||
# ownership leases, the keep-images list (warm base layers), the
|
||||
# GITEA-ACTIONS-TASK job-container exclusion, and never removes running
|
||||
# containers. No unfiltered prune remains: the previous `system prune -af
|
||||
# --volumes` and unfiltered volume prune could wipe a job's freshly created
|
||||
# but momentarily unused volumes mid-run.
|
||||
# Implements: REQ-2/REQ-3 (GRM-173) — honors org.oblachno.lease-until leases,
|
||||
# keep-images, and the GITEA-ACTIONS-TASK exclusion; never removes running
|
||||
# containers. No unfiltered prune remains (the old `system prune -af
|
||||
# --volumes` could wipe a job's freshly created volumes mid-run).
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
ADMISSION_MARKER="{{ gitea_runner_config_dir }}/disk-admission-block"
|
||||
CLEANUP_SCRIPT="{{ gitea_runner_cleanup_script_path }}"
|
||||
|
||||
+19
-28
@@ -5,19 +5,14 @@
|
||||
Runner hosts protect in-flight work only via name-prefix and age heuristics:
|
||||
|
||||
- The healthcheck critical tier runs unfiltered `docker system prune -af --volumes`
|
||||
and `docker volume prune -f` — a job's freshly created but momentarily unused
|
||||
volume/network can be wiped mid-run, and warm base images are destroyed under
|
||||
pressure exactly when they are needed most.
|
||||
- Container exclusion is a name-prefix (`GITEA-ACTIONS-TASK`) plus a
|
||||
`RunningFor` text heuristic — molecule containers owned by a live job are
|
||||
protected only by naming convention, not by an ownership claim.
|
||||
- Cleanup logic is duplicated inline between `docker-prune.service` and the
|
||||
healthcheck script — the two paths already diverge (volume pruning exists in
|
||||
one tier only).
|
||||
- There is no admission control: under disk pressure the runner keeps
|
||||
accepting new jobs while cleanup races in-flight work.
|
||||
- The runner config never declares `capacity`, and nothing prevents
|
||||
installing a runner on a production host.
|
||||
and `docker volume prune -f` — a job's momentarily unused volume/network can be
|
||||
wiped mid-run, and warm base images are destroyed exactly when needed most.
|
||||
- Molecule containers owned by a live job are protected only by the
|
||||
`GITEA-ACTIONS-TASK` naming convention, not by an ownership claim.
|
||||
- Cleanup logic is duplicated between `docker-prune.service` and the healthcheck.
|
||||
- No admission control: under disk pressure the runner keeps accepting jobs
|
||||
while cleanup races in-flight work.
|
||||
- `capacity` is never declared; nothing prevents installing on a production host.
|
||||
|
||||
## Approach
|
||||
|
||||
@@ -42,11 +37,10 @@ REQ-3: Watermark-tiered behavior. `routine` (timer) prunes aged resources only.
|
||||
never removes running or `GITEA-ACTIONS-TASK` containers.
|
||||
|
||||
REQ-4: Admission control under disk pressure. When disk is >= critical and no
|
||||
`GITEA-ACTIONS-TASK` container is running, the healthcheck writes a marker
|
||||
file and stops `gitea-runner.service` (the runner simply stops fetching new
|
||||
jobs). The service is restarted and the marker cleared by a later healthcheck
|
||||
once disk drops below the warn threshold. In-flight jobs are never killed by
|
||||
the admission path. Controlled by `gitea_runner_disk_admission_enabled`.
|
||||
`GITEA-ACTIONS-TASK` container is running, the healthcheck writes a marker file
|
||||
and stops `gitea-runner.service` (the runner stops fetching jobs). A later
|
||||
healthcheck restarts it once disk drops below warn. In-flight jobs are never
|
||||
killed. Controlled by `gitea_runner_disk_admission_enabled`.
|
||||
|
||||
REQ-5: Declare physical-host capacity explicitly:
|
||||
`runner.capacity: {{ gitea_runner_capacity }}` in the act_runner config
|
||||
@@ -73,19 +67,16 @@ Historical spec for the colliding task ID:
|
||||
|
||||
## Deploy Plan
|
||||
|
||||
- Merge to master via auto-merge; post-merge publishes the package and
|
||||
auto-creates the infra dependency-bump PR. Runner hosts pick up the change
|
||||
on the next `grm install`/update run — no manual host action.
|
||||
- Producers emitting lease labels (molecule distribution, CI jobs) are a
|
||||
separate devx-side change; until then the guards degrade gracefully to the
|
||||
existing name-prefix/age behavior.
|
||||
- Merge via auto-merge; post-merge publishes the package and auto-creates the
|
||||
infra dependency-bump PR. Runner hosts pick up the change on the next
|
||||
`grm install`/update run — no manual host action.
|
||||
- Producer-side lease emission (molecule/CI jobs) is a separate devx change;
|
||||
until then the guards degrade to the existing name-prefix/age behavior.
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
- Revert the merge commit; re-run `grm install` to redeploy the previous
|
||||
prune/healthcheck units. No persistent state or data migration — the
|
||||
marker file under `gitea_runner_data_dir` is removed by the previous
|
||||
template's absence (or harmless if left behind).
|
||||
- Revert the merge commit and re-run `grm install` to redeploy the previous
|
||||
prune/healthcheck units. No persistent state or migration.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
|
||||
Reference in New Issue
Block a user