Emil Simeonov
3c0696f3f2
Extend stopped-container protection to an explicit lease contract (org.oblachno.lease-until / org.oblachno.owner) honored by every cleanup path. Consolidate the duplicated inline prune logic from docker-prune and the healthcheck into a single tiered runner-cleanup.sh; remove the unfiltered `system prune -af --volumes` / `volume prune` paths that could wipe a job's volumes mid-run, and keep warm base images under pressure. At critical disk usage the healthcheck now stops admitting new work (stops gitea-runner.service once no CI job is in flight) and resumes it automatically after recovery. The runner config declares capacity, and the role refuses to install on production-marked hosts.
103 lines
4.4 KiB
Django/Jinja
103 lines
4.4 KiB
Django/Jinja
#!/bin/bash
|
|
# Scoped Docker cleanup for gitea-runner hosts.
|
|
# Implements: REQ-1..REQ-3 (GRM-173) — ownership leases, tiered watermarks,
|
|
# keep-images. Single entry point shared by docker-prune.service (routine)
|
|
# and runner-healthcheck.sh (pressure/critical).
|
|
# No `set -e`: a failing prune must not abort the remaining cleanup.
|
|
set -uo pipefail
|
|
|
|
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
|
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
|
export DOCKER_HOST XDG_RUNTIME_DIR
|
|
|
|
TIER="${1:-routine}"
|
|
# REQ-1 label contract: `org.oblachno.lease-until` (epoch) protects an
|
|
# object while in the future; `org.oblachno.owner` records the owning run.
|
|
LEASE_UNTIL_LABEL="org.oblachno.lease-until"
|
|
KEEP_IMAGES_RE="{{ gitea_runner_keep_images | join('|') }}"
|
|
now_epoch=$(date +%s)
|
|
|
|
# Implements: REQ-1 — a lease whose `lease-until` epoch lies in the future
|
|
# protects its object from every removal path in this script.
|
|
lease_active() {
|
|
local until="$1"
|
|
[[ -n "$until" && "$until" =~ ^[0-9]+$ && "$until" -gt "$now_epoch" ]]
|
|
}
|
|
|
|
# Remove stopped containers. $1 = "aged" (only >1h, RunningFor heuristic)
|
|
# or "all". CI job containers and valid leases are never removed.
|
|
remove_stopped_containers() {
|
|
local mode="$1"
|
|
# Implements: REQ-1/REQ-3 — pipe-separated fields; RunningFor contains
|
|
# spaces, so whitespace-splitting would break the age gate.
|
|
{ timeout 30 docker ps -a --filter "status=exited" --filter "status=dead" \
|
|
--format '{% raw %}{{.ID}}|{{.Names}}|{{.RunningFor}}|{{.Label "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true; } \
|
|
| while IFS='|' read -r cid cname running_for lease_until; do
|
|
[[ -z "$cid" ]] && continue
|
|
case "$cname" in GITEA-ACTIONS-TASK*) continue ;; esac
|
|
lease_active "$lease_until" && continue
|
|
if [[ "$mode" != "all" ]] \
|
|
&& ! grep -qE '(hour|day|week|month|year)s? ago' <<<"$running_for"; then
|
|
continue
|
|
fi
|
|
docker rm -f "$cid" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
# Remove unused images older than $1 ("all" = no age limit). The keep-list
|
|
# (warm base layers) and leased images are never removed; images referenced
|
|
# by any container are refused by the daemon anyway.
|
|
remove_old_images() {
|
|
local until="$1"
|
|
docker image prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
|
local filters=(--filter "dangling=false")
|
|
[[ "$until" != "all" ]] && filters+=(--filter "until=${until}")
|
|
{ timeout 30 docker images "${filters[@]}" \
|
|
--format '{% raw %}{{.ID}}|{{.Repository}}:{{.Tag}}|{{.Label "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true; } \
|
|
| while IFS='|' read -r iid ref lease_until; do
|
|
[[ -z "$iid" || "$ref" == *"<none>"* ]] && continue
|
|
[[ -n "$KEEP_IMAGES_RE" && "$ref" =~ $KEEP_IMAGES_RE ]] && continue
|
|
lease_active "$lease_until" && continue
|
|
docker image rm "$iid" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
case "$TIER" in
|
|
routine)
|
|
remove_stopped_containers aged
|
|
remove_old_images "{{ gitea_runner_prune_until }}"
|
|
docker volume prune -f \
|
|
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
|
--filter "until={{ gitea_runner_prune_until }}" >/dev/null 2>&1 || true
|
|
docker network prune -f \
|
|
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
|
--filter "until={{ gitea_runner_prune_until }}" >/dev/null 2>&1 || true
|
|
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
|
;;
|
|
pressure)
|
|
remove_stopped_containers aged
|
|
remove_old_images "1h"
|
|
docker volume prune -f \
|
|
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
|
--filter "until=1h" >/dev/null 2>&1 || true
|
|
docker network prune -f \
|
|
--filter "label!=${LEASE_UNTIL_LABEL}" \
|
|
--filter "until=1h" >/dev/null 2>&1 || true
|
|
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
|
;;
|
|
critical)
|
|
# Implements: REQ-3 — age limits dropped, ownership still honored.
|
|
remove_stopped_containers all
|
|
remove_old_images all
|
|
docker volume prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
|
docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
|
docker builder prune -af >/dev/null 2>&1 || true
|
|
;;
|
|
*)
|
|
echo "ERROR: unknown cleanup tier '$TIER' (expected routine|pressure|critical)" >&2
|
|
exit 2
|
|
;;
|
|
esac
|