64 lines
3.0 KiB
Django/Jinja
64 lines
3.0 KiB
Django/Jinja
#!/bin/bash
|
|
# Health check for gitea-runner: verifies Docker daemon and runner service.
|
|
# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
|
|
set -euo pipefail
|
|
|
|
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
|
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
|
export DOCKER_HOST XDG_RUNTIME_DIR
|
|
|
|
# 1. Check Docker daemon responsiveness (with timeout — a bare `docker info` can
|
|
# hang indefinitely on a stuck daemon, blocking the healthcheck itself).
|
|
if ! timeout 10 docker info >/dev/null 2>&1; then
|
|
echo "ERROR: Docker daemon not responding at ${DOCKER_HOST} (timed out after 10s)"
|
|
systemctl --user restart docker.service
|
|
sleep 3
|
|
if ! timeout 10 docker info >/dev/null 2>&1; then
|
|
echo "CRITICAL: Docker daemon still down after restart"
|
|
exit 1
|
|
fi
|
|
echo "RECOVERED: Docker daemon restarted successfully"
|
|
fi
|
|
|
|
# 1b. Clean up stuck containers — containers in "removing" or "stopping" state
|
|
# for too long cause "cannot kill container: did not receive an exit event"
|
|
# errors in molecule destroy phases. Force-remove them so subsequent CI jobs
|
|
# don't inherit the stuck state.
|
|
stuck_containers=$(timeout 10 docker ps -a --filter "status=removing" --filter "status=stopping" --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null || true)
|
|
if [[ -n "$stuck_containers" ]]; then
|
|
echo "WARN: Found stuck containers (removing/stopping), force-cleaning"
|
|
echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
|
|
fi
|
|
|
|
# 2. Check gitea-runner service is active
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "ERROR: gitea-runner service is ${runner_state}, restarting"
|
|
systemctl --user restart gitea-runner.service
|
|
sleep 2
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "CRITICAL: gitea-runner service still down after restart"
|
|
exit 2
|
|
fi
|
|
echo "RECOVERED: gitea-runner service restarted successfully"
|
|
fi
|
|
|
|
# 3. Check disk space — prune aggressively if below threshold
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
|
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
|
|
docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
|
|
docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true
|
|
# Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality)
|
|
docker image prune -f || true
|
|
# Clean up stopped containers and dangling networks that accumulate from failed jobs
|
|
docker container prune -f || true
|
|
docker network prune -f || true
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
echo "INFO: Disk usage after prune: ${disk_pct}%"
|
|
fi
|
|
|
|
echo "OK: runner healthy, disk at ${disk_pct}%"
|
|
exit 0
|