50 lines
2.0 KiB
Django/Jinja
50 lines
2.0 KiB
Django/Jinja
#!/bin/bash
|
|
# Health check for gitea-runner: verifies Docker daemon and runner service.
|
|
# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
|
|
set -euo pipefail
|
|
|
|
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
|
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
|
export DOCKER_HOST XDG_RUNTIME_DIR
|
|
|
|
# 1. Check Docker daemon responsiveness
|
|
if ! docker info >/dev/null 2>&1; then
|
|
echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}"
|
|
systemctl --user restart docker.service
|
|
sleep 3
|
|
if ! docker info >/dev/null 2>&1; then
|
|
echo "CRITICAL: Docker daemon still down after restart"
|
|
exit 1
|
|
fi
|
|
echo "RECOVERED: Docker daemon restarted successfully"
|
|
fi
|
|
|
|
# 2. Check gitea-runner service is active
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "ERROR: gitea-runner service is ${runner_state}, restarting"
|
|
systemctl --user restart gitea-runner.service
|
|
sleep 2
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "CRITICAL: gitea-runner service still down after restart"
|
|
exit 2
|
|
fi
|
|
echo "RECOVERED: gitea-runner service restarted successfully"
|
|
fi
|
|
|
|
# 3. Check disk space — prune aggressively if below threshold
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
|
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
|
|
docker system prune -af --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
|
|
docker volume prune -af --filter "label={{ gitea_runner_prune_label }}" || true
|
|
# Also prune dangling images (no label)
|
|
docker image prune -af || true
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
echo "INFO: Disk usage after prune: ${disk_pct}%"
|
|
fi
|
|
|
|
echo "OK: runner healthy, disk at ${disk_pct}%"
|
|
exit 0
|