269 lines
12 KiB
Django/Jinja
269 lines
12 KiB
Django/Jinja
#!/bin/bash
|
|
# Health check for gitea-runner: verifies Docker daemon and runner service.
|
|
# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
|
|
set -euo pipefail
|
|
|
|
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
|
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
|
export DOCKER_HOST XDG_RUNTIME_DIR
|
|
|
|
# 1. Check Docker daemon responsiveness (with timeout — a bare `docker info` can
|
|
# hang indefinitely on a stuck daemon, blocking the healthcheck itself).
|
|
if ! timeout 10 docker info >/dev/null 2>&1; then
|
|
echo "ERROR: Docker daemon not responding at ${DOCKER_HOST} (timed out after 10s)"
|
|
systemctl --user restart docker.service
|
|
sleep 3
|
|
if ! timeout 10 docker info >/dev/null 2>&1; then
|
|
echo "CRITICAL: Docker daemon still down after restart"
|
|
exit 1
|
|
fi
|
|
echo "RECOVERED: Docker daemon restarted successfully"
|
|
fi
|
|
|
|
# 1b. Clean up stuck containers — containers in "removing" or "stopping" state
|
|
# for too long cause "cannot kill container: did not receive an exit event"
|
|
# errors in molecule destroy phases. Force-remove them so subsequent CI jobs
|
|
# don't inherit the stuck state.
|
|
stuck_containers=$(timeout 10 docker ps -a --filter "status=removing" --filter "status=stopping" --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null || true)
|
|
if [[ -n "$stuck_containers" ]]; then
|
|
echo "WARN: Found stuck containers (removing/stopping), force-cleaning"
|
|
echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
|
|
fi
|
|
|
|
# 2. Check gitea-runner service is active
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "ERROR: gitea-runner service is ${runner_state}, restarting"
|
|
systemctl --user restart gitea-runner.service
|
|
sleep 2
|
|
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
|
if [[ "$runner_state" != "active" ]]; then
|
|
echo "CRITICAL: gitea-runner service still down after restart"
|
|
exit 2
|
|
fi
|
|
echo "RECOVERED: gitea-runner service restarted successfully"
|
|
fi
|
|
|
|
# 2b. Detect unregistered runner state. When Gitea no longer recognizes the
|
|
# runner (e.g., server restore, runner record deleted, Gitea restart with
|
|
# token salt change), the runner logs "unregistered runner" every few seconds.
|
|
# A service restart will not fix this; re-registration is required.
|
|
#
|
|
# Detection method: query the Gitea API to verify the runner's UUID still
|
|
# exists. This is more reliable than parsing journal logs (which requires
|
|
# journal access permissions that runner users may not have — see the
|
|
# 2026-08-08 incident where journalctl --user returned "No journal files
|
|
# were opened due to insufficient permissions" for all runner users,
|
|
# causing the healthcheck to always report "OK: runner healthy" even
|
|
# though all runners were unregistered).
|
|
{% if gitea_runner_auto_recover_api_token %}
|
|
# Auto-recovery is enabled: fetch a new registration token from the Gitea API
|
|
# and re-register the runner automatically. A cooldown prevents infinite loops.
|
|
GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token"
|
|
COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown"
|
|
COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }}
|
|
GITEA_URL="{{ gitea_url }}"
|
|
RUNNER_NAME="{{ gitea_runner_name }}"
|
|
RUNNER_LABELS="{{ gitea_runner_labels }}"
|
|
BINARY="{{ gitea_runner_binary_path }}"
|
|
RUNNER_FILE="{{ gitea_runner_data_dir }}/.runner"
|
|
{% endif %}
|
|
runner_unregistered=0
|
|
|
|
# Primary detection: query the Gitea API to check if the runner's ID
|
|
# still exists in Gitea's runner list. This works regardless of journal
|
|
# permissions.
|
|
{% if gitea_runner_auto_recover_api_token %}
|
|
if [[ -f "$GITEA_API_TOKEN_FILE" && -f "$RUNNER_FILE" ]]; then
|
|
API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
|
|
RUNNER_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
|
|
if [[ -n "$API_TOKEN" && -n "$RUNNER_ID" ]]; then
|
|
# List all runners and check if our ID is present
|
|
runner_found=$(curl -sf --connect-timeout 5 --max-time 10 \
|
|
-H "Authorization: token $API_TOKEN" \
|
|
"${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
runners = data if isinstance(data, list) else data.get('runners', [])
|
|
ids = [str(r.get('id', '')) for r in runners]
|
|
print('1' if '$RUNNER_ID' in ids else '0')
|
|
except Exception:
|
|
print('0')
|
|
" 2>/dev/null || echo "0")
|
|
if [[ "$runner_found" != "1" ]]; then
|
|
runner_unregistered=1
|
|
echo "CRITICAL: runner ID $RUNNER_ID not found in Gitea (unregistered)."
|
|
fi
|
|
fi
|
|
fi
|
|
{% endif %}
|
|
|
|
# Fallback detection: check journal logs (if accessible)
|
|
if [[ "$runner_unregistered" -eq 0 ]]; then
|
|
recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
|
|
if [[ "$recent_errors" -ge 3 ]]; then
|
|
runner_unregistered=1
|
|
echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)."
|
|
fi
|
|
fi
|
|
|
|
if [[ "$runner_unregistered" -ge 1 ]]; then
|
|
{% if gitea_runner_auto_recover_api_token %}
|
|
# Check cooldown — skip if we recently attempted recovery
|
|
if [[ -f "$COOLDOWN_FILE" ]]; then
|
|
last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
|
|
now=$(date +%s)
|
|
elapsed=$((now - last_attempt))
|
|
if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then
|
|
echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later."
|
|
exit 3
|
|
fi
|
|
fi
|
|
|
|
# Mark attempt time BEFORE trying (so failures also get cooldown)
|
|
date +%s > "$COOLDOWN_FILE" 2>/dev/null || true
|
|
|
|
# Read the API token
|
|
if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then
|
|
echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required."
|
|
exit 3
|
|
fi
|
|
API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
|
|
if [[ -z "$API_TOKEN" ]]; then
|
|
echo "ERROR: auto-recover token file is empty. Manual re-registration required."
|
|
exit 3
|
|
fi
|
|
|
|
echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..."
|
|
|
|
# Fetch a new registration token from the Gitea API
|
|
# Try org-level first (for org-scoped runners), then instance-level
|
|
REG_TOKEN=""
|
|
for endpoint in \
|
|
"api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \
|
|
"api/v1/admin/actions/runners/registration-token"; do
|
|
REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \
|
|
-H "Authorization: token $API_TOKEN" \
|
|
"${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
|
|
if [[ -n "$REG_TOKEN" ]]; then
|
|
echo "INFO: fetched registration token from ${endpoint}"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ -z "$REG_TOKEN" ]]; then
|
|
echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?"
|
|
exit 3
|
|
fi
|
|
|
|
# Stop the runner service
|
|
systemctl --user stop gitea-runner.service 2>/dev/null || true
|
|
sleep 1
|
|
|
|
# Remove the stale .runner file
|
|
rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true
|
|
|
|
# Re-register
|
|
cd "{{ gitea_runner_data_dir }}"
|
|
if "$BINARY" register \
|
|
--token "$REG_TOKEN" \
|
|
--name "$RUNNER_NAME" \
|
|
--instance "$GITEA_URL" \
|
|
--labels "$RUNNER_LABELS" \
|
|
--no-interactive 2>&1; then
|
|
echo "RECOVERED: runner re-registered successfully"
|
|
else
|
|
echo "ERROR: re-registration failed. Manual intervention required."
|
|
exit 3
|
|
fi
|
|
|
|
# Start the runner service
|
|
systemctl --user start gitea-runner.service
|
|
sleep 3
|
|
|
|
# Verify recovery — query the Gitea API to confirm the new ID is registered
|
|
NEW_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
|
|
if [[ -n "$NEW_ID" ]]; then
|
|
new_found=$(curl -sf --connect-timeout 5 --max-time 10 \
|
|
-H "Authorization: token $API_TOKEN" \
|
|
"${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
runners = data if isinstance(data, list) else data.get('runners', [])
|
|
ids = [str(r.get('id', '')) for r in runners]
|
|
print('1' if '$NEW_ID' in ids else '0')
|
|
except Exception:
|
|
print('0')
|
|
" 2>/dev/null || echo "0")
|
|
if [[ "$new_found" == "1" ]]; then
|
|
echo "OK: runner recovered and registered with new ID $NEW_ID"
|
|
# Clear cooldown on success
|
|
rm -f "$COOLDOWN_FILE" 2>/dev/null || true
|
|
else
|
|
echo "WARN: runner re-registered but ID not found in Gitea API. Will retry after cooldown."
|
|
exit 3
|
|
fi
|
|
else
|
|
echo "WARN: could not read new .runner file after re-registration. Will retry after cooldown."
|
|
exit 3
|
|
fi
|
|
{% else %}
|
|
echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true."
|
|
# Restart the service once in case it is a transient token refresh issue,
|
|
# but this cannot recover an unregistered runner without re-registration.
|
|
systemctl --user restart gitea-runner.service
|
|
sleep 2
|
|
exit 3
|
|
{% endif %}
|
|
fi
|
|
|
|
# 3. Check disk space — prune aggressively if below threshold
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then
|
|
echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune"
|
|
# Critical level: remove ALL stopped containers (no age filter) and ALL
|
|
# unused images/volumes. The until=1h gentle prune is insufficient here.
|
|
# Stop+rm stale non-CI containers regardless of age (failed molecule tests
|
|
# from the last 59 minutes also consume disk).
|
|
docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
|
|
| grep -v 'GITEA-ACTIONS-TASK' \
|
|
| awk '{print $1}' \
|
|
| xargs -r docker rm -f 2>/dev/null || true
|
|
docker system prune -af --volumes || true
|
|
docker network prune -f || true
|
|
docker builder prune -af || true
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
echo "INFO: Disk usage after full prune: ${disk_pct}%"
|
|
elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
|
echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)"
|
|
# Force-remove stale containers (including running ones from failed molecule tests)
|
|
# that are older than 1 hour. "docker container prune -f" only removes stopped
|
|
# containers, so running containers from crashed CI jobs accumulate and consume
|
|
# disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
|
|
# Only remove containers older than 1 hour to avoid killing molecule test
|
|
# containers that CI jobs are actively using.
|
|
docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \
|
|
| grep -v 'GITEA-ACTIONS-TASK' \
|
|
| grep -E '(hour|day|week|month|year)s? ago' \
|
|
| awk '{print $1}' \
|
|
| xargs -r docker rm -f 2>/dev/null || true
|
|
# Prune images and containers older than 1h (until filter is NOT
|
|
# supported with --volumes, so prune volumes separately without a filter).
|
|
docker image prune -af --filter "until=1h" 2>/dev/null || true
|
|
docker container prune -f --filter "until=1h" 2>/dev/null || true
|
|
docker volume prune -f 2>/dev/null || true
|
|
# Prune networks older than 1 hour to avoid removing networks that
|
|
# molecule tests are actively creating (e.g. 'traefik' network created
|
|
# during molecule create phase before containers are attached).
|
|
docker network prune -f --filter "until=1h" || true
|
|
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
echo "INFO: Disk usage after prune: ${disk_pct}%"
|
|
fi
|
|
|
|
echo "OK: runner healthy, disk at ${disk_pct}%"
|
|
exit 0
|