GRM-168: docs: fix vale quote punctuation in spec
This commit was merged in pull request #275.
This commit is contained in:
@@ -50,6 +50,11 @@ gitea_runner_healthcheck_disk_threshold: 70
|
||||
gitea_runner_healthcheck_disk_critical: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# CI job containers older than this many minutes get an exec-responsiveness
|
||||
# probe; a timeout writes one diagnostics bundle per container for
|
||||
# post-mortem analysis of recurring ~20min exec/archive stalls (GRM-168).
|
||||
gitea_runner_stall_minutes: 15
|
||||
|
||||
# Auto-recovery: when the healthcheck detects an unregistered runner, it
|
||||
# can automatically re-register if a Gitea API token is provided.
|
||||
# The token needs admin or org-level access to fetch registration tokens.
|
||||
|
||||
@@ -30,6 +30,41 @@ if [[ -n "$stuck_containers" ]]; then
|
||||
echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 1c. Detect stalled CI job containers — Implements: REQ-1..REQ-4 (GRM-168)
|
||||
# act_runner exec/archive calls into long-running job containers have
|
||||
# repeatedly timed out ~20min into jobs while the daemon stayed up.
|
||||
# Probe exec responsiveness on aged job containers and, on timeout,
|
||||
# write one diagnostics bundle per container for post-mortem analysis.
|
||||
STALL_MINUTES={{ gitea_runner_stall_minutes }}
|
||||
DIAG_DIR="{{ gitea_runner_config_dir }}"
|
||||
now_epoch=$(date +%s)
|
||||
timeout 15 docker ps --filter "name=GITEA-ACTIONS-TASK" \
|
||||
--format '{% raw %}{{.ID}} {{.Names}} {{.CreatedAt}}{% endraw %}' 2>/dev/null \
|
||||
| while read -r cid cname ccreated _rest; do
|
||||
created_epoch=$(date -d "$ccreated" +%s 2>/dev/null || echo 0)
|
||||
age_min=$(( (now_epoch - created_epoch) / 60 ))
|
||||
[[ "$age_min" -lt "$STALL_MINUTES" ]] && continue
|
||||
marker="$DIAG_DIR/.stall-diag-$cid"
|
||||
[[ -f "$marker" ]] && continue
|
||||
if ! timeout 10 docker exec "$cid" true 2>/dev/null; then
|
||||
diag="$DIAG_DIR/stall-diag-$cname-$(date +%Y%m%dT%H%M%S).log"
|
||||
{
|
||||
echo "=== stall diagnostics for $cname ($cid), age ${age_min}m ==="
|
||||
echo "--- exec probe: TIMEOUT (>10s) ---"
|
||||
echo "--- docker inspect ---"
|
||||
timeout 15 docker inspect "$cid" 2>/dev/null | head -200
|
||||
echo "--- docker top ---"
|
||||
timeout 15 docker top "$cid" 2>/dev/null
|
||||
echo "--- docker stats --no-stream ---"
|
||||
timeout 15 docker stats --no-stream "$cid" 2>/dev/null
|
||||
echo "--- docker events --since 30m ---"
|
||||
timeout 15 docker events --since 30m --until 0s 2>/dev/null | tail -50
|
||||
} > "$diag" 2>&1 || true
|
||||
touch "$marker"
|
||||
echo "WARN: job container $cname unresponsive to exec (${age_min}m old) — diagnostics at $diag"
|
||||
fi
|
||||
done
|
||||
|
||||
# 2. Check gitea-runner service is active
|
||||
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
||||
if [[ "$runner_state" != "active" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user