GRM-168: docs: fix vale quote punctuation in spec
Post-merge / detect-and-configure (push) Successful in 53s
Post-merge / release-and-maintain (push) Successful in 52s

This commit was merged in pull request #275.
This commit is contained in:
2026-09-18 10:43:49 +00:00
parent ca780c4f9a
commit df6bb2aaed
3 changed files with 112 additions and 9 deletions
@@ -50,6 +50,11 @@ gitea_runner_healthcheck_disk_threshold: 70
gitea_runner_healthcheck_disk_critical: 75
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
# CI job containers older than this many minutes get an exec-responsiveness
# probe; a timeout writes one diagnostics bundle per container for
# post-mortem analysis of recurring ~20min exec/archive stalls (GRM-168).
gitea_runner_stall_minutes: 15
# Auto-recovery: when the healthcheck detects an unregistered runner, it
# can automatically re-register if a Gitea API token is provided.
# The token needs admin or org-level access to fetch registration tokens.
@@ -30,6 +30,41 @@ if [[ -n "$stuck_containers" ]]; then
echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
fi
# 1c. Detect stalled CI job containers — Implements: REQ-1..REQ-4 (GRM-168)
# act_runner exec/archive calls into long-running job containers have
# repeatedly timed out ~20min into jobs while the daemon stayed up.
# Probe exec responsiveness on aged job containers and, on timeout,
# write one diagnostics bundle per container for post-mortem analysis.
STALL_MINUTES={{ gitea_runner_stall_minutes }}
DIAG_DIR="{{ gitea_runner_config_dir }}"
now_epoch=$(date +%s)
timeout 15 docker ps --filter "name=GITEA-ACTIONS-TASK" \
--format '{% raw %}{{.ID}} {{.Names}} {{.CreatedAt}}{% endraw %}' 2>/dev/null \
| while read -r cid cname ccreated _rest; do
created_epoch=$(date -d "$ccreated" +%s 2>/dev/null || echo 0)
age_min=$(( (now_epoch - created_epoch) / 60 ))
[[ "$age_min" -lt "$STALL_MINUTES" ]] && continue
marker="$DIAG_DIR/.stall-diag-$cid"
[[ -f "$marker" ]] && continue
if ! timeout 10 docker exec "$cid" true 2>/dev/null; then
diag="$DIAG_DIR/stall-diag-$cname-$(date +%Y%m%dT%H%M%S).log"
{
echo "=== stall diagnostics for $cname ($cid), age ${age_min}m ==="
echo "--- exec probe: TIMEOUT (>10s) ---"
echo "--- docker inspect ---"
timeout 15 docker inspect "$cid" 2>/dev/null | head -200
echo "--- docker top ---"
timeout 15 docker top "$cid" 2>/dev/null
echo "--- docker stats --no-stream ---"
timeout 15 docker stats --no-stream "$cid" 2>/dev/null
echo "--- docker events --since 30m ---"
timeout 15 docker events --since 30m --until 0s 2>/dev/null | tail -50
} > "$diag" 2>&1 || true
touch "$marker"
echo "WARN: job container $cname unresponsive to exec (${age_min}m old) — diagnostics at $diag"
fi
done
# 2. Check gitea-runner service is active
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
if [[ "$runner_state" != "active" ]]; then