Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 grm-ci-bot
|
7c9ff0a694 | ||
 kireto
|
2e14bc3141 | ||
|
gitea-actions-bot
|
148c9d3991 | ||
|
kireto
|
8d3e2e03f6 | ||
|
gitea-actions-bot
|
f7948cace7 | ||
|
kireto
|
df6bb2aaed | ||
|
gitea-actions-bot
|
ca780c4f9a |
@@ -283,7 +283,7 @@ jobs:
|
||||
needs.validate.result == 'success'
|
||||
runs-on: docker
|
||||
container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-base:latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 50
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -321,7 +321,7 @@ jobs:
|
||||
run: |
|
||||
. .venv/bin/activate 2>/dev/null || true
|
||||
# Poll commit status until all required checks pass or fail
|
||||
MAX_WAIT=600 # 10 minutes
|
||||
MAX_WAIT=2400 # 40 minutes — covers the ~25-min molecule suite
|
||||
ELAPSED=0
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
STATUS=$(curl -s -H "Authorization: token $CI_GITEA_API_TOKEN" \
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [0.23.3] - 2026-09-18
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Raise auto-merge molecule wait to cover suite duration
|
||||
- Harden stall-detection enumeration and timestamp parsing
|
||||
|
||||
## [0.23.2] - 2026-09-15
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Why GRM?
|
||||
|
||||
|
||||
@@ -50,6 +50,11 @@ gitea_runner_healthcheck_disk_threshold: 70
|
||||
gitea_runner_healthcheck_disk_critical: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# CI job containers older than this many minutes get an exec-responsiveness
|
||||
# probe; a timeout writes one diagnostics bundle per container for
|
||||
# post-mortem analysis of recurring ~20min exec/archive stalls (GRM-168).
|
||||
gitea_runner_stall_minutes: 15
|
||||
|
||||
# Auto-recovery: when the healthcheck detects an unregistered runner, it
|
||||
# can automatically re-register if a Gitea API token is provided.
|
||||
# The token needs admin or org-level access to fetch registration tokens.
|
||||
|
||||
@@ -30,6 +30,50 @@ if [[ -n "$stuck_containers" ]]; then
|
||||
echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 1c. Detect stalled CI job containers — Implements: REQ-1..REQ-4 (GRM-168)
|
||||
# act_runner exec/archive calls into long-running job containers have
|
||||
# repeatedly timed out ~20min into jobs while the daemon stayed up.
|
||||
# Probe exec responsiveness on aged job containers and, on timeout,
|
||||
# write one diagnostics bundle per container for post-mortem analysis.
|
||||
STALL_MINUTES={{ gitea_runner_stall_minutes }}
|
||||
DIAG_DIR="{{ gitea_runner_config_dir }}"
|
||||
now_epoch=$(date +%s)
|
||||
# Implements: REQ-1 — guard the enumeration: a slow/dead daemon must not
|
||||
# abort the healthcheck under pipefail; an empty list just skips probing.
|
||||
# Implements: REQ-2 — pipe-separate fields: CreatedAt contains spaces, so
|
||||
# whitespace-splitting `read` only captured the date and broke the age gate.
|
||||
{ timeout 15 docker ps --filter "name=GITEA-ACTIONS-TASK" \
|
||||
--format '{% raw %}{{.ID}}|{{.Names}}|{{.CreatedAt}}{% endraw %}' 2>/dev/null || true; } \
|
||||
| while IFS='|' read -r cid cname ccreated _rest; do
|
||||
# GNU date rejects the redundant " +0000 UTC" suffix — drop it.
|
||||
created_epoch=$(date -d "${ccreated% UTC}" +%s 2>/dev/null || echo 0)
|
||||
age_min=$(( (now_epoch - created_epoch) / 60 ))
|
||||
[[ "$age_min" -lt "$STALL_MINUTES" ]] && continue
|
||||
marker="$DIAG_DIR/.stall-diag-$cid"
|
||||
[[ -f "$marker" ]] && continue
|
||||
if ! timeout 10 docker exec "$cid" true 2>/dev/null; then
|
||||
diag="$DIAG_DIR/stall-diag-$cname-$(date +%Y%m%dT%H%M%S).log"
|
||||
{
|
||||
echo "=== stall diagnostics for $cname ($cid), age ${age_min}m ==="
|
||||
echo "--- exec probe: TIMEOUT (>10s) ---"
|
||||
echo "--- docker inspect ---"
|
||||
# Implements: REQ-3 — full inspect, but redact the Env block:
|
||||
# job containers carry CI tokens in env vars; the bundle must
|
||||
# not become a secret-material artifact.
|
||||
timeout 15 docker inspect "$cid" 2>/dev/null \
|
||||
| sed -E 's/("[^"]*(TOKEN|PASSWORD|SECRET|KEY)[^=]*=)[^",]*/\1<redacted>/Ig'
|
||||
echo "--- docker top ---"
|
||||
timeout 15 docker top "$cid" 2>/dev/null
|
||||
echo "--- docker stats --no-stream ---"
|
||||
timeout 15 docker stats --no-stream "$cid" 2>/dev/null
|
||||
echo "--- docker events --since 30m ---"
|
||||
timeout 15 docker events --since 30m --until 0s 2>/dev/null | tail -50
|
||||
} > "$diag" 2>&1 || true
|
||||
touch "$marker"
|
||||
echo "WARN: job container $cname unresponsive to exec (${age_min}m old) — diagnostics at $diag"
|
||||
fi
|
||||
done
|
||||
|
||||
# 2. Check gitea-runner service is active
|
||||
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
||||
if [[ "$runner_state" != "active" ]]; then
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
+72
-9
@@ -1,21 +1,84 @@
|
||||
# GRM-168: Bump devx to v0.51.9
|
||||
# GRM-168: Capture dockerd diagnostics when a CI job container stalls
|
||||
|
||||
## Problem
|
||||
grm pins devx@v0.51.0 which rejects `deps:` as a conventional commit type,
|
||||
causing post-merge CI failures on dependency bump commits.
|
||||
|
||||
Recurring CI failures (5+ times on 2026-09-17/18, infra runs 5913, 5925,
|
||||
5943, 5955 notify-sso-bridge): ~20 min into a long-running job,
|
||||
act_runner's API calls into the job container (`docker exec`, archive
|
||||
fetch of `/var/run/act/workflow/*.txt`) time out with
|
||||
`docker daemon ping during version negotiation failed /
|
||||
context deadline exceeded` — killing the job.
|
||||
|
||||
Established facts:
|
||||
|
||||
- Host rootless dockerd never restarted (all daemons up since Sep 14);
|
||||
the healthcheck's 10 s `docker info` never timed out — the daemon API
|
||||
stayed responsive at daemon level.
|
||||
- No OOM, disk, inode, or load pressure on the host.
|
||||
- The wedge is therefore per-container (shim/exec path), most consistent
|
||||
with attach-stdio backpressure or a containerd-shim event stall — but
|
||||
cannot be confirmed post-mortem because job containers and their
|
||||
dockerd goroutine state are gone by the time anyone looks.
|
||||
|
||||
A `SIGUSR1` dockerd dump is not useful here: it lands in the user
|
||||
journal, which runner users cannot read (2026-08-08 journal-permission
|
||||
incident documented in this file's header comments).
|
||||
|
||||
## Approach
|
||||
REQ-1: Bump devx from v0.51.0 to v0.51.9 in pyproject.toml
|
||||
|
||||
Extend `runner-healthcheck.sh.j2` with a stall-detection section that
|
||||
runs after the daemon liveness check. On every healthcheck tick (2 min):
|
||||
|
||||
REQ-1: For each running `GITEA-ACTIONS-TASK-*` container older than
|
||||
`gitea_runner_stall_minutes` (default 15), probe exec responsiveness
|
||||
with `timeout 10 docker exec <id> true`.
|
||||
|
||||
REQ-2: If the probe times out, write a diagnostics bundle to
|
||||
`{{ gitea_runner_config_dir }}/stall-diag-<container>-<timestamp>.log`
|
||||
containing: probe result, `docker inspect` output (State, OOMKilled,
|
||||
Pid, finished/started times), `docker top` output, `docker stats
|
||||
--no-stream` for the container, and `docker events --since 30m` output.
|
||||
Each line prefixed with the container name for grepability.
|
||||
|
||||
REQ-3: Cooldown per container — write at most one diagnostics bundle
|
||||
per container id (marker file under the same dir), so a 2-minute
|
||||
healthcheck does not spam dumps on a persistent stall.
|
||||
|
||||
REQ-4: Do not kill or restart anything — diagnostics only. The job may
|
||||
recover on its own; if it does not, the captured evidence isolates
|
||||
shim-vs-daemon and stream-vs-exec for the follow-up fix.
|
||||
|
||||
## Files Affected
|
||||
|
||||
- `ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2` (extend)
|
||||
- `ansible/roles/gitea_runner/defaults/main.yml` (add `gitea_runner_stall_minutes`)
|
||||
- `docs/specs/GRM-168.md` (new)
|
||||
|
||||
## Test Plan
|
||||
- `make lint-all` passes
|
||||
- `make pytest-cov` passes
|
||||
|
||||
- `make lint-all` (ansible-lint + shellcheck-adjacent linters) passes.
|
||||
- Molecule fast-converge on the gitea_runner role scenario that deploys
|
||||
the healthcheck template (template renders without error).
|
||||
- Manual trace: the new section only touches containers matching
|
||||
`GITEA-ACTIONS-TASK-*` older than the threshold; a stalled exec probe
|
||||
writes exactly one bundle per container.
|
||||
|
||||
## Deploy Plan
|
||||
- Merge to master
|
||||
|
||||
Merge via auto-merge → GRM release → infra picks up the new version via
|
||||
the automated dependency PR. Runner hosts get the updated healthcheck on
|
||||
the next `gitea_runner` role apply (nightly or manual run).
|
||||
|
||||
## Rollback Plan
|
||||
- Revert the merge commit
|
||||
|
||||
Revert the template change — the healthcheck returns to the previous
|
||||
probe set. The diagnostics path is additive; removing it risks nothing.
|
||||
|
||||
## Acceptance Criteria
|
||||
- [x] REQ-1: Bump devx from v0.51.0 to v0.51.9 in pyproject.toml
|
||||
|
||||
- [x] Stalled job containers probed via `timeout docker exec`.
|
||||
- [x] One diagnostics bundle per stalled container, written to the
|
||||
runner config dir (readable without journal access).
|
||||
- [x] Per-container cooldown prevents dump spam.
|
||||
- [x] Nothing is killed/restarted — diagnostics only.
|
||||
- [x] `make lint-all` passes.
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# GRM-169: Fix auto-merge timeout — molecule wait exceeds 10-min job cap
|
||||
|
||||
## Problem
|
||||
|
||||
The `auto-merge` job in `ci.yml` polls molecule-tests status with
|
||||
`MAX_WAIT=600` (10 minutes) inside a job capped at
|
||||
`timeout-minutes: 10`. The molecule suite takes ~25 minutes under the
|
||||
4-runner distribution. Result: every `pull_request` synchronize run of
|
||||
auto-merge exhausts MAX_WAIT, prints `Timed out waiting for molecule
|
||||
tests`, and fails — observed on PR #275 (run 5988) where all molecule
|
||||
jobs were green but auto-merge died before they finished. The merge
|
||||
only completed via a manual rerun-failed-jobs call after molecule was
|
||||
already green.
|
||||
|
||||
## Approach
|
||||
|
||||
REQ-1: Raise the molecule wait budget in `.gitea/workflows/ci.yml` so it
|
||||
exceeds the observed suite duration: `MAX_WAIT=2400` (40 minutes — ~1.6x
|
||||
the observed 25-minute suite) and the job `timeout-minutes` to `50`
|
||||
(wait budget plus setup/post overhead).
|
||||
|
||||
REQ-2: No other behavior changes — the wait loop, success/failure/skipped
|
||||
classification, and merge semantics stay identical. The job still runs
|
||||
on every pull_request event; it simply no longer aborts early.
|
||||
|
||||
## Test Plan
|
||||
|
||||
- `make workflow-lint` (actionlint) passes on the edited file.
|
||||
- `make workflow-dryrun` where available.
|
||||
- Next PR's auto-merge run waits past the 10-minute mark and merges
|
||||
after molecule turns green (verified on a subsequent PR).
|
||||
|
||||
## Deploy Plan
|
||||
|
||||
Merge via auto-merge — ironically exercised by this very PR's auto-merge
|
||||
run: it must wait for this PR's own molecule jobs, demonstrating the fix
|
||||
in production immediately.
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Revert the two changed lines. Risk of keeping the fix: none — a longer
|
||||
wait can only extend a job that was previously guaranteed to fail.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `MAX_WAIT` raised to 2400 in the auto-merge wait loop.
|
||||
- [x] `timeout-minutes` raised to 50 on the auto-merge job.
|
||||
- [x] `make workflow-lint` passes.
|
||||
@@ -0,0 +1,73 @@
|
||||
# GRM-170: Fix stall-detection robustness bugs in runner healthcheck
|
||||
|
||||
## Problem
|
||||
|
||||
Post-merge review of the GRM-168 stall-detection block in
|
||||
`runner-healthcheck.sh.j2` found three defects:
|
||||
|
||||
1. **Missing `|| true` on the container enumeration.** `timeout 15
|
||||
docker ps … | while …` runs under `set -euo pipefail`. If the daemon
|
||||
is unresponsive — precisely the condition the section exists to
|
||||
diagnose — `docker ps` exits nonzero, pipefail propagates it, and the
|
||||
healthcheck dies mid-run before reaching the runner-service check.
|
||||
Every other docker call in the script is guarded; this one is not.
|
||||
|
||||
2. **CreatedAt split bug.** `docker ps --format '{{.ID}} {{.Names}}
|
||||
{{.CreatedAt}}'` emits a timestamp containing spaces
|
||||
(`2026-09-18 10:30:00 +0000 UTC`), but `read -r cid cname ccreated
|
||||
_rest` only captures `2026-09-18` — the date part. `date -d` then
|
||||
computes age from midnight: containers created today always appear
|
||||
≥N hours old, so the 15-minute gate effectively never filters.
|
||||
|
||||
3. **`head -200` truncates `docker inspect`.** Inspect output is ~300+
|
||||
lines and the `State` block (OOMKilled, Pid, times) the spec requires
|
||||
can be cut off.
|
||||
|
||||
## Approach
|
||||
|
||||
REQ-1: Wrap the enumeration so a failed `docker ps` yields empty input
|
||||
instead of aborting the script: `{ timeout 15 docker ps … || true; } |
|
||||
while …`.
|
||||
|
||||
REQ-2: Emit fields separated by `|` (`{{.ID}}|{{.Names}}|{{.CreatedAt}}`)
|
||||
and parse with `IFS='|' read -r cid cname ccreated _rest` so the full
|
||||
timestamp reaches `date -d`; also strip the redundant ` UTC` suffix
|
||||
because GNU date rejects `+0000 UTC` together. The age gate then
|
||||
compares real minutes.
|
||||
|
||||
REQ-3: Remove the `head -200` truncation on `docker inspect` output so
|
||||
the full State block is captured — but pipe through a `sed` filter that
|
||||
redacts the value of any env entry whose name contains TOKEN, PASSWORD,
|
||||
SECRET, or KEY. Job containers carry CI tokens in their Env block; the
|
||||
diagnostics bundle must not become a secret-material artifact
|
||||
(OBL-INFRA-548 S02).
|
||||
|
||||
REQ-4: Diagnostics-only constraint unchanged — no kills, no restarts.
|
||||
|
||||
## Test Plan
|
||||
|
||||
- Render the template and run `bash -n` on the output.
|
||||
- Shell-simulate: feed a fake `docker ps` line with spaced CreatedAt and
|
||||
verify `date -d` computes minutes correctly (manual check).
|
||||
- `make lint-all` (ansible-lint, actionlint, ruff) passes.
|
||||
- Molecule gitea_runner scenario converges with the template change.
|
||||
|
||||
## Deploy Plan
|
||||
|
||||
Merge via auto-merge → release (fix: commit bumps patch) → infra
|
||||
dependency-bump PR picks up the new role version → runner role applied
|
||||
on next infra run. This PR also carries the merged-but-unreleased
|
||||
GRM-168 healthcheck into the release.
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Revert the three-line change set; the section degrades to the GRM-168
|
||||
behavior (still diagnostics-only, just less robust).
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `docker ps` enumeration guarded against nonzero exit.
|
||||
- [x] Full CreatedAt timestamp parsed via `|` separator.
|
||||
- [x] `docker inspect` captured without truncation.
|
||||
- [x] Rendered script passes `bash -n`.
|
||||
- [x] `make lint-all` passes.
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
"""Gitea Runner Manager — lean CLI for managing Gitea Actions runners."""
|
||||
|
||||
__version__ = "0.23.2"
|
||||
__version__ = "0.23.3"
|
||||
|
||||
Reference in New Issue
Block a user