chore: update badge URLs to commit 08a663dc [skip ci]

.gitea/workflows/ci.yml

@@ -283,7 +283,7 @@ jobs:
       needs.validate.result == 'success'
     runs-on: docker
     container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-base:latest
-    timeout-minutes: 10
+    timeout-minutes: 50
     defaults:
       run:
         shell: bash
@@ -321,7 +321,7 @@ jobs:
         run: |
           . .venv/bin/activate 2>/dev/null || true
           # Poll commit status until all required checks pass or fail
-          MAX_WAIT=600  # 10 minutes
+          MAX_WAIT=2400  # 40 minutes — covers the ~25-min molecule suite
           ELAPSED=0
           while [ $ELAPSED -lt $MAX_WAIT ]; do
             STATUS=$(curl -s -H "Authorization: token $CI_GITEA_API_TOKEN" \

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.23.3] - 2026-09-18
+
+### Bug Fixes
+
+- Raise auto-merge molecule wait to cover suite duration
+- Harden stall-detection enumeration and timestamp parsing
+
 ## [0.23.2] - 2026-09-15
 
 ### Bug Fixes

README.md

@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
 
 [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
 [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
-[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
+[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
-[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
+[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
-[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/python.svg)](https://www.python.org/downloads/)
+[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/python.svg)](https://www.python.org/downloads/)
 
 ## Why GRM?
 

ansible/roles/gitea_runner/defaults/main.yml

@@ -50,6 +50,11 @@ gitea_runner_healthcheck_disk_threshold: 70
 gitea_runner_healthcheck_disk_critical: 75
 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
 
+# CI job containers older than this many minutes get an exec-responsiveness
+# probe; a timeout writes one diagnostics bundle per container for
+# post-mortem analysis of recurring ~20min exec/archive stalls (GRM-168).
+gitea_runner_stall_minutes: 15
+
 # Auto-recovery: when the healthcheck detects an unregistered runner, it
 # can automatically re-register if a Gitea API token is provided.
 # The token needs admin or org-level access to fetch registration tokens.

ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2

@@ -30,6 +30,50 @@ if [[ -n "$stuck_containers" ]]; then
     echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
 fi
 
+# 1c. Detect stalled CI job containers — Implements: REQ-1..REQ-4 (GRM-168)
+#     act_runner exec/archive calls into long-running job containers have
+#     repeatedly timed out ~20min into jobs while the daemon stayed up.
+#     Probe exec responsiveness on aged job containers and, on timeout,
+#     write one diagnostics bundle per container for post-mortem analysis.
+STALL_MINUTES={{ gitea_runner_stall_minutes }}
+DIAG_DIR="{{ gitea_runner_config_dir }}"
+now_epoch=$(date +%s)
+# Implements: REQ-1 — guard the enumeration: a slow/dead daemon must not
+# abort the healthcheck under pipefail; an empty list just skips probing.
+# Implements: REQ-2 — pipe-separate fields: CreatedAt contains spaces, so
+# whitespace-splitting `read` only captured the date and broke the age gate.
+{ timeout 15 docker ps --filter "name=GITEA-ACTIONS-TASK" \
+    --format '{% raw %}{{.ID}}|{{.Names}}|{{.CreatedAt}}{% endraw %}' 2>/dev/null || true; } \
+    | while IFS='|' read -r cid cname ccreated _rest; do
+    # GNU date rejects the redundant " +0000 UTC" suffix — drop it.
+    created_epoch=$(date -d "${ccreated% UTC}" +%s 2>/dev/null || echo 0)
+    age_min=$(( (now_epoch - created_epoch) / 60 ))
+    [[ "$age_min" -lt "$STALL_MINUTES" ]] && continue
+    marker="$DIAG_DIR/.stall-diag-$cid"
+    [[ -f "$marker" ]] && continue
+    if ! timeout 10 docker exec "$cid" true 2>/dev/null; then
+        diag="$DIAG_DIR/stall-diag-$cname-$(date +%Y%m%dT%H%M%S).log"
+        {
+            echo "=== stall diagnostics for $cname ($cid), age ${age_min}m ==="
+            echo "--- exec probe: TIMEOUT (>10s) ---"
+            echo "--- docker inspect ---"
+            # Implements: REQ-3 — full inspect, but redact the Env block:
+            # job containers carry CI tokens in env vars; the bundle must
+            # not become a secret-material artifact.
+            timeout 15 docker inspect "$cid" 2>/dev/null \
+                | sed -E 's/("[^"]*(TOKEN|PASSWORD|SECRET|KEY)[^=]*=)[^",]*/\1<redacted>/Ig'
+            echo "--- docker top ---"
+            timeout 15 docker top "$cid" 2>/dev/null
+            echo "--- docker stats --no-stream ---"
+            timeout 15 docker stats --no-stream "$cid" 2>/dev/null
+            echo "--- docker events --since 30m ---"
+            timeout 15 docker events --since 30m --until 0s 2>/dev/null | tail -50
+        } > "$diag" 2>&1 || true
+        touch "$marker"
+        echo "WARN: job container $cname unresponsive to exec (${age_min}m old) — diagnostics at $diag"
+    fi
+done
+
 # 2. Check gitea-runner service is active
 runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
 if [[ "$runner_state" != "active" ]]; then

docs/index.md

@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
 
 [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
 [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
-[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
+[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
-[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
+[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
-[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/8d3e7baa433114be3568d0f351576347b8f9d862/python.svg)](https://www.python.org/downloads/)
+[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/08a663dcbf10dade948ff96bddb89647c210e77e/python.svg)](https://www.python.org/downloads/)
 
 ## Overview
 

docs/specs/GRM-168.md

@@ -1,21 +1,84 @@
-# GRM-168: Bump devx to v0.51.9
+# GRM-168: Capture dockerd diagnostics when a CI job container stalls
 
 ## Problem
-grm pins devx@v0.51.0 which rejects `deps:` as a conventional commit type,
+
-causing post-merge CI failures on dependency bump commits.
+Recurring CI failures (5+ times on 2026-09-17/18, infra runs 5913, 5925,
+5943, 5955 notify-sso-bridge): ~20 min into a long-running job,
+act_runner's API calls into the job container (`docker exec`, archive
+fetch of `/var/run/act/workflow/*.txt`) time out with
+`docker daemon ping during version negotiation failed /
+context deadline exceeded` — killing the job.
+
+Established facts:
+
+- Host rootless dockerd never restarted (all daemons up since Sep 14);
+  the healthcheck's 10 s `docker info` never timed out — the daemon API
+  stayed responsive at daemon level.
+- No OOM, disk, inode, or load pressure on the host.
+- The wedge is therefore per-container (shim/exec path), most consistent
+  with attach-stdio backpressure or a containerd-shim event stall — but
+  cannot be confirmed post-mortem because job containers and their
+  dockerd goroutine state are gone by the time anyone looks.
+
+A `SIGUSR1` dockerd dump is not useful here: it lands in the user
+journal, which runner users cannot read (2026-08-08 journal-permission
+incident documented in this file's header comments).
 
 ## Approach
-REQ-1: Bump devx from v0.51.0 to v0.51.9 in pyproject.toml
+
+Extend `runner-healthcheck.sh.j2` with a stall-detection section that
+runs after the daemon liveness check. On every healthcheck tick (2 min):
+
+REQ-1: For each running `GITEA-ACTIONS-TASK-*` container older than
+`gitea_runner_stall_minutes` (default 15), probe exec responsiveness
+with `timeout 10 docker exec <id> true`.
+
+REQ-2: If the probe times out, write a diagnostics bundle to
+`{{ gitea_runner_config_dir }}/stall-diag-<container>-<timestamp>.log`
+containing: probe result, `docker inspect` output (State, OOMKilled,
+Pid, finished/started times), `docker top` output, `docker stats
+--no-stream` for the container, and `docker events --since 30m` output.
+Each line prefixed with the container name for grepability.
+
+REQ-3: Cooldown per container — write at most one diagnostics bundle
+per container id (marker file under the same dir), so a 2-minute
+healthcheck does not spam dumps on a persistent stall.
+
+REQ-4: Do not kill or restart anything — diagnostics only. The job may
+recover on its own; if it does not, the captured evidence isolates
+shim-vs-daemon and stream-vs-exec for the follow-up fix.
+
+## Files Affected
+
+- `ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2` (extend)
+- `ansible/roles/gitea_runner/defaults/main.yml` (add `gitea_runner_stall_minutes`)
+- `docs/specs/GRM-168.md` (new)
 
 ## Test Plan
-- `make lint-all` passes
+
-- `make pytest-cov` passes
+- `make lint-all` (ansible-lint + shellcheck-adjacent linters) passes.
+- Molecule fast-converge on the gitea_runner role scenario that deploys
+  the healthcheck template (template renders without error).
+- Manual trace: the new section only touches containers matching
+  `GITEA-ACTIONS-TASK-*` older than the threshold; a stalled exec probe
+  writes exactly one bundle per container.
 
 ## Deploy Plan
-- Merge to master
+
+Merge via auto-merge → GRM release → infra picks up the new version via
+the automated dependency PR. Runner hosts get the updated healthcheck on
+the next `gitea_runner` role apply (nightly or manual run).
 
 ## Rollback Plan
-- Revert the merge commit
+
+Revert the template change — the healthcheck returns to the previous
+probe set. The diagnostics path is additive; removing it risks nothing.
 
 ## Acceptance Criteria
-- [x] REQ-1: Bump devx from v0.51.0 to v0.51.9 in pyproject.toml
+
+- [x] Stalled job containers probed via `timeout docker exec`.
+- [x] One diagnostics bundle per stalled container, written to the
+      runner config dir (readable without journal access).
+- [x] Per-container cooldown prevents dump spam.
+- [x] Nothing is killed/restarted — diagnostics only.
+- [x] `make lint-all` passes.

docs/specs/GRM-169.md

@@ -0,0 +1,48 @@
+# GRM-169: Fix auto-merge timeout — molecule wait exceeds 10-min job cap
+
+## Problem
+
+The `auto-merge` job in `ci.yml` polls molecule-tests status with
+`MAX_WAIT=600` (10 minutes) inside a job capped at
+`timeout-minutes: 10`. The molecule suite takes ~25 minutes under the
+4-runner distribution. Result: every `pull_request` synchronize run of
+auto-merge exhausts MAX_WAIT, prints `Timed out waiting for molecule
+tests`, and fails — observed on PR #275 (run 5988) where all molecule
+jobs were green but auto-merge died before they finished. The merge
+only completed via a manual rerun-failed-jobs call after molecule was
+already green.
+
+## Approach
+
+REQ-1: Raise the molecule wait budget in `.gitea/workflows/ci.yml` so it
+exceeds the observed suite duration: `MAX_WAIT=2400` (40 minutes — ~1.6x
+the observed 25-minute suite) and the job `timeout-minutes` to `50`
+(wait budget plus setup/post overhead).
+
+REQ-2: No other behavior changes — the wait loop, success/failure/skipped
+classification, and merge semantics stay identical. The job still runs
+on every pull_request event; it simply no longer aborts early.
+
+## Test Plan
+
+- `make workflow-lint` (actionlint) passes on the edited file.
+- `make workflow-dryrun` where available.
+- Next PR's auto-merge run waits past the 10-minute mark and merges
+  after molecule turns green (verified on a subsequent PR).
+
+## Deploy Plan
+
+Merge via auto-merge — ironically exercised by this very PR's auto-merge
+run: it must wait for this PR's own molecule jobs, demonstrating the fix
+in production immediately.
+
+## Rollback Plan
+
+Revert the two changed lines. Risk of keeping the fix: none — a longer
+wait can only extend a job that was previously guaranteed to fail.
+
+## Acceptance Criteria
+
+- [x] `MAX_WAIT` raised to 2400 in the auto-merge wait loop.
+- [x] `timeout-minutes` raised to 50 on the auto-merge job.
+- [x] `make workflow-lint` passes.

docs/specs/GRM-170.md

@@ -0,0 +1,73 @@
+# GRM-170: Fix stall-detection robustness bugs in runner healthcheck
+
+## Problem
+
+Post-merge review of the GRM-168 stall-detection block in
+`runner-healthcheck.sh.j2` found three defects:
+
+1. **Missing `|| true` on the container enumeration.** `timeout 15
+   docker ps … | while …` runs under `set -euo pipefail`. If the daemon
+   is unresponsive — precisely the condition the section exists to
+   diagnose — `docker ps` exits nonzero, pipefail propagates it, and the
+   healthcheck dies mid-run before reaching the runner-service check.
+   Every other docker call in the script is guarded; this one is not.
+
+2. **CreatedAt split bug.** `docker ps --format '{{.ID}} {{.Names}}
+   {{.CreatedAt}}'` emits a timestamp containing spaces
+   (`2026-09-18 10:30:00 +0000 UTC`), but `read -r cid cname ccreated
+   _rest` only captures `2026-09-18` — the date part. `date -d` then
+   computes age from midnight: containers created today always appear
+   ≥N hours old, so the 15-minute gate effectively never filters.
+
+3. **`head -200` truncates `docker inspect`.** Inspect output is ~300+
+   lines and the `State` block (OOMKilled, Pid, times) the spec requires
+   can be cut off.
+
+## Approach
+
+REQ-1: Wrap the enumeration so a failed `docker ps` yields empty input
+instead of aborting the script: `{ timeout 15 docker ps … || true; } |
+while …`.
+
+REQ-2: Emit fields separated by `|` (`{{.ID}}|{{.Names}}|{{.CreatedAt}}`)
+and parse with `IFS='|' read -r cid cname ccreated _rest` so the full
+timestamp reaches `date -d`; also strip the redundant ` UTC` suffix
+because GNU date rejects `+0000 UTC` together. The age gate then
+compares real minutes.
+
+REQ-3: Remove the `head -200` truncation on `docker inspect` output so
+the full State block is captured — but pipe through a `sed` filter that
+redacts the value of any env entry whose name contains TOKEN, PASSWORD,
+SECRET, or KEY. Job containers carry CI tokens in their Env block; the
+diagnostics bundle must not become a secret-material artifact
+(OBL-INFRA-548 S02).
+
+REQ-4: Diagnostics-only constraint unchanged — no kills, no restarts.
+
+## Test Plan
+
+- Render the template and run `bash -n` on the output.
+- Shell-simulate: feed a fake `docker ps` line with spaced CreatedAt and
+  verify `date -d` computes minutes correctly (manual check).
+- `make lint-all` (ansible-lint, actionlint, ruff) passes.
+- Molecule gitea_runner scenario converges with the template change.
+
+## Deploy Plan
+
+Merge via auto-merge → release (fix: commit bumps patch) → infra
+dependency-bump PR picks up the new role version → runner role applied
+on next infra run. This PR also carries the merged-but-unreleased
+GRM-168 healthcheck into the release.
+
+## Rollback Plan
+
+Revert the three-line change set; the section degrades to the GRM-168
+behavior (still diagnostics-only, just less robust).
+
+## Acceptance Criteria
+
+- [x] `docker ps` enumeration guarded against nonzero exit.
+- [x] Full CreatedAt timestamp parsed via `|` separator.
+- [x] `docker inspect` captured without truncation.
+- [x] Rendered script passes `bash -n`.
+- [x] `make lint-all` passes.

src/grm/__init__.py

@@ -1,3 +1,3 @@
 """Gitea Runner Manager — lean CLI for managing Gitea Actions runners."""
 
-__version__ = "0.23.2"
+__version__ = "0.23.3"