diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index a02ffb6..aec3ea7 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -112,6 +112,8 @@ - "'docker network prune' in healthcheck_script.content | b64decode" - "'status=removing' in healthcheck_script.content | b64decode" - "'status=stopping' in healthcheck_script.content | b64decode" + - "'status=exited' in healthcheck_script.content | b64decode" + - "'status=dead' in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode" fail_msg: "Healthcheck script template is missing expected content" diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index 3df68a1..7dcd9b3 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -227,9 +227,12 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune" # Critical level: remove ALL stopped containers (no age filter) and ALL # unused images/volumes. The until=1h gentle prune is insufficient here. - # Stop+rm stale non-CI containers regardless of age (failed molecule tests - # from the last 59 minutes also consume disk). - docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ + # Implements: REQ-1 (GRM-167) — only exited/dead containers are removed. + # Running molecule instances are never killed: RunningFor counts creation + # time, so an adopted stale instance looks old; and a running container's + # writable layer is tiny — images/volumes are what actually fills the disk. + docker ps -a --filter "status=exited" --filter "status=dead" \ + --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ | grep -v 'GITEA-ACTIONS-TASK' \ | awk '{print $1}' \ | xargs -r docker rm -f 2>/dev/null || true @@ -240,13 +243,16 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then echo "INFO: Disk usage after full prune: ${disk_pct}%" elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)" - # Force-remove stale containers (including running ones from failed molecule tests) - # that are older than 1 hour. "docker container prune -f" only removes stopped - # containers, so running containers from crashed CI jobs accumulate and consume - # disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). - # Only remove containers older than 1 hour to avoid killing molecule test - # containers that CI jobs are actively using. - docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \ + # Force-remove stale stopped containers older than 1 hour. + # Implements: REQ-1 (GRM-167) — only exited/dead containers are removed. + # A running molecule instance must never be janitor-killed: RunningFor + # measures creation time, so a stale instance restarted by an active run + # looks ">1h old" and would die mid-converge ("No such container", + # infra nightly run 5710). Running leftovers are reused or destroyed by + # the next molecule create/destroy cycle. + # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). + docker ps -a --filter "status=exited" --filter "status=dead" \ + --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \ | grep -v 'GITEA-ACTIONS-TASK' \ | grep -E '(hour|day|week|month|year)s? ago' \ | awk '{print $1}' \ diff --git a/docs/specs/GRM-167-devx-bump-historical.md b/docs/specs/GRM-167-devx-bump-historical.md new file mode 100644 index 0000000..d618a81 --- /dev/null +++ b/docs/specs/GRM-167-devx-bump-historical.md @@ -0,0 +1,21 @@ +# GRM-167: Bump devx to v0.51.0 + +## Problem +devx v0.51.0 released with role defaults path support for create_dependency_pr. grm pins v0.50.2. + +## Approach +Bump devx pin in pyproject.toml. + +REQ-1: Bump devx from v0.50.2 to v0.51.0 in pyproject.toml + +## Test Plan +- Verify CI passes with new devx version + +## Deploy Plan +- Merge to master, auto-release + +## Rollback Plan +- Revert the merge commit + +## Acceptance Criteria +- [x] REQ-1: devx pinned to v0.51.0 in pyproject.toml diff --git a/docs/specs/GRM-167.md b/docs/specs/GRM-167.md index d618a81..2497daa 100644 --- a/docs/specs/GRM-167.md +++ b/docs/specs/GRM-167.md @@ -1,21 +1,51 @@ -# GRM-167: Bump devx to v0.51.0 +# GRM-167: Fix runner healthcheck force-removing active molecule containers ## Problem -devx v0.51.0 released with role defaults path support for create_dependency_pr. grm pins v0.50.2. +`runner-healthcheck.sh` runs every 2 minutes and force-removes molecule +instance containers on two disk paths: + +- **Warn path (disk >= 70%)**: `docker rm -f` any non-`GITEA-ACTIONS-TASK` + container with `RunningFor` >= 1h — same creation-age race fixed in + GRM-166 for the prune service. A stale molecule instance restarted by a + new run still reads >1h old and is killed mid-converge. +- **Critical path (disk >= 75%)**: `docker rm -f` ALL + non-`GITEA-ACTIONS-TASK` containers with no age or status filter — + running molecule instances die instantly. Under parallel DinD load the + runner disk crosses 75% routinely (infra nightly run 5710: `No such + container: ubuntu-2604` across three different runners). ## Approach -Bump devx pin in pyproject.toml. - -REQ-1: Bump devx from v0.50.2 to v0.51.0 in pyproject.toml +REQ-1: In `ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2`, + restrict both disk-pressure removal paths to containers that are not + running: add `--filter "status=exited" --filter "status=created"` is + NOT sufficient for the critical path since a just-created molecule + instance is in `created` state — use `status=exited` and + `status=dead` only. Running containers are never janitor-killed; the + image/volume/system prunes still reclaim the actual disk. +REQ-2: Keep `GITEA-ACTIONS-TASK` exclusion, the 1h age gate on the warn + path, and all image/volume/network/builder prunes unchanged. +REQ-3: Update comments documenting the running-container race. +REQ-4: Extend molecule `template-content` verify assertions for the + rendered healthcheck script (`status=exited` present in both paths). ## Test Plan -- Verify CI passes with new devx version +- `make molecule` template-content + default scenarios pass. +- `make pytest-cov`, `make lint-all` pass. ## Deploy Plan -- Merge to master, auto-release +- Merge to master → package publishes. Runner hosts apply the role on + their next grm install/upgrade run; affected runners may need a manual + role re-run for immediate relief. +- Vikunja counter reused GRM-167; prior spec preserved in + [GRM-167-devx-bump-historical](GRM-167-devx-bump-historical.md). ## Rollback Plan -- Revert the merge commit +- Revert the merge commit — restores the aggressive cleanup that kills + active CI jobs. ## Acceptance Criteria -- [x] REQ-1: devx pinned to v0.51.0 in pyproject.toml +- [x] REQ-1: both disk paths only remove `status=exited`/`status=dead` + containers +- [x] REQ-2: exclusions, age gate, and non-container prunes unchanged +- [x] REQ-3: comments updated +- [x] REQ-4: template-content verify covers the filters