diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index eaf72b3..a02ffb6 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -48,6 +48,7 @@ that: - "'Type=oneshot' in prune_service.content | b64decode" - "'docker rm -f' in prune_service.content | b64decode" + - "'status=exited' in prune_service.content | b64decode" - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode" - "'docker system prune -af' in prune_service.content | b64decode" - "'docker network prune' in prune_service.content | b64decode" diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index fb546f9..a794097 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -5,16 +5,18 @@ Description=Docker prune for Gitea runner resources Type=oneshot Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} -# Force-remove stale containers (including running ones) left behind by failed -# molecule tests. "docker container prune -f" only removes stopped containers, -# so running containers from crashed/interrupted CI jobs accumulate indefinitely, -# consuming disk and memory. We stop+rm everything first, then prune the rest. +# Force-remove stale *stopped* containers left behind by failed molecule tests. +# Implements: REQ-1 (GRM-166) — only containers with status=exited are +# eligible. RunningFor measures creation time, so a stale molecule instance +# (e.g. ubuntu-2604) that a new run restarts still looks ">1h old"; removing +# running containers kills active converges with "No such container" +# (infra nightly run 5710). Running leftovers are instead reused or destroyed +# by the next molecule create/destroy cycle. # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing # them kills the active CI job and causes "RWLayer is unexpectedly nil" errors. # Only remove containers older than 1 hour (grep for "hour/day/week/month/year -# ago" in RunningFor) to avoid killing molecule test containers that CI jobs -# are actively using. -ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' +# ago" in RunningFor) to avoid removing containers a job just created. +ExecStart=/bin/sh -c 'docker ps -a --filter "status=exited" --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes # Prune networks older than the prune-until threshold to avoid removing # networks that molecule tests are actively creating (e.g. 'traefik' network diff --git a/docs/specs/GRM-166.md b/docs/specs/GRM-166.md new file mode 100644 index 0000000..1e30f93 --- /dev/null +++ b/docs/specs/GRM-166.md @@ -0,0 +1,43 @@ +# GRM-166: Fix docker-prune killing running molecule containers + +## Problem +`docker-prune.service` force-removes containers older than 1h by +**creation** time (`RunningFor`). Molecule instance containers +(`ubuntu-2604` etc.) run on the runner's shared rootless daemon and are +not name-excluded (only `GITEA-ACTIONS-TASK` jobs are). A stale molecule +container left by a crashed run is *restarted/reused* by the next run's +create phase — it then shows `RunningFor` >1h and gets `docker rm -f`'d +mid-converge: `No such container: ubuntu-2604`. Observed on infra nightly +run 5710 across multiple runners (incl. a 32 GB host), causing +shard failures. + +## Approach +REQ-1: Change the container cleanup `ExecStart` in + `ansible/roles/gitea_runner/templates/docker-prune.service.j2` to only + target containers with `status=exited` (add + `--filter "status=exited"`). Running containers — including stale + molecule instances adopted by an active run — are never force-removed. +REQ-2: Update the template comment to document the race and why only + stopped containers are removed. +REQ-3: Extend the `template-content` molecule verify assertions to + require the `status=exited` filter in the rendered unit. + +## Test Plan +- `make molecule` (template-content scenario) verifies the rendered unit. +- `make pytest-cov` and `make lint-all` pass. + +## Deploy Plan +- Merge to master → package publishes; runner hosts pick up the role on + their next grm install/upgrade cycle (or manual re-run of the role on + affected runners). +- Until then, nightly molecule reruns remain exposed to the race — + mitigated by the fact that swept daemons have no stale containers left. + +## Rollback Plan +- Revert the merge commit. Running stale leftovers would then again be + force-removed (the pre-existing risky behaviour). + +## Acceptance Criteria +- [x] REQ-1: prune rm step filtered to `status=exited` +- [x] REQ-2: comment documents the creation-time vs running race +- [x] REQ-3: template-content verify asserts the exited filter