From 5712804310c8cdcb2ebe11025e866412c4e4448f Mon Sep 17 00:00:00 2001
From: Emil Simeonov <emil@theliberatededge.org>
Date: Sat, 8 Aug 2026 22:52:51 +0200
Subject: [PATCH] fix(prune): only remove containers older than 1 hour
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The prune service and healthcheck were force-removing ALL containers
(except GITEA-ACTIONS-TASK), including molecule test containers that
CI jobs were actively using. This caused "No such container" errors
during molecule prepare/converge phases.

Filter by RunningFor field — only remove containers showing
"hour/day/week/month/year ago", excluding "minutes/seconds ago".
This prevents killing molecule containers from running CI jobs
while still cleaning up stale containers from crashed jobs.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../gitea_runner/templates/docker-prune.service.j2  |  5 ++++-
 .../gitea_runner/templates/runner-healthcheck.sh.j2 | 13 ++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2
index ce23d43..fb546f9 100644
--- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2
+++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2
@@ -11,7 +11,10 @@ Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
 # consuming disk and memory. We stop+rm everything first, then prune the rest.
 # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing
 # them kills the active CI job and causes "RWLayer is unexpectedly nil" errors.
-ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true'
+# Only remove containers older than 1 hour (grep for "hour/day/week/month/year
+# ago" in RunningFor) to avoid killing molecule test containers that CI jobs
+# are actively using.
+ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true'
 ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes
 # Prune networks older than the prune-until threshold to avoid removing
 # networks that molecule tests are actively creating (e.g. 'traefik' network
diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2
index 88ba20b..16432ba 100644
--- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2
+++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2
@@ -225,12 +225,15 @@ fi
 disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
 if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
     echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
-    # Force-remove stale containers (including running ones from failed molecule tests).
-    # "docker container prune -f" only removes stopped containers, so running
-    # containers from crashed CI jobs accumulate and consume disk/memory.
-    # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
-    docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
+    # Force-remove stale containers (including running ones from failed molecule tests)
+    # that are older than 1 hour. "docker container prune -f" only removes stopped
+    # containers, so running containers from crashed CI jobs accumulate and consume
+    # disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
+    # Only remove containers older than 1 hour to avoid killing molecule test
+    # containers that CI jobs are actively using.
+    docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \
         | grep -v 'GITEA-ACTIONS-TASK' \
+        | grep -E '(hour|day|week|month|year)s? ago' \
         | awk '{print $1}' \
         | xargs -r docker rm -f 2>/dev/null || true
     docker system prune -af --filter "until=1h" --volumes || true