GRM-159: feat(healthcheck): add two-tier disk prune with critical threshold

Co-authored-by: emil User <emil.simeonov@tutanota.com>

.gitea/workflows/ci.yml

@@ -200,16 +200,12 @@ jobs:
           fi
       - name: Run molecule tests
         if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false'
+        shell: bash
         env:
-          GITEA_URL: ${{ github.server_url }}
           CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }}
           CI_GITEA_USERNAME: ${{ vars.CI_GITEA_USERNAME }}
-          RUN_ID: ${{ github.run_id }}
-          ANSIBLE_INJECT_INVOCATION: "1"
-          JOB_NAME: ${{ github.job }}
-          MATRIX_INDEX: ${{ matrix.runner-index }}
-          GITEA_REPOSITORY: ${{ github.repository }}
           DOCKER_HOST: unix:///var/run/docker.sock
+          ANSIBLE_INJECT_INVOCATION: "1"
         run: |
           . .venv/bin/activate 2>/dev/null || true
           if [ -z "$TEST_PAIRS" ]; then exit 0; fi
@@ -220,8 +216,44 @@ jobs:
           _TOKEN="$CI_GITEA_API_TOKEN"; [ -z "$_TOKEN" ] && _TOKEN="$CI_GITEA_TOKEN"
           [ -z "$_TOKEN" ] && { echo "Gitea API token not set — skipping Docker login"; exit 0; }
           echo "$_TOKEN" | docker login git.oblachno.oblachno.fyi -u "$CI_GITEA_USERNAME" --password-stdin
-          # shellcheck disable=SC2086 # intentional word splitting for argument expansion
-          python3 -m devx.molecule.molecule_ci_guard $TEST_PAIRS
+          # Run each molecule test pair sequentially.
+          # Pairs are 4-part: scenario|platform_name|platform_image|platform_command
+          # Spaces in platform_command are encoded as __SPACE__.
+          role_dir="ansible/roles/gitea_runner"
+          # shellcheck disable=SC2086 # intentional word splitting for pair list
+          for pair in $TEST_PAIRS; do
+            IFS='|' read -r scenario platform_name platform_image platform_command <<< "$pair"
+            platform_command="${platform_command//__SPACE__/ }"
+            export MOLECULE_PLATFORM_NAME="$platform_name"
+            export MOLECULE_PLATFORM_IMAGE="$platform_image"
+            if [ -n "$platform_command" ]; then
+              export MOLECULE_PLATFORM_COMMAND="$platform_command"
+            else
+              unset MOLECULE_PLATFORM_COMMAND
+            fi
+            export ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true
+            echo "--- Running: $scenario on $platform_name ---"
+            pushd "$role_dir" >/dev/null
+            if [ "$scenario" = "default" ]; then
+              molecule test || {
+                echo "FAILED: $pair — running molecule destroy"
+                molecule destroy 2>/dev/null || true
+                popd >/dev/null
+                exit 1
+              }
+            else
+              molecule test -s "$scenario" || {
+                echo "FAILED: $pair — running molecule destroy"
+                molecule destroy -s "$scenario" 2>/dev/null || true
+                popd >/dev/null
+                exit 1
+              }
+            fi
+            popd >/dev/null
+            echo "PASSED: $pair"
+            docker system prune -af --volumes 2>/dev/null || true
+          done
+          echo "All molecule tests passed."
 
   auto-merge:
     # Auto-merge runs after validate + molecule-tests pass (or molecule is skipped).

AGENTS.md

@@ -278,9 +278,9 @@ via `[tool.devx.classify]` in `pyproject.toml`.
 - Any new file type not in the allowlist
 
 **devx module structure** (installed from git, not in this repo):
-- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, molecule_ci_guard, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg
+- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg
 - `devx.tools.*` — Dev tools (run locally): check_test_speed, configure_repo, install_checkmake, install_tools, setup, generate_badges, create_task, create_pr, pr_status, pr_logs, pr_label, rebase, pr_rebase
-- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule, molecule_ci_guard
+- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule
 - `devx.gitea_cli` — Tea CLI wrapper
 - `devx.i18n` — i18n translation system
 - `devx.config` — Shared configuration (DEVX_* env vars)
@@ -348,7 +348,7 @@ Since devx is installed as a package (via `pip install` from git), it is importa
 | PYTHONPATH | When to use | Example modules |
 |------------|-------------|-----------------|
 | `src` | Module imports from `grm` | `devx.ci.auto_merge`, `devx.ci.pr_review`, `devx.ci.pr_review`, `devx.ci.sync_wiki`, `devx.ci.post_merge`, `devx.ci.classify_changes`, `devx.molecule.discover_runners`, `devx.ci.doc_coverage` |
-| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.molecule.molecule_ci_guard`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` |
+| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` |
 
 **In workflows**, always use `env:` blocks (not inline `PYTHONPATH=value`):
 ```yaml

ansible/roles/gitea_runner/defaults/main.yml

@@ -41,7 +41,13 @@ gitea_runner_service_restart_sec: "5"
 # jobs in the window between healthcheck runs.
 gitea_runner_healthcheck_interval: "2min"
 gitea_runner_healthcheck_boot_delay: "2min"
-gitea_runner_healthcheck_disk_threshold: 75
+gitea_runner_healthcheck_disk_threshold: 70
+# When disk reaches this level, prune EVERYTHING (no until-filter) — the
+# runner is dangerously full and the gentle until=1h prune isn't enough.
+# This removes all stopped containers and unused images regardless of age.
+# At 75%+, molecule containers fail with "container is not running" because
+# overlay2 runs out of space under parallel DinD load.
+gitea_runner_healthcheck_disk_critical: 75
 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
 
 # Auto-recovery: when the healthcheck detects an unregistered runner, it

ansible/roles/gitea_runner/molecule/template-content/verify.yml

@@ -112,4 +112,5 @@
           - "'status=removing' in healthcheck_script.content | b64decode"
           - "'status=stopping' in healthcheck_script.content | b64decode"
           - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
+          - "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode"
         fail_msg: "Healthcheck script template is missing expected content"

ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2

@@ -223,8 +223,23 @@ fi
 
 # 3. Check disk space — prune aggressively if below threshold
 disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
-if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
-    echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
+if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then
+    echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune"
+    # Critical level: remove ALL stopped containers (no age filter) and ALL
+    # unused images/volumes. The until=1h gentle prune is insufficient here.
+    # Stop+rm stale non-CI containers regardless of age (failed molecule tests
+    # from the last 59 minutes also consume disk).
+    docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
+        | grep -v 'GITEA-ACTIONS-TASK' \
+        | awk '{print $1}' \
+        | xargs -r docker rm -f 2>/dev/null || true
+    docker system prune -af --volumes || true
+    docker network prune -f || true
+    docker builder prune -af || true
+    disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+    echo "INFO: Disk usage after full prune: ${disk_pct}%"
+elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
+    echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)"
     # Force-remove stale containers (including running ones from failed molecule tests)
     # that are older than 1 hour. "docker container prune -f" only removes stopped
     # containers, so running containers from crashed CI jobs accumulate and consume
@@ -236,7 +251,11 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
         | grep -E '(hour|day|week|month|year)s? ago' \
         | awk '{print $1}' \
         | xargs -r docker rm -f 2>/dev/null || true
-    docker system prune -af --filter "until=1h" --volumes || true
+    # Prune images and containers older than 1h (until filter is NOT
+    # supported with --volumes, so prune volumes separately without a filter).
+    docker image prune -af --filter "until=1h" 2>/dev/null || true
+    docker container prune -f --filter "until=1h" 2>/dev/null || true
+    docker volume prune -f 2>/dev/null || true
     # Prune networks older than 1 hour to avoid removing networks that
     # molecule tests are actively creating (e.g. 'traefik' network created
     # during molecule create phase before containers are attached).

docs/tech/ci-cd-workflow.md

@@ -259,9 +259,9 @@ OS platform matrix (defined in `devx.molecule.platforms`), then splits
 the resulting test pairs evenly across the requested number of runners.
 Each pair is encoded as `scenario|platform_name|platform_image|platform_command`.
 
-`devx.molecule.molecule_ci_guard` runs the actual molecule test for a
-given test pair, with CI context (Gitea URL, token, run ID) for
-reporting results back to the commit status API.
+The CI workflow runs each test pair sequentially via a shell loop that
+sets the appropriate `MOLECULE_PLATFORM_*` environment variables and
+invokes `molecule test` directly.
 
 ### Commit Message Validation
 

docs/tech/testing-strategy.md

@@ -91,7 +91,7 @@ The `molecule-tests` job uses `fromJSON()` to consume the dynamic matrix, and pa
 
 `devx.molecule.distribute_molecule` discovers all molecule scenarios under `ansible/roles/*/molecule/` and crosses them with the supported OS platform matrix, then splits the resulting test pairs evenly across the requested number of runners. Each pair is encoded as `scenario|platform_name|platform_image|platform_command`.
 
-`devx.molecule.molecule_ci_guard` runs the actual molecule test for a given test pair, with CI context (Gitea URL, token, run ID) for reporting results back to the commit status API.
+The CI workflow runs each test pair sequentially via a shell loop that sets the appropriate `MOLECULE_PLATFORM_*` environment variables and invokes `molecule test` directly.
 
 ### Path-based CI filtering