From 2ffedaa793425984aa011226372d17ddcd20a36d Mon Sep 17 00:00:00 2001 From: emil User Date: Sun, 9 Aug 2026 11:12:37 +0000 Subject: [PATCH] GRM-159: feat(healthcheck): add two-tier disk prune with critical threshold Co-authored-by: emil User --- .gitea/workflows/ci.yml | 48 +++++++++++++++---- AGENTS.md | 6 +-- ansible/roles/gitea_runner/defaults/main.yml | 8 +++- .../molecule/template-content/verify.yml | 1 + .../templates/runner-healthcheck.sh.j2 | 25 ++++++++-- docs/tech/ci-cd-workflow.md | 6 +-- docs/tech/testing-strategy.md | 2 +- 7 files changed, 77 insertions(+), 19 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6a125c6..50f0626 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -200,16 +200,12 @@ jobs: fi - name: Run molecule tests if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false' + shell: bash env: - GITEA_URL: ${{ github.server_url }} CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }} CI_GITEA_USERNAME: ${{ vars.CI_GITEA_USERNAME }} - RUN_ID: ${{ github.run_id }} - ANSIBLE_INJECT_INVOCATION: "1" - JOB_NAME: ${{ github.job }} - MATRIX_INDEX: ${{ matrix.runner-index }} - GITEA_REPOSITORY: ${{ github.repository }} DOCKER_HOST: unix:///var/run/docker.sock + ANSIBLE_INJECT_INVOCATION: "1" run: | . .venv/bin/activate 2>/dev/null || true if [ -z "$TEST_PAIRS" ]; then exit 0; fi @@ -220,8 +216,44 @@ jobs: _TOKEN="$CI_GITEA_API_TOKEN"; [ -z "$_TOKEN" ] && _TOKEN="$CI_GITEA_TOKEN" [ -z "$_TOKEN" ] && { echo "Gitea API token not set — skipping Docker login"; exit 0; } echo "$_TOKEN" | docker login git.oblachno.oblachno.fyi -u "$CI_GITEA_USERNAME" --password-stdin - # shellcheck disable=SC2086 # intentional word splitting for argument expansion - python3 -m devx.molecule.molecule_ci_guard $TEST_PAIRS + # Run each molecule test pair sequentially. + # Pairs are 4-part: scenario|platform_name|platform_image|platform_command + # Spaces in platform_command are encoded as __SPACE__. + role_dir="ansible/roles/gitea_runner" + # shellcheck disable=SC2086 # intentional word splitting for pair list + for pair in $TEST_PAIRS; do + IFS='|' read -r scenario platform_name platform_image platform_command <<< "$pair" + platform_command="${platform_command//__SPACE__/ }" + export MOLECULE_PLATFORM_NAME="$platform_name" + export MOLECULE_PLATFORM_IMAGE="$platform_image" + if [ -n "$platform_command" ]; then + export MOLECULE_PLATFORM_COMMAND="$platform_command" + else + unset MOLECULE_PLATFORM_COMMAND + fi + export ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true + echo "--- Running: $scenario on $platform_name ---" + pushd "$role_dir" >/dev/null + if [ "$scenario" = "default" ]; then + molecule test || { + echo "FAILED: $pair — running molecule destroy" + molecule destroy 2>/dev/null || true + popd >/dev/null + exit 1 + } + else + molecule test -s "$scenario" || { + echo "FAILED: $pair — running molecule destroy" + molecule destroy -s "$scenario" 2>/dev/null || true + popd >/dev/null + exit 1 + } + fi + popd >/dev/null + echo "PASSED: $pair" + docker system prune -af --volumes 2>/dev/null || true + done + echo "All molecule tests passed." auto-merge: # Auto-merge runs after validate + molecule-tests pass (or molecule is skipped). diff --git a/AGENTS.md b/AGENTS.md index 7a1d707..2a84ace 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -278,9 +278,9 @@ via `[tool.devx.classify]` in `pyproject.toml`. - Any new file type not in the allowlist **devx module structure** (installed from git, not in this repo): -- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, molecule_ci_guard, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg +- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg - `devx.tools.*` — Dev tools (run locally): check_test_speed, configure_repo, install_checkmake, install_tools, setup, generate_badges, create_task, create_pr, pr_status, pr_logs, pr_label, rebase, pr_rebase -- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule, molecule_ci_guard +- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule - `devx.gitea_cli` — Tea CLI wrapper - `devx.i18n` — i18n translation system - `devx.config` — Shared configuration (DEVX_* env vars) @@ -348,7 +348,7 @@ Since devx is installed as a package (via `pip install` from git), it is importa | PYTHONPATH | When to use | Example modules | |------------|-------------|-----------------| | `src` | Module imports from `grm` | `devx.ci.auto_merge`, `devx.ci.pr_review`, `devx.ci.pr_review`, `devx.ci.sync_wiki`, `devx.ci.post_merge`, `devx.ci.classify_changes`, `devx.molecule.discover_runners`, `devx.ci.doc_coverage` | -| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.molecule.molecule_ci_guard`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` | +| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` | **In workflows**, always use `env:` blocks (not inline `PYTHONPATH=value`): ```yaml diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 3a5041c..5a5fe71 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -41,7 +41,13 @@ gitea_runner_service_restart_sec: "5" # jobs in the window between healthcheck runs. gitea_runner_healthcheck_interval: "2min" gitea_runner_healthcheck_boot_delay: "2min" -gitea_runner_healthcheck_disk_threshold: 75 +gitea_runner_healthcheck_disk_threshold: 70 +# When disk reaches this level, prune EVERYTHING (no until-filter) — the +# runner is dangerously full and the gentle until=1h prune isn't enough. +# This removes all stopped containers and unused images regardless of age. +# At 75%+, molecule containers fail with "container is not running" because +# overlay2 runs out of space under parallel DinD load. +gitea_runner_healthcheck_disk_critical: 75 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" # Auto-recovery: when the healthcheck detects an unregistered runner, it diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index 5cea828..eaf72b3 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -112,4 +112,5 @@ - "'status=removing' in healthcheck_script.content | b64decode" - "'status=stopping' in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" + - "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode" fail_msg: "Healthcheck script template is missing expected content" diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index 16432ba..3df68a1 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -223,8 +223,23 @@ fi # 3. Check disk space — prune aggressively if below threshold disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') -if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then - echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources" +if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then + echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune" + # Critical level: remove ALL stopped containers (no age filter) and ALL + # unused images/volumes. The until=1h gentle prune is insufficient here. + # Stop+rm stale non-CI containers regardless of age (failed molecule tests + # from the last 59 minutes also consume disk). + docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ + | grep -v 'GITEA-ACTIONS-TASK' \ + | awk '{print $1}' \ + | xargs -r docker rm -f 2>/dev/null || true + docker system prune -af --volumes || true + docker network prune -f || true + docker builder prune -af || true + disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') + echo "INFO: Disk usage after full prune: ${disk_pct}%" +elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then + echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)" # Force-remove stale containers (including running ones from failed molecule tests) # that are older than 1 hour. "docker container prune -f" only removes stopped # containers, so running containers from crashed CI jobs accumulate and consume @@ -236,7 +251,11 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then | grep -E '(hour|day|week|month|year)s? ago' \ | awk '{print $1}' \ | xargs -r docker rm -f 2>/dev/null || true - docker system prune -af --filter "until=1h" --volumes || true + # Prune images and containers older than 1h (until filter is NOT + # supported with --volumes, so prune volumes separately without a filter). + docker image prune -af --filter "until=1h" 2>/dev/null || true + docker container prune -f --filter "until=1h" 2>/dev/null || true + docker volume prune -f 2>/dev/null || true # Prune networks older than 1 hour to avoid removing networks that # molecule tests are actively creating (e.g. 'traefik' network created # during molecule create phase before containers are attached). diff --git a/docs/tech/ci-cd-workflow.md b/docs/tech/ci-cd-workflow.md index 3e15107..96c39cf 100644 --- a/docs/tech/ci-cd-workflow.md +++ b/docs/tech/ci-cd-workflow.md @@ -259,9 +259,9 @@ OS platform matrix (defined in `devx.molecule.platforms`), then splits the resulting test pairs evenly across the requested number of runners. Each pair is encoded as `scenario|platform_name|platform_image|platform_command`. -`devx.molecule.molecule_ci_guard` runs the actual molecule test for a -given test pair, with CI context (Gitea URL, token, run ID) for -reporting results back to the commit status API. +The CI workflow runs each test pair sequentially via a shell loop that +sets the appropriate `MOLECULE_PLATFORM_*` environment variables and +invokes `molecule test` directly. ### Commit Message Validation diff --git a/docs/tech/testing-strategy.md b/docs/tech/testing-strategy.md index e0132a8..3a9e9d7 100644 --- a/docs/tech/testing-strategy.md +++ b/docs/tech/testing-strategy.md @@ -91,7 +91,7 @@ The `molecule-tests` job uses `fromJSON()` to consume the dynamic matrix, and pa `devx.molecule.distribute_molecule` discovers all molecule scenarios under `ansible/roles/*/molecule/` and crosses them with the supported OS platform matrix, then splits the resulting test pairs evenly across the requested number of runners. Each pair is encoded as `scenario|platform_name|platform_image|platform_command`. -`devx.molecule.molecule_ci_guard` runs the actual molecule test for a given test pair, with CI context (Gitea URL, token, run ID) for reporting results back to the commit status API. +The CI workflow runs each test pair sequentially via a shell loop that sets the appropriate `MOLECULE_PLATFORM_*` environment variables and invokes `molecule test` directly. ### Path-based CI filtering