Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 gitea-actions-bot
|
d6549de2e0 | ||
|
grm-ci-bot
|
dc4bb0936d | ||
 emo
|
52477e558a | ||
|
gitea-actions-bot
|
28214de583 | ||
|
grm-ci-bot
|
7fd023d192 | ||
 emilandemo
|
2ffedaa793 | ||
|
gitea-actions-bot
|
e7b2d4e6af |
+46
-14
@@ -200,16 +200,12 @@ jobs:
|
||||
fi
|
||||
- name: Run molecule tests
|
||||
if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false'
|
||||
shell: bash
|
||||
env:
|
||||
GITEA_URL: ${{ github.server_url }}
|
||||
CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }}
|
||||
CI_GITEA_USERNAME: ${{ vars.CI_GITEA_USERNAME }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
ANSIBLE_INJECT_INVOCATION: "1"
|
||||
JOB_NAME: ${{ github.job }}
|
||||
MATRIX_INDEX: ${{ matrix.runner-index }}
|
||||
GITEA_REPOSITORY: ${{ github.repository }}
|
||||
DOCKER_HOST: unix:///var/run/docker.sock
|
||||
ANSIBLE_INJECT_INVOCATION: "1"
|
||||
run: |
|
||||
. .venv/bin/activate 2>/dev/null || true
|
||||
if [ -z "$TEST_PAIRS" ]; then exit 0; fi
|
||||
@@ -220,19 +216,55 @@ jobs:
|
||||
_TOKEN="$CI_GITEA_API_TOKEN"; [ -z "$_TOKEN" ] && _TOKEN="$CI_GITEA_TOKEN"
|
||||
[ -z "$_TOKEN" ] && { echo "Gitea API token not set — skipping Docker login"; exit 0; }
|
||||
echo "$_TOKEN" | docker login git.oblachno.oblachno.fyi -u "$CI_GITEA_USERNAME" --password-stdin
|
||||
# shellcheck disable=SC2086 # intentional word splitting for argument expansion
|
||||
python3 -m devx.molecule.molecule_ci_guard $TEST_PAIRS
|
||||
# Run each molecule test pair sequentially.
|
||||
# Pairs are 4-part: scenario|platform_name|platform_image|platform_command
|
||||
# Spaces in platform_command are encoded as __SPACE__.
|
||||
role_dir="ansible/roles/gitea_runner"
|
||||
# shellcheck disable=SC2086 # intentional word splitting for pair list
|
||||
for pair in $TEST_PAIRS; do
|
||||
IFS='|' read -r scenario platform_name platform_image platform_command <<< "$pair"
|
||||
platform_command="${platform_command//__SPACE__/ }"
|
||||
export MOLECULE_PLATFORM_NAME="$platform_name"
|
||||
export MOLECULE_PLATFORM_IMAGE="$platform_image"
|
||||
if [ -n "$platform_command" ]; then
|
||||
export MOLECULE_PLATFORM_COMMAND="$platform_command"
|
||||
else
|
||||
unset MOLECULE_PLATFORM_COMMAND
|
||||
fi
|
||||
export ANSIBLE_ALLOW_BROKEN_CONDITIONALS=true
|
||||
echo "--- Running: $scenario on $platform_name ---"
|
||||
pushd "$role_dir" >/dev/null
|
||||
if [ "$scenario" = "default" ]; then
|
||||
molecule test || {
|
||||
echo "FAILED: $pair — running molecule destroy"
|
||||
molecule destroy 2>/dev/null || true
|
||||
popd >/dev/null
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
molecule test -s "$scenario" || {
|
||||
echo "FAILED: $pair — running molecule destroy"
|
||||
molecule destroy -s "$scenario" 2>/dev/null || true
|
||||
popd >/dev/null
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
popd >/dev/null
|
||||
echo "PASSED: $pair"
|
||||
docker system prune -af --volumes 2>/dev/null || true
|
||||
done
|
||||
echo "All molecule tests passed."
|
||||
|
||||
auto-merge:
|
||||
# Auto-merge runs after validate + molecule-tests pass (or molecule is skipped).
|
||||
# Uses always() so it evaluates even when molecule-tests is skipped
|
||||
# (Gitea Actions skips dependent jobs of skipped jobs by default).
|
||||
needs: [validate, molecule-tests]
|
||||
# Auto-merge runs after validate passes. molecule-tests is NOT in needs
|
||||
# because Gitea Actions skips dependent jobs of skipped jobs without
|
||||
# evaluating if: conditions — having molecule-tests in needs would
|
||||
# cascade the skip to auto-merge when ansible-changed=false.
|
||||
needs: [validate]
|
||||
if: >-
|
||||
always() &&
|
||||
github.event_name == 'pull_request' &&
|
||||
needs.validate.result == 'success' &&
|
||||
(needs.molecule-tests.result == 'success' || needs.molecule-tests.result == 'skipped')
|
||||
needs.validate.result == 'success'
|
||||
runs-on: docker
|
||||
container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-base:latest
|
||||
timeout-minutes: 10
|
||||
|
||||
@@ -96,3 +96,43 @@ repos:
|
||||
types: [python]
|
||||
pass_filenames: false
|
||||
stages: [pre-push]
|
||||
|
||||
- id: check-ansible-no-log
|
||||
name: ansible no_log on secret tasks
|
||||
entry: make check-ansible-no-log
|
||||
language: system
|
||||
files: ^ansible/.*\.(yml|yaml)$
|
||||
pass_filenames: false
|
||||
stages: [pre-commit]
|
||||
|
||||
- id: check-ansible-no-state-absent-on-db
|
||||
name: no state absent on DB paths
|
||||
entry: make check-ansible-no-state-absent-on-db
|
||||
language: system
|
||||
files: ^ansible/.*\.(yml|yaml)$
|
||||
pass_filenames: false
|
||||
stages: [pre-commit]
|
||||
|
||||
- id: check-ansible-patterns
|
||||
name: ansible failure-masking patterns
|
||||
entry: make check-ansible-patterns
|
||||
language: system
|
||||
files: ^ansible/.*\.(yml|yaml)$
|
||||
pass_filenames: false
|
||||
stages: [pre-commit]
|
||||
|
||||
- id: check-jinja-expr
|
||||
name: jinja2 expression validation
|
||||
entry: make check-jinja-expr
|
||||
language: system
|
||||
files: ^ansible/.*\.(yml|yaml|j2)$
|
||||
pass_filenames: false
|
||||
stages: [pre-commit]
|
||||
|
||||
- id: check-ansible-set-fact-to-json
|
||||
name: set_fact to_json misuse check
|
||||
entry: make check-ansible-set-fact-to-json
|
||||
language: system
|
||||
files: ^ansible/.*\.(yml|yaml)$
|
||||
pass_filenames: false
|
||||
stages: [pre-commit]
|
||||
|
||||
@@ -278,9 +278,9 @@ via `[tool.devx.classify]` in `pyproject.toml`.
|
||||
- Any new file type not in the allowlist
|
||||
|
||||
**devx module structure** (installed from git, not in this repo):
|
||||
- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, molecule_ci_guard, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg
|
||||
- `devx.ci.*` — CI/CD automation (run by workflows): release, publish, auto_merge, classify_changes, detect_release_commit, push_badges, doc_coverage, sync_wiki, distribute_molecule, discover_runners, notify_failure, post_merge, pr_review, validate_commit_msg
|
||||
- `devx.tools.*` — Dev tools (run locally): check_test_speed, configure_repo, install_checkmake, install_tools, setup, generate_badges, create_task, create_pr, pr_status, pr_logs, pr_label, rebase, pr_rebase
|
||||
- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule, molecule_ci_guard
|
||||
- `devx.molecule.*` — Molecule helpers: molecule_all, platforms, discover_runners, distribute_molecule
|
||||
- `devx.gitea_cli` — Tea CLI wrapper
|
||||
- `devx.i18n` — i18n translation system
|
||||
- `devx.config` — Shared configuration (DEVX_* env vars)
|
||||
@@ -348,7 +348,7 @@ Since devx is installed as a package (via `pip install` from git), it is importa
|
||||
| PYTHONPATH | When to use | Example modules |
|
||||
|------------|-------------|-----------------|
|
||||
| `src` | Module imports from `grm` | `devx.ci.auto_merge`, `devx.ci.pr_review`, `devx.ci.pr_review`, `devx.ci.sync_wiki`, `devx.ci.post_merge`, `devx.ci.classify_changes`, `devx.molecule.discover_runners`, `devx.ci.doc_coverage` |
|
||||
| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.molecule.molecule_ci_guard`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` |
|
||||
| (none) | Module has no GRM imports | `devx.ci.detect_release_commit`, `devx.molecule.distribute_molecule`, `devx.ci.push_badges`, `devx.ci.validate_commit_msg` |
|
||||
|
||||
**In workflows**, always use `env:` blocks (not inline `PYTHONPATH=value`):
|
||||
```yaml
|
||||
|
||||
@@ -2,6 +2,18 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [0.21.0] - 2026-08-09
|
||||
|
||||
### Features
|
||||
|
||||
- *(healthcheck)* Add two-tier disk prune with critical threshold
|
||||
|
||||
## [0.20.0] - 2026-08-09
|
||||
|
||||
### Features
|
||||
|
||||
- *(healthcheck)* Add two-tier disk prune with critical threshold
|
||||
|
||||
## [0.19.0] - 2026-08-08
|
||||
|
||||
### Features
|
||||
|
||||
@@ -175,11 +175,36 @@ makefile-lint:
|
||||
echo "checkmake not found, skipping Makefile lint"; \
|
||||
fi
|
||||
|
||||
lint-all: lint ansible-lint makefile-lint workflow-lint check-api-identity-checks
|
||||
lint-all: lint ansible-lint makefile-lint workflow-lint check-api-identity-checks check-ansible-no-log check-ansible-no-state-absent-on-db check-ansible-patterns check-jinja-expr check-ansible-set-fact-to-json
|
||||
|
||||
check-api-identity-checks:
|
||||
@$(BIN)/python -m devx.tools.check_api_identity_checks
|
||||
|
||||
check-ansible-no-log:
|
||||
@echo "[check-ansible-no-log] Checking Ansible tasks for missing no_log on secret-handling tasks..."
|
||||
@$(BIN)/python -m devx.tools.check_ansible_no_log
|
||||
@echo "[check-ansible-no-log] Passed."
|
||||
|
||||
check-ansible-no-state-absent-on-db:
|
||||
@echo "[check-ansible-no-state-absent-on-db] Checking for state: absent on DB data directories..."
|
||||
@$(BIN)/python -m devx.tools.check_ansible_no_state_absent_on_db
|
||||
@echo "[check-ansible-no-state-absent-on-db] Passed."
|
||||
|
||||
check-ansible-patterns:
|
||||
@echo "[check-ansible-patterns] Checking for dangerous failure-masking patterns..."
|
||||
@$(BIN)/python -m devx.tools.check_ansible_patterns
|
||||
@echo "[check-ansible-patterns] Passed."
|
||||
|
||||
check-jinja-expr:
|
||||
@echo "[check-jinja-expr] Validating Jinja2 expressions in Ansible files..."
|
||||
@$(BIN)/python -m devx.tools.check_jinja_expr
|
||||
@echo "[check-jinja-expr] Passed."
|
||||
|
||||
check-ansible-set-fact-to-json:
|
||||
@echo "[check-ansible-set-fact-to-json] Checking set_fact tasks for to_json misuse..."
|
||||
@$(BIN)/python -m devx.tools.check_ansible_set_fact_to_json
|
||||
@echo "[check-ansible-set-fact-to-json] Passed."
|
||||
|
||||
test-integration:
|
||||
$(BIN)/pytest tests/integration/ -v --no-cov
|
||||
|
||||
|
||||
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Why GRM?
|
||||
|
||||
|
||||
@@ -41,7 +41,13 @@ gitea_runner_service_restart_sec: "5"
|
||||
# jobs in the window between healthcheck runs.
|
||||
gitea_runner_healthcheck_interval: "2min"
|
||||
gitea_runner_healthcheck_boot_delay: "2min"
|
||||
gitea_runner_healthcheck_disk_threshold: 75
|
||||
gitea_runner_healthcheck_disk_threshold: 70
|
||||
# When disk reaches this level, prune EVERYTHING (no until-filter) — the
|
||||
# runner is dangerously full and the gentle until=1h prune isn't enough.
|
||||
# This removes all stopped containers and unused images regardless of age.
|
||||
# At 75%+, molecule containers fail with "container is not running" because
|
||||
# overlay2 runs out of space under parallel DinD load.
|
||||
gitea_runner_healthcheck_disk_critical: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# Auto-recovery: when the healthcheck detects an unregistered runner, it
|
||||
|
||||
@@ -112,4 +112,5 @@
|
||||
- "'status=removing' in healthcheck_script.content | b64decode"
|
||||
- "'status=stopping' in healthcheck_script.content | b64decode"
|
||||
- "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
|
||||
- "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode"
|
||||
fail_msg: "Healthcheck script template is missing expected content"
|
||||
|
||||
@@ -223,8 +223,23 @@ fi
|
||||
|
||||
# 3. Check disk space — prune aggressively if below threshold
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
|
||||
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then
|
||||
echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune"
|
||||
# Critical level: remove ALL stopped containers (no age filter) and ALL
|
||||
# unused images/volumes. The until=1h gentle prune is insufficient here.
|
||||
# Stop+rm stale non-CI containers regardless of age (failed molecule tests
|
||||
# from the last 59 minutes also consume disk).
|
||||
docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
|
||||
| grep -v 'GITEA-ACTIONS-TASK' \
|
||||
| awk '{print $1}' \
|
||||
| xargs -r docker rm -f 2>/dev/null || true
|
||||
docker system prune -af --volumes || true
|
||||
docker network prune -f || true
|
||||
docker builder prune -af || true
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
echo "INFO: Disk usage after full prune: ${disk_pct}%"
|
||||
elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)"
|
||||
# Force-remove stale containers (including running ones from failed molecule tests)
|
||||
# that are older than 1 hour. "docker container prune -f" only removes stopped
|
||||
# containers, so running containers from crashed CI jobs accumulate and consume
|
||||
@@ -236,7 +251,11 @@ if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
| grep -E '(hour|day|week|month|year)s? ago' \
|
||||
| awk '{print $1}' \
|
||||
| xargs -r docker rm -f 2>/dev/null || true
|
||||
docker system prune -af --filter "until=1h" --volumes || true
|
||||
# Prune images and containers older than 1h (until filter is NOT
|
||||
# supported with --volumes, so prune volumes separately without a filter).
|
||||
docker image prune -af --filter "until=1h" 2>/dev/null || true
|
||||
docker container prune -f --filter "until=1h" 2>/dev/null || true
|
||||
docker volume prune -f 2>/dev/null || true
|
||||
# Prune networks older than 1 hour to avoid removing networks that
|
||||
# molecule tests are actively creating (e.g. 'traefik' network created
|
||||
# during molecule create phase before containers are attached).
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
@@ -259,9 +259,9 @@ OS platform matrix (defined in `devx.molecule.platforms`), then splits
|
||||
the resulting test pairs evenly across the requested number of runners.
|
||||
Each pair is encoded as `scenario|platform_name|platform_image|platform_command`.
|
||||
|
||||
`devx.molecule.molecule_ci_guard` runs the actual molecule test for a
|
||||
given test pair, with CI context (Gitea URL, token, run ID) for
|
||||
reporting results back to the commit status API.
|
||||
The CI workflow runs each test pair sequentially via a shell loop that
|
||||
sets the appropriate `MOLECULE_PLATFORM_*` environment variables and
|
||||
invokes `molecule test` directly.
|
||||
|
||||
### Commit Message Validation
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ The `molecule-tests` job uses `fromJSON()` to consume the dynamic matrix, and pa
|
||||
|
||||
`devx.molecule.distribute_molecule` discovers all molecule scenarios under `ansible/roles/*/molecule/` and crosses them with the supported OS platform matrix, then splits the resulting test pairs evenly across the requested number of runners. Each pair is encoded as `scenario|platform_name|platform_image|platform_command`.
|
||||
|
||||
`devx.molecule.molecule_ci_guard` runs the actual molecule test for a given test pair, with CI context (Gitea URL, token, run ID) for reporting results back to the commit status API.
|
||||
The CI workflow runs each test pair sequentially via a shell loop that sets the appropriate `MOLECULE_PLATFORM_*` environment variables and invokes `molecule test` directly.
|
||||
|
||||
### Path-based CI filtering
|
||||
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ ci = [
|
||||
"build==1.5.1",
|
||||
"twine==6.2.0",
|
||||
# Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.)
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0",
|
||||
]
|
||||
# Lint and type-checking tools (validate job)
|
||||
lint = [
|
||||
@@ -56,7 +56,7 @@ molecule = [
|
||||
dev = [
|
||||
"grm[ci,lint,molecule]",
|
||||
# Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr)
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0",
|
||||
# Non-Python dev dependency: checkmake (Makefile linter)
|
||||
# Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest
|
||||
]
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
"""Gitea Runner Manager — lean CLI for managing Gitea Actions runners."""
|
||||
|
||||
__version__ = "0.19.0"
|
||||
__version__ = "0.21.0"
|
||||
|
||||
Reference in New Issue
Block a user