diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 91d2437..6a125c6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -160,10 +160,10 @@ jobs: container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-full:latest timeout-minutes: 15 strategy: - fail-fast: true - max-parallel: 6 + fail-fast: false + max-parallel: 4 matrix: - runner-index: [1, 2, 3, 4, 5, 6] + runner-index: [1, 2, 3, 4] steps: - uses: actions/checkout@v4 - name: Set up environment @@ -178,15 +178,28 @@ jobs: - name: Discover assigned test pairs env: RUNNER_INDEX: ${{ matrix.runner-index }} - MAX_RUNNERS: 6 + MAX_RUNNERS: 4 run: | . .venv/bin/activate 2>/dev/null || true python3 -m devx.molecule.distribute_molecule \ --runner-index "$RUNNER_INDEX" \ --max-runners "$MAX_RUNNERS" \ --github-env - - name: Run molecule tests + - name: Prune stale Docker data + id: prune if: env.SKIP != 'true' + run: | + docker system prune -af --volumes 2>/dev/null || true + disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') + echo "Disk usage after prune: ${disk_pct}%" + if [ "$disk_pct" -ge 85 ]; then + echo "should-run=false" >> "$GITHUB_OUTPUT" + echo "::warning::Disk usage at ${disk_pct}% after prune — skipping molecule tests to avoid ENOSPC failures" + else + echo "should-run=true" >> "$GITHUB_OUTPUT" + fi + - name: Run molecule tests + if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false' env: GITEA_URL: ${{ github.server_url }} CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 367603d..9a7d733 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - Pin containerd.io to compatible version for Docker 28.x + ## [0.18.7] - 2026-08-06 ### Bug Fixes diff --git a/README.md b/README.md index 8e2929b..ec1762e 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE) -[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki) -[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases) -[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/python.svg)](https://www.python.org/downloads/) +[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki) +[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases) +[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/python.svg)](https://www.python.org/downloads/) ## Why GRM? diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 0c2292f..cf405d0 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,7 +1,11 @@ +--- collections: - name: community.general - version: "==13.1.0" + type: url + source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/13.1.0/community-general-13.1.0.tar.gz - name: ansible.posix - version: "==2.2.1" + type: url + source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/2.2.1/ansible-posix-2.2.1.tar.gz - name: community.docker - version: "==5.2.1" + type: url + source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/5.2.1/community-docker-5.2.1.tar.gz diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 184a1bd..3a5041c 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -3,6 +3,13 @@ gitea_runner_version: "2.0.1" gitea_runner_labels: "docker,ubuntu-latest:docker://runner-images:ubuntu-26.04" gitea_runner_skip_registration: false +# Force re-registration even if .runner file exists. +# Use this when Gitea no longer recognizes the runner (e.g., after a Gitea +# server restore/reinstall or when the runner record was deleted from the +# admin UI). The existing .runner file is removed and a new registration is +# performed. Requires registration_token. +gitea_runner_force_reregister: false + # Per-runner user (rootless isolation) gitea_runner_user_prefix: "grm-" gitea_runner_base_home: "/home" @@ -37,6 +44,18 @@ gitea_runner_healthcheck_boot_delay: "2min" gitea_runner_healthcheck_disk_threshold: 75 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" +# Auto-recovery: when the healthcheck detects an unregistered runner, it +# can automatically re-register if a Gitea API token is provided. +# The token needs admin or org-level access to fetch registration tokens. +# Stored in a file readable by the runner user (mode 0400). +# Set to empty string to disable auto-recovery (manual re-registration required). +gitea_runner_auto_recover_api_token: "" + +# Cooldown file to prevent auto-recovery loops (e.g., if Gitea is down). +# The healthcheck writes a timestamp to this file after a re-registration +# attempt and skips further attempts for the cooldown period. +gitea_runner_auto_recover_cooldown_sec: 300 + # Docker daemon resilience settings (applied to daemon.json). # live-restore: containers survive daemon restarts — prevents stuck container # states when the healthcheck restarts a hung daemon. diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index f0ebdfa..5cea828 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -47,9 +47,9 @@ ansible.builtin.assert: that: - "'Type=oneshot' in prune_service.content | b64decode" - - "'docker system prune' in prune_service.content | b64decode" - - "'docker volume prune' in prune_service.content | b64decode" - - "'docker container prune' in prune_service.content | b64decode" + - "'docker rm -f' in prune_service.content | b64decode" + - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode" + - "'docker system prune -af' in prune_service.content | b64decode" - "'docker network prune' in prune_service.content | b64decode" - "'docker builder prune' in prune_service.content | b64decode" fail_msg: "Prune service template is missing expected directives" @@ -105,23 +105,11 @@ - "'timeout 10 docker info' in healthcheck_script.content | b64decode" - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode" - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" - - "'docker system prune' in healthcheck_script.content | b64decode" - - "'docker container prune' in healthcheck_script.content | b64decode" + - "'docker rm -f' in healthcheck_script.content | b64decode" + - "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode" + - "'docker system prune -af' in healthcheck_script.content | b64decode" - "'docker network prune' in healthcheck_script.content | b64decode" - "'status=removing' in healthcheck_script.content | b64decode" - "'status=stopping' in healthcheck_script.content | b64decode" - - "'docker rm -f' in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" fail_msg: "Healthcheck script template is missing expected content" - - - name: Assert healthcheck script does NOT use aggressive prune (-af) - ansible.builtin.assert: - that: - - "'prune -af' not in healthcheck_script.content | b64decode" - - "'image prune -af' not in healthcheck_script.content | b64decode" - - "'system prune -af' not in healthcheck_script.content | b64decode" - - "'volume prune -af' not in healthcheck_script.content | b64decode" - fail_msg: >- - Healthcheck script uses 'prune -af' which removes ALL images - (including tagged runner images like ci-full). Use 'prune -f' - (dangling only) to preserve tagged images. diff --git a/ansible/roles/gitea_runner/tasks/healthcheck.yml b/ansible/roles/gitea_runner/tasks/healthcheck.yml index a85d4e1..30bd8c7 100644 --- a/ansible/roles/gitea_runner/tasks/healthcheck.yml +++ b/ansible/roles/gitea_runner/tasks/healthcheck.yml @@ -7,6 +7,22 @@ group: "{{ gitea_runner_service_user }}" mode: "0755" +- name: Write auto-recovery API token file + ansible.builtin.copy: + content: "{{ gitea_runner_auto_recover_api_token }}" + dest: "{{ gitea_runner_config_dir }}/auto-recover.token" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0400" + no_log: true + when: gitea_runner_auto_recover_api_token | length > 0 + +- name: Remove stale auto-recovery token file (if auto-recovery disabled) + ansible.builtin.file: + path: "{{ gitea_runner_config_dir }}/auto-recover.token" + state: absent + when: gitea_runner_auto_recover_api_token | length == 0 + - name: Create healthcheck user service file ansible.builtin.template: src: runner-healthcheck.service.j2 diff --git a/ansible/roles/gitea_runner/tasks/register.yml b/ansible/roles/gitea_runner/tasks/register.yml index 152d921..d0a9248 100644 --- a/ansible/roles/gitea_runner/tasks/register.yml +++ b/ansible/roles/gitea_runner/tasks/register.yml @@ -7,11 +7,20 @@ group: "{{ gitea_runner_service_user }}" mode: "0755" -- name: Check if runner is already registered +- name: Check if runner registration file exists ansible.builtin.stat: path: "{{ gitea_runner_data_dir }}/.runner" register: gitea_runner_registered +- name: Remove stale runner registration file + ansible.builtin.file: + path: "{{ gitea_runner_data_dir }}/.runner" + state: absent + when: + - gitea_runner_registered.stat.exists + - gitea_runner_force_reregister | bool + register: gitea_runner_registration_removed + - name: Register runner with Gitea ansible.builtin.command: > {{ gitea_runner_binary_path }} register @@ -28,7 +37,23 @@ XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default(0) }}" DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" DOCKER_HOST: "unix:///run/user/{{ gitea_runner_uid | default(0) }}/docker.sock" - when: not gitea_runner_registered.stat.exists + when: not gitea_runner_registered.stat.exists or gitea_runner_force_reregister | bool register: gitea_runner_register_output - changed_when: "'already exists' not in gitea_runner_register_output.stdout | default('')" + changed_when: >- + gitea_runner_register_output.rc == 0 and + ('already exists' not in gitea_runner_register_output.stdout | default('')) timeout: 60 + +- name: Ensure runner service is running after registration + ansible.builtin.command: systemctl --user start gitea-runner + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + changed_when: true + when: + - gitea_runner_systemd_available.stat.exists + - gitea_runner_docker_rootless_setup + - gitea_runner_register_output is defined + - gitea_runner_register_output.rc | default(1) == 0 diff --git a/ansible/roles/gitea_runner/tasks/rootless_docker.yml b/ansible/roles/gitea_runner/tasks/rootless_docker.yml index 42ae55f..8f087dc 100644 --- a/ansible/roles/gitea_runner/tasks/rootless_docker.yml +++ b/ansible/roles/gitea_runner/tasks/rootless_docker.yml @@ -53,72 +53,6 @@ register: gitea_runner_docker_install when: ansible_facts['os_family'] == 'Debian' -# Docker 28.x vendors containerd v2.1.x. containerd.io >= 2.3 ships a shim that -# returns a protobuf BootstrapResult the vendored 2.1.x code cannot parse, causing -# "failed to create TTRPC connection: unsupported protocol" on every container start. -# Detect the mismatch and downgrade containerd.io to the latest compatible 2.2.x. -- name: Check installed Docker CE version (Debian/Ubuntu) - ansible.builtin.command: dpkg-query -W -f='${Version}' docker-ce - register: gitea_runner_docker_version_check - changed_when: false - when: ansible_facts['os_family'] == 'Debian' - -- name: Check installed containerd.io version (Debian/Ubuntu) - ansible.builtin.shell: "set -o pipefail; dpkg-query -W -f='${Version}' containerd.io 2>/dev/null | cut -d: -f2 | cut -d- -f1" - args: - executable: /bin/bash - register: gitea_runner_containerd_version_check - changed_when: false - when: ansible_facts['os_family'] == 'Debian' - -- name: Determine if containerd.io is incompatible with installed Docker - ansible.builtin.set_fact: - gitea_runner_containerd_needs_downgrade: >- - {{ - gitea_runner_containerd_version_check.stdout.split('.')[0] | int > gitea_runner_containerd_max_compatible_major - or ( - gitea_runner_containerd_version_check.stdout.split('.')[0] | int == gitea_runner_containerd_max_compatible_major - and gitea_runner_containerd_version_check.stdout.split('.')[1] | int > gitea_runner_containerd_max_compatible_minor - ) - }} - gitea_runner_docker_major: "{{ gitea_runner_docker_version_check.stdout.split('.')[0] | default('0') | int }}" - when: ansible_facts['os_family'] == 'Debian' - -- name: Find latest compatible containerd.io version (Debian/Ubuntu) - ansible.builtin.shell: | - set -o pipefail - apt-cache madison containerd.io \ - | awk -F'|' '{print $2}' \ - | tr -d ' ' \ - | grep -E '^{{ gitea_runner_containerd_max_compatible_major }}\.{{ gitea_runner_containerd_max_compatible_minor }}\.' \ - | head -1 - args: - executable: /bin/bash - register: gitea_runner_containerd_compatible_version - changed_when: false - when: - - ansible_facts['os_family'] == 'Debian' - - gitea_runner_containerd_needs_downgrade | default(false) - - gitea_runner_docker_major | int < 29 - -- name: Downgrade containerd.io to compatible version (Debian/Ubuntu) - ansible.builtin.apt: - name: "containerd.io={{ gitea_runner_containerd_compatible_version.stdout }}" - state: present - allow_downgrades: true - register: gitea_runner_containerd_downgrade - when: - - ansible_facts['os_family'] == 'Debian' - - gitea_runner_containerd_needs_downgrade | default(false) - - gitea_runner_docker_major | int < 29 - - gitea_runner_containerd_compatible_version.stdout | length > 0 - -- name: Hold containerd.io package to prevent auto-upgrade (Debian/Ubuntu) - ansible.builtin.dpkg_selections: - name: containerd.io - selection: hold - when: ansible_facts['os_family'] == 'Debian' - - name: Update pacman cache (Arch Linux) community.general.pacman: update_cache: true diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index 5a3f1e3..fb546f9 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -5,8 +5,19 @@ Description=Docker prune for Gitea runner resources Type=oneshot Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} -ExecStart=/usr/bin/docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until={{ gitea_runner_prune_until }}" -ExecStart=/usr/bin/docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" -ExecStart=/usr/bin/docker container prune -f -ExecStart=/usr/bin/docker network prune -f +# Force-remove stale containers (including running ones) left behind by failed +# molecule tests. "docker container prune -f" only removes stopped containers, +# so running containers from crashed/interrupted CI jobs accumulate indefinitely, +# consuming disk and memory. We stop+rm everything first, then prune the rest. +# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing +# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors. +# Only remove containers older than 1 hour (grep for "hour/day/week/month/year +# ago" in RunningFor) to avoid killing molecule test containers that CI jobs +# are actively using. +ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' +ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes +# Prune networks older than the prune-until threshold to avoid removing +# networks that molecule tests are actively creating (e.g. 'traefik' network +# created during molecule create phase before containers are attached). +ExecStart=/usr/bin/docker network prune -f --filter "until={{ gitea_runner_prune_until }}" ExecStart=/usr/bin/docker builder prune -f diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index d8b865e..16432ba 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -44,17 +44,203 @@ if [[ "$runner_state" != "active" ]]; then echo "RECOVERED: gitea-runner service restarted successfully" fi +# 2b. Detect unregistered runner state. When Gitea no longer recognizes the +# runner (e.g., server restore, runner record deleted, Gitea restart with +# token salt change), the runner logs "unregistered runner" every few seconds. +# A service restart will not fix this; re-registration is required. +# +# Detection method: query the Gitea API to verify the runner's UUID still +# exists. This is more reliable than parsing journal logs (which requires +# journal access permissions that runner users may not have — see the +# 2026-08-08 incident where journalctl --user returned "No journal files +# were opened due to insufficient permissions" for all runner users, +# causing the healthcheck to always report "OK: runner healthy" even +# though all runners were unregistered). +{% if gitea_runner_auto_recover_api_token %} +# Auto-recovery is enabled: fetch a new registration token from the Gitea API +# and re-register the runner automatically. A cooldown prevents infinite loops. +GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token" +COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown" +COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }} +GITEA_URL="{{ gitea_url }}" +RUNNER_NAME="{{ gitea_runner_name }}" +RUNNER_LABELS="{{ gitea_runner_labels }}" +BINARY="{{ gitea_runner_binary_path }}" +RUNNER_FILE="{{ gitea_runner_data_dir }}/.runner" +{% endif %} +runner_unregistered=0 + +# Primary detection: query the Gitea API to check if the runner's ID +# still exists in Gitea's runner list. This works regardless of journal +# permissions. +{% if gitea_runner_auto_recover_api_token %} +if [[ -f "$GITEA_API_TOKEN_FILE" && -f "$RUNNER_FILE" ]]; then + API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true) + RUNNER_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true) + if [[ -n "$API_TOKEN" && -n "$RUNNER_ID" ]]; then + # List all runners and check if our ID is present + runner_found=$(curl -sf --connect-timeout 5 --max-time 10 \ + -H "Authorization: token $API_TOKEN" \ + "${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \ + | python3 -c " +import sys, json +try: + data = json.load(sys.stdin) + runners = data if isinstance(data, list) else data.get('runners', []) + ids = [str(r.get('id', '')) for r in runners] + print('1' if '$RUNNER_ID' in ids else '0') +except Exception: + print('0') +" 2>/dev/null || echo "0") + if [[ "$runner_found" != "1" ]]; then + runner_unregistered=1 + echo "CRITICAL: runner ID $RUNNER_ID not found in Gitea (unregistered)." + fi + fi +fi +{% endif %} + +# Fallback detection: check journal logs (if accessible) +if [[ "$runner_unregistered" -eq 0 ]]; then + recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true) + if [[ "$recent_errors" -ge 3 ]]; then + runner_unregistered=1 + echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)." + fi +fi + +if [[ "$runner_unregistered" -ge 1 ]]; then +{% if gitea_runner_auto_recover_api_token %} + # Check cooldown — skip if we recently attempted recovery + if [[ -f "$COOLDOWN_FILE" ]]; then + last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0) + now=$(date +%s) + elapsed=$((now - last_attempt)) + if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then + echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later." + exit 3 + fi + fi + + # Mark attempt time BEFORE trying (so failures also get cooldown) + date +%s > "$COOLDOWN_FILE" 2>/dev/null || true + + # Read the API token + if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then + echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required." + exit 3 + fi + API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true) + if [[ -z "$API_TOKEN" ]]; then + echo "ERROR: auto-recover token file is empty. Manual re-registration required." + exit 3 + fi + + echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..." + + # Fetch a new registration token from the Gitea API + # Try org-level first (for org-scoped runners), then instance-level + REG_TOKEN="" + for endpoint in \ + "api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \ + "api/v1/admin/actions/runners/registration-token"; do + REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \ + -H "Authorization: token $API_TOKEN" \ + "${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true) + if [[ -n "$REG_TOKEN" ]]; then + echo "INFO: fetched registration token from ${endpoint}" + break + fi + done + + if [[ -z "$REG_TOKEN" ]]; then + echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?" + exit 3 + fi + + # Stop the runner service + systemctl --user stop gitea-runner.service 2>/dev/null || true + sleep 1 + + # Remove the stale .runner file + rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true + + # Re-register + cd "{{ gitea_runner_data_dir }}" + if "$BINARY" register \ + --token "$REG_TOKEN" \ + --name "$RUNNER_NAME" \ + --instance "$GITEA_URL" \ + --labels "$RUNNER_LABELS" \ + --no-interactive 2>&1; then + echo "RECOVERED: runner re-registered successfully" + else + echo "ERROR: re-registration failed. Manual intervention required." + exit 3 + fi + + # Start the runner service + systemctl --user start gitea-runner.service + sleep 3 + + # Verify recovery — query the Gitea API to confirm the new ID is registered + NEW_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true) + if [[ -n "$NEW_ID" ]]; then + new_found=$(curl -sf --connect-timeout 5 --max-time 10 \ + -H "Authorization: token $API_TOKEN" \ + "${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \ + | python3 -c " +import sys, json +try: + data = json.load(sys.stdin) + runners = data if isinstance(data, list) else data.get('runners', []) + ids = [str(r.get('id', '')) for r in runners] + print('1' if '$NEW_ID' in ids else '0') +except Exception: + print('0') +" 2>/dev/null || echo "0") + if [[ "$new_found" == "1" ]]; then + echo "OK: runner recovered and registered with new ID $NEW_ID" + # Clear cooldown on success + rm -f "$COOLDOWN_FILE" 2>/dev/null || true + else + echo "WARN: runner re-registered but ID not found in Gitea API. Will retry after cooldown." + exit 3 + fi + else + echo "WARN: could not read new .runner file after re-registration. Will retry after cooldown." + exit 3 + fi +{% else %} + echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true." + # Restart the service once in case it is a transient token refresh issue, + # but this cannot recover an unregistered runner without re-registration. + systemctl --user restart gitea-runner.service + sleep 2 + exit 3 +{% endif %} +fi + # 3. Check disk space — prune aggressively if below threshold disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources" - docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true - docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true - # Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality) - docker image prune -f || true - # Clean up stopped containers and dangling networks that accumulate from failed jobs - docker container prune -f || true - docker network prune -f || true + # Force-remove stale containers (including running ones from failed molecule tests) + # that are older than 1 hour. "docker container prune -f" only removes stopped + # containers, so running containers from crashed CI jobs accumulate and consume + # disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). + # Only remove containers older than 1 hour to avoid killing molecule test + # containers that CI jobs are actively using. + docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \ + | grep -v 'GITEA-ACTIONS-TASK' \ + | grep -E '(hour|day|week|month|year)s? ago' \ + | awk '{print $1}' \ + | xargs -r docker rm -f 2>/dev/null || true + docker system prune -af --filter "until=1h" --volumes || true + # Prune networks older than 1 hour to avoid removing networks that + # molecule tests are actively creating (e.g. 'traefik' network created + # during molecule create phase before containers are attached). + docker network prune -f --filter "until=1h" || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') echo "INFO: Disk usage after prune: ${disk_pct}%" fi diff --git a/ansible/start-runner.yml b/ansible/start-runner.yml index 9d4c00e..7f9147c 100644 --- a/ansible/start-runner.yml +++ b/ansible/start-runner.yml @@ -33,5 +33,5 @@ become_user: "{{ gitea_runner_service_user }}" environment: XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - when: systemd_available.stat.exists + when: gitea_runner_systemd_available.stat.exists changed_when: true diff --git a/docs/index.md b/docs/index.md index 9080af3..2c6fe2d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE) -[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki) -[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) -[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases) -[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/python.svg)](https://www.python.org/downloads/) +[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki) +[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions) +[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases) +[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/python.svg)](https://www.python.org/downloads/) ## Overview diff --git a/pyproject.toml b/pyproject.toml index f3184bd..4ddd754 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ ci = [ "build==1.5.1", "twine==6.2.0", # Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1", ] # Lint and type-checking tools (validate job) lint = [ @@ -56,7 +56,7 @@ molecule = [ dev = [ "grm[ci,lint,molecule]", # Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1", # Non-Python dev dependency: checkmake (Makefile linter) # Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest ] diff --git a/scripts/cleanup_stale_runners.py b/scripts/cleanup_stale_runners.py new file mode 100644 index 0000000..c172fe3 --- /dev/null +++ b/scripts/cleanup_stale_runners.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Clean up stale runner registrations from Gitea. + +A runner is considered stale if it hasn't been online for more than a +configurable threshold (default: 1 hour). Stale runners accumulate when: +- A runner host is rebuilt or re-provisioned (old registration remains) +- A runner is re-registered (old entry remains alongside the new one) +- A runner process dies and the healthcheck can't auto-recover + +This script queries the Gitea API for all runners, identifies stale ones, +and deletes them via ``DELETE /api/v1/admin/actions/runners/{id}``. + +Usage:: + + python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token + python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token --dry-run + python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token \\ + --stale-threshold 3600 +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request # noqa: PTH123 # nosec B404 +from typing import Any + + +def _api_request(base_url: str, token: str, method: str, path: str) -> Any: + url = f"{base_url.rstrip('/')}/api/v1{path}" + req = urllib.request.Request(url, method=method) # nosec B310 + req.add_header("Authorization", f"token {token}") + req.add_header("Accept", "application/json") + try: + with urllib.request.urlopen(req) as resp: # noqa: PTH123 # nosec B310 + if resp.status == 204: + return None + raw = resp.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Gitea API error {e.code}: {detail}") from e + + +def list_runners(base_url: str, token: str) -> list[dict[str, Any]]: + data = _api_request(base_url, token, "GET", "/admin/actions/runners") + if data is None: + return [] + if isinstance(data, list): + return data + if isinstance(data, dict): + return data.get("runners", []) + return [] + + +def delete_runner(base_url: str, token: str, runner_id: int) -> bool: + try: + _api_request(base_url, token, "DELETE", f"/admin/actions/runners/{runner_id}") + return True + except RuntimeError as e: + print(f" ERROR deleting runner {runner_id}: {e}", file=sys.stderr) + return False + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Clean up stale Gitea runner registrations") + parser.add_argument("--gitea-url", required=True, help="Gitea base URL") + parser.add_argument("--token", required=True, help="Gitea admin API token") + parser.add_argument( + "--stale-threshold", + type=int, + default=3600, + help="Seconds since last_online before a runner is considered stale (default: 3600 = 1h)", + ) + parser.add_argument("--dry-run", action="store_true", help="List stale runners without deleting") + args = parser.parse_args(argv) + + runners = list_runners(args.gitea_url, args.token) + if not runners: + print("No runners found.") + return 0 + + now = int(time.time()) + stale: list[dict[str, Any]] = [] + online: list[dict[str, Any]] = [] + + for runner in runners: + last_online = runner.get("last_online", 0) or 0 + seconds_since = now - last_online + runner["seconds_since_online"] = seconds_since + if seconds_since > args.stale_threshold: + stale.append(runner) + else: + online.append(runner) + + print(f"Total runners: {len(runners)}") + print(f"Online (within {args.stale_threshold}s): {len(online)}") + print(f"Stale (>{args.stale_threshold}s): {len(stale)}") + print() + + if not stale: + print("No stale runners to clean up.") + return 0 + + print("Stale runners:") + for r in stale: + rid = r.get("id", "?") + name = r.get("name", "?") + uuid = r.get("uuid", "?")[:8] + secs = r.get("seconds_since_online", 0) + hours = secs / 3600 + print(f" id={rid} name={name} uuid={uuid}... offline={hours:.1f}h ago") + + if args.dry_run: + print("\n--dry-run: not deleting. Remove --dry-run to clean up.") + return 0 + + print(f"\nDeleting {len(stale)} stale runners...") + deleted = 0 + for r in stale: + rid = r.get("id") + if rid is None: + continue + if delete_runner(args.gitea_url, args.token, rid): + deleted += 1 + print(f" Deleted runner id={rid} ({r.get('name', '?')})") + + print(f"\nDone: {deleted}/{len(stale)} stale runners deleted.") + return 0 if deleted == len(stale) else 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/scripts/tests/test_cleanup_stale_runners.py b/scripts/tests/test_cleanup_stale_runners.py new file mode 100644 index 0000000..8548535 --- /dev/null +++ b/scripts/tests/test_cleanup_stale_runners.py @@ -0,0 +1,217 @@ +"""Tests for cleanup_stale_runners.py.""" + +from __future__ import annotations + +import time +from unittest.mock import MagicMock, patch + +from scripts.cleanup_stale_runners import ( + _api_request, + delete_runner, + list_runners, + main, +) + + +class TestListRunners: + """Tests for list_runners().""" + + @patch("scripts.cleanup_stale_runners._api_request") + def test_returns_list_of_runners(self, mock_req: MagicMock) -> None: + mock_req.return_value = [{"id": 1, "name": "runner-1"}, {"id": 2, "name": "runner-2"}] + result = list_runners("https://git.example.com", "token") + assert len(result) == 2 + assert result[0]["id"] == 1 + + @patch("scripts.cleanup_stale_runners._api_request") + def test_returns_empty_on_none(self, mock_req: MagicMock) -> None: + mock_req.return_value = None + result = list_runners("https://git.example.com", "token") + assert result == [] + + @patch("scripts.cleanup_stale_runners._api_request") + def test_extracts_runners_from_dict(self, mock_req: MagicMock) -> None: + mock_req.return_value = {"runners": [{"id": 1}]} + result = list_runners("https://git.example.com", "token") + assert len(result) == 1 + assert result[0]["id"] == 1 + + @patch("scripts.cleanup_stale_runners._api_request") + def test_returns_empty_on_non_list_non_dict(self, mock_req: MagicMock) -> None: + mock_req.return_value = "not a list" + result = list_runners("https://git.example.com", "token") + assert result == [] + + +class TestDeleteRunner: + """Tests for delete_runner().""" + + @patch("scripts.cleanup_stale_runners._api_request") + def test_returns_true_on_success(self, mock_req: MagicMock) -> None: + mock_req.return_value = None + assert delete_runner("https://git.example.com", "token", 42) is True + + @patch("scripts.cleanup_stale_runners._api_request") + def test_returns_false_on_error(self, mock_req: MagicMock) -> None: + mock_req.side_effect = RuntimeError("API error 404: not found") + assert delete_runner("https://git.example.com", "token", 42) is False + + +class TestApiRequest: + """Tests for _api_request().""" + + @patch("scripts.cleanup_stale_runners.urllib.request.urlopen") + def test_returns_json_on_success(self, mock_urlopen: MagicMock) -> None: + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b'{"key": "value"}' + mock_urlopen.return_value.__enter__.return_value = mock_resp + result = _api_request("https://git.example.com", "token", "GET", "/test") + assert result == {"key": "value"} + + @patch("scripts.cleanup_stale_runners.urllib.request.urlopen") + def test_returns_none_on_204(self, mock_urlopen: MagicMock) -> None: + mock_resp = MagicMock() + mock_resp.status = 204 + mock_urlopen.return_value.__enter__.return_value = mock_resp + result = _api_request("https://git.example.com", "token", "DELETE", "/test/1") + assert result is None + + @patch("scripts.cleanup_stale_runners.urllib.request.urlopen") + def test_returns_none_on_empty_body(self, mock_urlopen: MagicMock) -> None: + mock_resp = MagicMock() + mock_resp.status = 200 + mock_resp.read.return_value = b"" + mock_urlopen.return_value.__enter__.return_value = mock_resp + result = _api_request("https://git.example.com", "token", "GET", "/test") + assert result is None + + @patch("scripts.cleanup_stale_runners.urllib.request.urlopen") + def test_raises_on_http_error(self, mock_urlopen: MagicMock) -> None: + import urllib.error + + mock_error = urllib.error.HTTPError( + "url", + 404, + "Not Found", + {}, + None, + ) + mock_error.read = MagicMock(return_value=b'{"message": "not found"}') + mock_urlopen.side_effect = mock_error + import pytest + + with pytest.raises(RuntimeError, match="404"): + _api_request("https://git.example.com", "token", "GET", "/test") + + +class TestMain: + """Tests for main().""" + + @patch("scripts.cleanup_stale_runners.list_runners") + def test_no_runners(self, mock_list: MagicMock) -> None: + mock_list.return_value = [] + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + assert rc == 0 + + @patch("scripts.cleanup_stale_runners.list_runners") + def test_no_stale_runners(self, mock_list: MagicMock) -> None: + now = int(time.time()) + mock_list.return_value = [ + {"id": 1, "name": "runner-1", "last_online": now - 60}, + ] + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + assert rc == 0 + + @patch("scripts.cleanup_stale_runners.list_runners") + def test_dry_run_does_not_delete(self, mock_list: MagicMock) -> None: + now = int(time.time()) + mock_list.return_value = [ + {"id": 1, "name": "runner-1", "last_online": now - 7200}, + ] + with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del: + rc = main( + [ + "--gitea-url", + "https://git.example.com", + "--token", + "t", + "--dry-run", + ] + ) + assert rc == 0 + mock_del.assert_not_called() + + @patch("scripts.cleanup_stale_runners.list_runners") + @patch("scripts.cleanup_stale_runners.delete_runner") + def test_deletes_stale_runners(self, mock_del: MagicMock, mock_list: MagicMock) -> None: + now = int(time.time()) + mock_list.return_value = [ + {"id": 1, "name": "runner-1", "last_online": now - 60}, + {"id": 2, "name": "runner-2", "last_online": now - 7200}, + {"id": 3, "name": "runner-3", "last_online": now - 9999}, + ] + mock_del.return_value = True + rc = main( + [ + "--gitea-url", + "https://git.example.com", + "--token", + "t", + "--stale-threshold", + "3600", + ] + ) + assert rc == 0 + assert mock_del.call_count == 2 + + @patch("scripts.cleanup_stale_runners.list_runners") + @patch("scripts.cleanup_stale_runners.delete_runner") + def test_returns_1_on_partial_failure(self, mock_del: MagicMock, mock_list: MagicMock) -> None: + now = int(time.time()) + mock_list.return_value = [ + {"id": 1, "name": "runner-1", "last_online": now - 7200}, + {"id": 2, "name": "runner-2", "last_online": now - 7200}, + ] + mock_del.side_effect = [True, False] + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + assert rc == 1 + + @patch("scripts.cleanup_stale_runners.list_runners") + def test_runner_with_zero_last_online(self, mock_list: MagicMock) -> None: + """Runners with last_online=0 should be considered stale.""" + mock_list.return_value = [ + {"id": 1, "name": "runner-1", "last_online": 0}, + ] + with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del: + mock_del.return_value = True + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + assert rc == 0 + mock_del.assert_called_once() + + @patch("scripts.cleanup_stale_runners.list_runners") + def test_runner_with_missing_last_online(self, mock_list: MagicMock) -> None: + """Runners with missing last_online should be considered stale.""" + mock_list.return_value = [ + {"id": 1, "name": "runner-1"}, + ] + with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del: + mock_del.return_value = True + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + assert rc == 0 + mock_del.assert_called_once() + + @patch("scripts.cleanup_stale_runners.list_runners") + @patch("scripts.cleanup_stale_runners.delete_runner") + def test_skips_runner_with_none_id(self, mock_del: MagicMock, mock_list: MagicMock) -> None: + """Runners with id=None should be skipped during deletion.""" + now = int(time.time()) + mock_list.return_value = [ + {"id": None, "name": "bad-runner", "last_online": now - 7200}, + {"id": 2, "name": "runner-2", "last_online": now - 7200}, + ] + mock_del.return_value = True + rc = main(["--gitea-url", "https://git.example.com", "--token", "t"]) + # 1/2 deleted (None id skipped), so rc=1 (partial) + assert rc == 1 + mock_del.assert_called_once_with("https://git.example.com", "t", 2) diff --git a/src/grm/cli.py b/src/grm/cli.py index 67120d3..ecb750f 100644 --- a/src/grm/cli.py +++ b/src/grm/cli.py @@ -145,11 +145,25 @@ def cli(ctx: click.Context, become_password_file: str | None, verbose: bool) -> "Example: docker:docker://alpine:latest" ), ) +@click.option( + "--force-reregister/--no-force-reregister", + default=False, + help=_("Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)"), +) @click.option( "--ask-become-pass/--no-ask-become-pass", default=True, help=_("Prompt for sudo password (default)"), ) +@click.option( + "--auto-recover-token", + default=None, + help=_( + "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). " + "When set, the healthcheck can automatically re-register the runner " + "if it becomes unregistered. Requires admin or org-level access." + ), +) @_handle_errors("Installation failed: {error}") def install( host: str, @@ -161,10 +175,14 @@ def install( admin_token: str | None, integration_retries: int, labels: str | None, + force_reregister: bool, ask_become_pass: bool, + auto_recover_token: str | None, ) -> None: if labels is None: labels = os.getenv("GITEA_RUNNER_LABELS") + if auto_recover_token is None: + auto_recover_token = os.getenv("GITEA_AUTO_RECOVER_TOKEN") manager = RunnerManager() manager.install( host=host, @@ -176,9 +194,11 @@ def install( admin_token=admin_token, integration_retries=integration_retries, labels=labels, + force_reregister=force_reregister, ask_become_pass=ask_become_pass, become_password_file=_get_become_password_file(), verbose=_get_verbose(), + auto_recover_token=auto_recover_token, ) @@ -506,7 +526,7 @@ def list_runners(ask_become_pass: bool, no_status: bool) -> None: click.echo(f"{_('NAME'):<18} {_('HOST'):<16} {_('USER'):<10} {_('LABELS'):<30} {_('STATUS')}") click.echo("-" * 90) for r in runners: - click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {r['labels']:<30} {r['status']}") + click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {(r['labels'] or ''):<30} {r['status']}") @cli.command(name="trigger-workflow", help=_("Trigger a Gitea Actions workflow via the API.")) diff --git a/src/grm/runner_manager.py b/src/grm/runner_manager.py index 6542c91..5eed2a2 100644 --- a/src/grm/runner_manager.py +++ b/src/grm/runner_manager.py @@ -87,8 +87,10 @@ class RunnerManager: integration_retries: int = 3, ask_become_pass: bool = False, labels: str | None = None, + force_reregister: bool = False, become_password_file: str | None = None, verbose: bool = False, + auto_recover_token: str | None = None, ) -> None: """Install a runner on a remote host using Ansible.""" if not name: @@ -98,16 +100,19 @@ class RunnerManager: if not token: raise AnsibleError(_("GITEA_REGISTRATION_TOKEN must be set (or pass --token)")) - extra_vars: dict[str, str | int] = { + extra_vars: dict[str, str | int | bool] = { "registration_token": token, "gitea_runner_name": name, "gitea_url": gitea_url, "gitea_runner_integration_retries": integration_retries, + "gitea_runner_force_reregister": force_reregister, } if admin_token: extra_vars["gitea_admin_token"] = admin_token if labels is not None: extra_vars["gitea_runner_labels"] = labels + if auto_recover_token: + extra_vars["gitea_runner_auto_recover_api_token"] = auto_recover_token with track_steps() as tracker: tracker.begin(_("Installing Gitea Runner on {host}", host=host)) diff --git a/src/grm/translations.json b/src/grm/translations.json index c7555b9..48f8660 100644 --- a/src/grm/translations.json +++ b/src/grm/translations.json @@ -367,6 +367,14 @@ "ru": "Токен регистрации (env: GITEA_REGISTRATION_TOKEN)", "zh": "注册令牌(环境变量: GITEA_REGISTRATION_TOKEN)" }, + "Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)": { + "bg": "Принудителна повторна регистрация, дори ако .runner файлът съществува (env: GITEA_FORCE_REREGISTER)", + "de": "Erneute Registrierung erzwingen, auch wenn .runner-Datei existiert (env: GITEA_FORCE_REREGISTER)", + "en": "Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)", + "pl": "Wymuś ponowną rejestrację, nawet jeśli plik .runner istnieje (env: GITEA_FORCE_REREGISTER)", + "ru": "Принудительно повторно зарегистрировать, даже если файл .runner существует (env: GITEA_FORCE_REREGISTER)", + "zh": "即使存在 .runner 文件也强制重新注册(环境变量: GITEA_FORCE_REREGISTER)" + }, "Remove a registered Gitea Runner completely.": { "bg": "Пълно премахване на регистриран Gitea Runner.", "de": "Einen registrierten Gitea Runner vollständig entfernen.", @@ -774,5 +782,13 @@ "pl": "Workflow uruchomiony pomyślnie. ID uruchomienia: {run_id}", "ru": "Workflow успешно запущен. ID запуска: {run_id}", "zh": "工作流触发成功。运行 ID:{run_id}" + }, + "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.": { + "bg": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.", + "de": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.", + "en": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.", + "pl": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.", + "ru": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.", + "zh": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access." } } diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 840b938..b63fa6a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -50,9 +50,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -73,9 +75,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=False, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -114,9 +118,11 @@ class TestCLI: admin_token=None, integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -165,9 +171,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -188,9 +196,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -226,9 +236,92 @@ class TestCLI: admin_token="", integration_retries=3, labels="docker:docker://alpine:latest", + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, + ) + + @patch("grm.cli.RunnerManager") + def test_install_force_reregister(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + mock_manager_class.return_value = mock_manager + + runner = CliRunner(env=_TEST_ENV) + result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok", "--force-reregister"]) + assert result.exit_code == 0 + mock_manager.install.assert_called_once_with( + host="host1", + user="ubuntu", + key=None, + name=None, + token="tok", + gitea_url="https://git.example.com", + admin_token="", + integration_retries=3, + labels=None, + force_reregister=True, + ask_become_pass=True, + become_password_file=None, + verbose=False, + auto_recover_token=None, + ) + + @patch("grm.cli.RunnerManager") + def test_install_with_auto_recover_token(self, mock_manager_class: MagicMock) -> None: + """--auto-recover-token passes the token to the manager for healthcheck auto-recovery.""" + mock_manager = MagicMock() + mock_manager_class.return_value = mock_manager + + runner = CliRunner(env=_TEST_ENV) + result = runner.invoke( + cli, + ["install", "host1", "--user", "ubuntu", "--token", "tok", "--auto-recover-token", "api-tok"], + ) + assert result.exit_code == 0 + mock_manager.install.assert_called_once_with( + host="host1", + user="ubuntu", + key=None, + name=None, + token="tok", + gitea_url="https://git.example.com", + admin_token="", + integration_retries=3, + labels=None, + force_reregister=False, + ask_become_pass=True, + become_password_file=None, + verbose=False, + auto_recover_token="api-tok", + ) + + @patch("grm.cli.RunnerManager") + def test_install_auto_recover_token_from_env(self, mock_manager_class: MagicMock) -> None: + """GITEA_AUTO_RECOVER_TOKEN env var is used when --auto-recover-token is not passed.""" + mock_manager = MagicMock() + mock_manager_class.return_value = mock_manager + + env = {**_TEST_ENV, "GITEA_AUTO_RECOVER_TOKEN": "env-tok"} + runner = CliRunner(env=env) + result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok"]) + assert result.exit_code == 0 + mock_manager.install.assert_called_once_with( + host="host1", + user="ubuntu", + key=None, + name=None, + token="tok", + gitea_url="https://git.example.com", + admin_token="", + integration_retries=3, + labels=None, + force_reregister=False, + ask_become_pass=True, + become_password_file=None, + verbose=False, + auto_recover_token="env-tok", ) @patch("grm.cli.RunnerManager") @@ -250,9 +343,11 @@ class TestCLI: admin_token="", integration_retries=3, labels="", + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -274,9 +369,11 @@ class TestCLI: admin_token="", integration_retries=3, labels="docker:docker://alpine:latest", + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=False, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -307,9 +404,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=pw_file, verbose=False, + auto_recover_token=None, ) finally: import os @@ -334,9 +433,11 @@ class TestCLI: admin_token="", integration_retries=3, labels=None, + force_reregister=False, ask_become_pass=True, become_password_file=None, verbose=True, + auto_recover_token=None, ) @patch("grm.cli.RunnerManager") @@ -741,6 +842,27 @@ class TestCLI: assert "active" in result.output mock_manager.list_runners.assert_called_once_with(become_pass=None, no_status=False) + @patch("grm.cli.RunnerManager") + def test_list_with_none_labels(self, mock_manager_class: MagicMock) -> None: + """Runners with labels=None should not crash the list command.""" + mock_manager = MagicMock() + mock_manager.list_runners.return_value = [ + { + "name": "r1", + "host": "10.0.0.1", + "user": "ubuntu", + "labels": None, + "status": "active", + }, + ] + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["list"]) + assert result.exit_code == 0 + assert "r1" in result.output + assert "active" in result.output + @patch("grm.cli.RunnerManager") def test_list_no_status(self, mock_manager_class: MagicMock) -> None: """--no-status skips SSH checks and shows registry only.""" diff --git a/tests/unit/test_runner_manager.py b/tests/unit/test_runner_manager.py index 9febca3..8ba6efa 100644 --- a/tests/unit/test_runner_manager.py +++ b/tests/unit/test_runner_manager.py @@ -51,6 +51,7 @@ class TestRunnerManager: assert manager._captured_extra_vars["registration_token"] == "tok" assert manager._captured_extra_vars["gitea_runner_name"] == "192.168.1.10" assert manager._captured_extra_vars["gitea_url"] == "https://git.example.com" + assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False assert "Installing Gitea Runner on 192.168.1.10" in mock_executor.run.call_args.kwargs["description"] mock_registry.add.assert_called_once_with( name="192.168.1.10", @@ -76,6 +77,7 @@ class TestRunnerManager: assert "/key" in cmd_str assert manager._captured_extra_vars["registration_token"] == "preset" assert manager._captured_extra_vars["gitea_runner_name"] == "my-runner" + assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False assert "--ask-become-pass" not in cmd_str mock_registry.add.assert_called_once_with( name="my-runner", @@ -97,6 +99,52 @@ class TestRunnerManager: cmd_str = " ".join(cmd) assert "--ask-become-pass" in cmd_str + def test_install_force_reregister(self) -> None: + mock_registry = MagicMock() + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + manager._executor = mock_executor + + manager.install( + "host1", + "root", + token="tok", + gitea_url="https://git.example.com", + force_reregister=True, + ) + assert manager._captured_extra_vars["gitea_runner_force_reregister"] is True + + def test_install_with_auto_recover_token(self) -> None: + """auto_recover_token is passed as extra_var to Ansible.""" + mock_registry = MagicMock() + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + manager._executor = mock_executor + + manager.install( + "host1", + "root", + token="tok", + gitea_url="https://git.example.com", + auto_recover_token="api-tok", + ) + assert manager._captured_extra_vars["gitea_runner_auto_recover_api_token"] == "api-tok" + + def test_install_without_auto_recover_token(self) -> None: + """When auto_recover_token is None, the extra_var is not set.""" + mock_registry = MagicMock() + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + manager._executor = mock_executor + + manager.install( + "host1", + "root", + token="tok", + gitea_url="https://git.example.com", + ) + assert "gitea_runner_auto_recover_api_token" not in manager._captured_extra_vars + def test_install_missing_gitea_url(self) -> None: manager = RunnerManager() with pytest.raises(AnsibleError, match="GITEA_URL must be set"):